diff --git a/scripts/collect-report-checks.py b/scripts/collect-report-checks.py new file mode 100644 index 0000000..6e20fb4 --- /dev/null +++ b/scripts/collect-report-checks.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Run lab3 checks and retain one compact result without request logs or secrets.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] + + +def capture(command, timeout=180): + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', errors='replace', timeout=timeout) + + +def snapshot(): + result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30) + result.check_returncode() + ids = result.stdout.split() + inspected = capture(['docker', 'inspect', *ids], 30) + inspected.check_returncode() + rows = [] + for c in json.loads(inspected.stdout): + service = c['Config']['Labels'].get('com.docker.compose.service') + if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}: + continue + pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None) + rows.append({'service': service, 'id': c['Id'], 'image': c['Image'], + 'started_at': c['State']['StartedAt'], 'status': c['State']['Status'], + 'health': c['State'].get('Health', {}).get('Status'), + 'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9, + 'memory_mib': c['HostConfig']['Memory'] / 1024**2, + 'pool_max': pool, 'command': c['Config']['Cmd'], + 'ports': c['HostConfig'].get('PortBindings'), + 'volumes': [{'name': m['Name'], 'destination': m['Destination']} + for m in c['Mounts'] if m['Type'] == 'volume']}) + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json') + parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts') + args = parser.parse_args() + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.retry_failed: + report = json.loads(args.output.read_text(encoding='utf-8')) + if report.get('source_sha256'): + report['source_before_recheck_sha256'] = report['source_sha256'] + selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'} + report['recheck_started'] = datetime.now(timezone.utc).isoformat() + else: + previous = (json.loads(args.output.read_text(encoding='utf-8')) + if args.output.exists() else None) + report = {'recorded_at': datetime.now(timezone.utc).isoformat(), + 'base_url': args.base_url, 'checks': [], 'initial': snapshot()} + if previous: + report['previous_validation'] = { + 'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'), + 'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}), + 'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c} + for c in previous['checks']], + } + selected = None + report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() + for name in ('src/main.ts', 'src/app.controller.ts', + 'src/common/http/fastify-adapter.ts', + 'infra/start-nginx.sh', 'scripts/stop-one.sh', + 'scripts/restart.sh', 'test/check_shutdown.py')} + + def save(): + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + + def run(name, command, timeout=180, details=None, parse_json=False): + if selected is not None and name not in selected and name != 'final-smoke': + return None + print(f'START {name}', flush=True) + started = time.monotonic() + item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()} + try: + result = capture(command, timeout) + item['exit_code'] = result.returncode + item['status'] = 'PASS' if result.returncode == 0 else 'FAIL' + lines = (result.stdout + '\n' + result.stderr).splitlines() + item['summary'] = [line for line in lines if line.startswith( + ('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:] + if details is not None and details.exists(): + item['details'] = json.loads(details.read_text(encoding='utf-8')) + elif parse_json: + for pos in (i for i, char in enumerate(result.stdout) if char == '{'): + try: + item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:]) + break + except json.JSONDecodeError: + continue + except (OSError, subprocess.TimeoutExpired) as exc: + item.update(status='FAIL', error=type(exc).__name__) + item['seconds'] = round(time.monotonic() - started, 3) + previous = next((c for c in report['checks'] if c['name'] == name), None) + if previous is not None: + item['previous_attempts'] = previous.get('previous_attempts', []) + [ + {key: value for key, value in previous.items() if key != 'previous_attempts'}] + report['checks'][report['checks'].index(previous)] = item + else: + report['checks'].append(item) + save() + print(f"{item['status']} {name} ({item['seconds']}s)", flush=True) + for line in item.get('summary', [])[-3:]: + print(line, flush=True) + return item + + def restore(): + result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml', + 'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180) + result.check_returncode() + time.sleep(6) + + py = [sys.executable, '-X', 'utf8'] + pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm' + hook_suffix = 'cmd' if os.name == 'nt' else 'sh' + save() + with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp: + temp = Path(tmp) + run('contract', py + ['scripts/check_contract.py']) + run('lint', [pnpm, 'lint']) + run('build', [pnpm, 'build'], 240) + run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120) + for name, script in (('nginx-readiness', 'nginx-readiness.sh'), + ('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')): + run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh', + '-v', f'{ROOT / "infra"}:/source:ro', + '-v', f'{ROOT / "test" / script}:/test.sh:ro', + 'nginx:1.30.6-alpine', '/test.sh'], 60) + run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url]) + run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T', + '-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro', + 'api', 'node', '--test', 'test/idempotency.mjs'], 180) + if selected is None or selected & {'persistence', 'public-failover'}: + try: + if selected is None or 'persistence' in selected: + before = snapshot() + item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence', + '--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240) + after = snapshot() + item['before'] = before + item['after'] = after + restarted = {c['id']: c for c in before if c['service'] != 'migrate'} + same = {c['id']: c for c in after if c['service'] != 'migrate'} + item['containers_restarted'] = restarted.keys() == same.keys() and all( + c['started_at'] != same[key]['started_at'] for key, c in restarted.items()) + item['volumes_preserved'] = restarted.keys() == same.keys() and all( + c['volumes'] == same[key]['volumes'] for key, c in restarted.items()) + if not item['containers_restarted'] or not item['volumes_preserved']: + item['status'] = 'FAIL' + save() + run('public-failover', py + ['tests/resilience.py', '--mode', 'failover', + '--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120, + parse_json=True) + finally: + restore() + for scenario in ('resources', 'failover', 'rolling'): + result_file = temp / f'grade4-{scenario}.json' + run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario, + '--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10', + '--output', str(result_file)], 300, details=result_file) + run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True) + images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest', + 'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest'] + if selected is None or 'migration-compatibility' in selected: + available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30) + if available.returncode == 0: + run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True) + else: + report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN', + 'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'}) + run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url]) + report['final'] = snapshot() + report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks']) + report['finished'] = datetime.now(timezone.utc).isoformat() + save() + print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True) + if not report['all_executed_checks_pass']: + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/test-persistence.py b/scripts/test-persistence.py new file mode 100644 index 0000000..e55872f --- /dev/null +++ b/scripts/test-persistence.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = {'backend', 'postgres', 'redis'} + + +def snapshot(): + ids = subprocess.check_output( + ['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)], + cwd=ROOT, text=True, timeout=30, + ).split() + if len(ids) != len(SERVICES): + raise RuntimeError('Start backend, postgres and redis before this test') + template = ( + '{"service":{{json (index .Config.Labels "com.docker.compose.service")}},' + '"id":{{json .Id}},"started_at":{{json .State.StartedAt}},' + '"running":{{json .State.Running}},"mounts":{{json .Mounts}}}' + ) + output = subprocess.check_output( + ['docker', 'inspect', '--format', template, *ids], + cwd=ROOT, text=True, timeout=30, + ) + result = {} + for line in output.splitlines(): + entry = json.loads(line) + service = entry.pop('service') + entry['volumes'] = [ + {'name': mount['Name'], 'destination': mount['Destination']} + for mount in entry.pop('mounts') if mount['Type'] == 'volume' + ] + result[service] = entry + if set(result) != SERVICES or not all(v['running'] for v in result.values()): + raise RuntimeError('All three services must be running') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080')) + parser.add_argument('--ca-file') + parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public') + parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json') + args = parser.parse_args() + before = snapshot() + hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh') + if args.scenario == 'public': + command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence'] + else: + command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')] + command += ['--action-script', str(hook), '--base-url', args.base_url] + if args.ca_file: + command += ['--ca-file', args.ca_file] + started = time.monotonic() + proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True) + print(proc.stdout, end='') + print(proc.stderr, end='', file=sys.stderr) + if proc.returncode: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps({ + 'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'scenario': args.scenario, + 'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}', + 'hook': hook.relative_to(ROOT).as_posix(), + 'result': 'FAIL', 'exit_code': proc.returncode, + 'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(), + 'before': before, + }, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + return proc.returncode + after = snapshot() + for service in sorted(SERVICES): + old, new = before[service], after[service] + if old['id'] != new['id'] or old['started_at'] == new['started_at']: + raise RuntimeError(f'{service}: expected a restart of the same container') + if not old['volumes'] or old['volumes'] != new['volumes']: + raise RuntimeError(f'{service}: persistent volumes changed or are missing') + evidence = { + 'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'scenario': args.scenario, + 'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}', + 'hook': hook.relative_to(ROOT).as_posix(), + 'elapsed_seconds': round(time.monotonic() - started, 3), + 'result': 'PASS', 'test_output': proc.stdout.strip(), + 'before': before, 'after': after, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('PASS: backend, postgres and redis restarted; named volumes preserved.') + print(f'Evidence: {args.output}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/auth-startup.mjs b/test/auth-startup.mjs new file mode 100644 index 0000000..6bd1c28 --- /dev/null +++ b/test/auth-startup.mjs @@ -0,0 +1,50 @@ +import { createRequire } from 'node:module'; +import { randomUUID } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +const require = createRequire(`${process.cwd()}/package.json`); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require(`${process.cwd()}/dist/app.module`); +const { PrismaService } = require(`${process.cwd()}/dist/prisma/prisma.service`); +const { UserService } = require(`${process.cwd()}/dist/User/user.service`); +const { PasswordService } = require(`${process.cwd()}/dist/Auth/password.service`); +const { createFastifyAdapter } = require(`${process.cwd()}/dist/common/http/fastify-adapter`); +process.env.CACHE_URL = ''; +const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false }); +await app.init(); +require('nestjs-pino').PinoLogger.root.level = 'silent'; +const server = app.getHttpAdapter().getInstance(); +await server.ready(); +const prisma = app.get(PrismaService); +const passwords = app.get(PasswordService); +const originalHash = passwords.hash.bind(passwords); +const spans = []; +passwords.hash = async (password) => { + const start = performance.now(); + const value = await originalHash(password); + spans.push({ phase: 'kdf', ms: Math.round(performance.now() - start) }); + return value; +}; +app.get(UserService).prisma = prisma.$extends({ query: { user: { async create({ args, query }) { + const start = performance.now(); + const value = await query(args); + spans.push({ phase: 'sql_user_create', ms: Math.round(performance.now() - start) }); + return value; +} } } }); +const ids = []; +try { + const health = await server.inject({ method: 'GET', url: '/health/ready' }); + console.log(JSON.stringify({ phase: 'health', status: health.statusCode })); + for (let sample = 1; sample <= 3; sample++) { + spans.length = 0; + const start = performance.now(); + const response = await server.inject({ method: 'POST', url: '/api/v1/users', payload: { + username: `warm_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + display_name: 'Synthetic startup diagnostic', password: 'Synthetic-diagnostic-password-123', + } }); + if (response.statusCode === 201) ids.push(response.json().id); + console.log(JSON.stringify({ sample, status: response.statusCode, total_ms: Math.round(performance.now() - start), spans })); + } +} finally { + await prisma.user.deleteMany({ where: { id: { in: ids } } }); + await app.close(); +} diff --git a/test/benchmark-scaling.mjs b/test/benchmark-scaling.mjs new file mode 100644 index 0000000..4ee8bfe --- /dev/null +++ b/test/benchmark-scaling.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import test from 'node:test'; + +test('scaling benchmark schedules reads and writes and reports distinct instances', async () => { + let sequence = 0; + const server = createServer(async (request, response) => { + for await (const _ of request) { + // Consume the request body before responding. + } + response.setHeader('Content-Type', 'application/json'); + response.setHeader('X-Instance-Id', 'mock1'); + let status = 200; + let body = {}; + if (request.url === '/api/v1/users' && request.method === 'POST') { + status = 201; + body = { id: `user-${++sequence}` }; + } else if (request.url === '/api/v1/auth/sessions') { + status = 201; + body = { token: 'synthetic-token' }; + } else if (request.url === '/api/v1/chats' && request.method === 'POST') { + status = 201; + body = { id: 'chat-1' }; + } else if (request.url.startsWith('/api/v1/chats/chat-1/messages')) { + if (request.method === 'POST') { + status = 201; + body = { id: `message-${++sequence}` }; + } else { + body = { items: [] }; + } + } else { + status = 404; + } + response.writeHead(status); + response.end(JSON.stringify(body)); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const port = server.address().port; + const args = [ + 'scripts/benchmark-scaling.mjs', '--base-url', `http://127.0.0.1:${port}`, + '--label', 'mock', '--rate', '20', '--duration-seconds', '2', + '--warmup-seconds', '1', '--seed-messages', '2', + '--read-percent', '50', '--expected-instances', '1', + ]; + const child = spawn(process.execPath, args, { cwd: process.cwd() }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + const code = await new Promise((resolve) => child.on('close', resolve)); + assert.equal(code, 0, stderr); + const result = JSON.parse(stdout); + assert.equal(result.measurement.scheduled_slots, 40); + assert.equal(result.measurement.client_dropped_slots, 0); + assert.equal(result.measurement.all.successful, 40); + assert.equal(result.measurement.all.instance_counts.mock1, 40); + assert.ok(result.measurement.read.successful > 0); + assert.ok(result.measurement.write.successful > 0); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +}); diff --git a/test/cache-benchmark.mjs b/test/cache-benchmark.mjs new file mode 100644 index 0000000..007291e --- /dev/null +++ b/test/cache-benchmark.mjs @@ -0,0 +1,541 @@ +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { readFile, writeFile } from 'node:fs/promises'; +import { performance } from 'node:perf_hooks'; +import { faultProxy } from './helpers/fault-proxy.mjs'; +import { messageEpoch } from './helpers/message-epoch.mjs'; + +const require = createRequire(`${process.cwd()}/package.json`); +const pg = require('pg'); +let counts = {}; +const originalQuery = pg.Client.prototype.query; +pg.Client.prototype.query = function (query, ...args) { + const sql = typeof query === 'string' ? query : query.text; + const verb = sql.trim().split(/\s+/)[0].toUpperCase(); + const table = + ['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) => + new RegExp(`\\b${name}\\b`).test(sql), + ) ?? 'other'; + const label = `${verb}:${table}`; + counts[label] = (counts[label] ?? 0) + 1; + return originalQuery.call(this, query, ...args); +}; +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require(`${process.cwd()}/dist/app.module`); +const { PrismaService } = require( + `${process.cwd()}/dist/prisma/prisma.service`, +); +const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`); +const { createFastifyAdapter, exposeRequestIdHeader } = require( + `${process.cwd()}/dist/common/http/fastify-adapter`, +); +const stage = process.argv[2]; +const compareRedis = process.argv.includes('--compare-redis'); +const historyControl = !process.argv.includes('--skip-history-control'); +function integerOption(name, fallback) { + const index = process.argv.indexOf(`--${name}`); + const value = index === -1 ? fallback : Number(process.argv[index + 1]); + assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`); + return value; +} +const concurrency = integerOption('concurrency', 16); +const duration = integerOption('duration-seconds', 10); +const warmup = integerOption('warmup-seconds', 3); +const repetitions = integerOption('repetitions', 3); +const seedMessages = integerOption('seed-messages', 10000); +const coldBurst = integerOption('cold-burst', 32); +const writesPerRepeat = integerOption('writes-per-repeat', 512); +assert.ok(['before', 'after'].includes(stage)); +const prefix = `experiment:epochs:${randomUUID()}`; +const proxy = await faultProxy(process.env.CACHE_URL, 6379); +const observer = new Redis(process.env.CACHE_URL); +process.env.CACHE_PREFIX = prefix; +process.env.CACHE_URL = proxy.url; +const apps = []; +async function start() { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + apps.push(app); + exposeRequestIdHeader(app); + await app.listen(0, '127.0.0.1'); + require('nestjs-pino').PinoLogger.root.level = 'silent'; + return { app, url: await app.getUrl() }; +} +const first = await start(); +const prisma = first.app.get(PrismaService); +let userId; +const chatIds = []; +const report = { + stage, + recorded_at: new Date().toISOString(), + node: process.version, + methodology: { + requests: + 'real loopback HTTP; instrumented API process in measured backend image', + sql: 'pg.Client.query calls; transaction control counted separately', + concurrency, + duration_seconds: duration, + warmup_seconds: warmup, + repetitions, + seed_messages: seedMessages, + cold_burst_requests: coldBurst, + writes_per_repeat: writesPerRepeat, + history_control: historyControl, + resources: + 'process CPU/RSS includes HTTP generator and instrumentation; not container CPU', + limitations: + 'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF', + }, + dist_sha256: {}, + dependencies: {}, + samples: [], + correctness: {}, +}; +for (const name of [ + '@nestjs/core', + '@prisma/adapter-pg', + 'pg', + 'ioredis', + 'fastify', +]) { + try { + report.dependencies[name] = require(`${name}/package.json`).version; + } catch { + report.dependencies[name] = 'package metadata not exported'; + } +} +for (const name of [ + 'Chat/chat.service.js', + 'cache/cache.service.js', + 'Auth/auth.service.js', + 'User/user.guard.js', + 'Chat/chat.controller.js', +]) { + report.dist_sha256[name] = createHash('sha256') + .update(await readFile(`${process.cwd()}/dist/${name}`)) + .digest('hex'); +} +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function clearCache() { + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); +} +let token; +async function request(method, path, body, target = first) { + const response = await fetch(`${target.url}${path}`, { + method, + headers: { + authorization: `Bearer ${token}`, + ...(body ? { 'content-type': 'application/json' } : {}), + ...(method === 'POST' && /\/messages$/.test(path) + ? { 'idempotency-key': randomUUID() } + : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + signal: AbortSignal.timeout(10000), + }); + const json = await response.json(); + assert.equal( + response.status, + method === 'POST' ? 201 : 200, + `HTTP ${response.status}`, + ); + return json; +} +async function newChat() { + const chat = await prisma.chat.create({ + data: { + title: 'Synthetic epoch benchmark', + members: { create: { userId, position: 0 } }, + }, + }); + chatIds.push(chat.id); + return chat.id; +} +async function measure( + workload, + repeat, + operation, + { + duration: secondsToRun = duration, + total, + concurrency: clients = concurrency, + } = {}, +) { + counts = {}; + const cpu = process.cpuUsage(); + let peakRss = process.memoryUsage().rss; + const timer = setInterval(() => { + peakRss = Math.max(peakRss, process.memoryUsage().rss); + }, 100); + const latencies = []; + let errors = 0, + index = 0; + const start = performance.now(); + await Promise.all( + Array.from({ length: clients }, async () => { + while ( + total === undefined + ? performance.now() - start < secondsToRun * 1000 + : index < total + ) { + const item = index++; + const began = performance.now(); + try { + await operation(item); + latencies.push(performance.now() - began); + } catch { + errors++; + } + } + }), + ); + const seconds = (performance.now() - start) / 1000; + clearInterval(timer); + peakRss = Math.max(peakRss, process.memoryUsage().rss); + const used = process.cpuUsage(cpu); + latencies.sort((a, b) => a - b); + const percentile = (p) => + latencies.length + ? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3) + : null; + const sql = Object.entries(counts) + .filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key)) + .reduce((sum, [, n]) => sum + n, 0); + const sample = { + workload, + repeat, + concurrency: clients, + elapsed_seconds: +seconds.toFixed(3), + successes: latencies.length, + errors, + error_ratio: errors / (latencies.length + errors), + successful_rps: +(latencies.length / seconds).toFixed(3), + latency_ms: { + mean: latencies.length + ? +( + latencies.reduce((sum, value) => sum + value, 0) / latencies.length + ).toFixed(3) + : null, + p50: percentile(0.5), + p95: percentile(0.95), + p99: percentile(0.99), + }, + sql_queries: sql, + sql_per_request: +(sql / (latencies.length + errors)).toFixed(4), + sql_calls: { ...counts }, + process_cpu_percent_one_core: +( + (used.user + used.system) / + (seconds * 10000) + ).toFixed(2), + peak_process_rss_mib: +(peakRss / 1048576).toFixed(2), + }; + report.samples.push(sample); + console.log(JSON.stringify(sample)); + assert.equal(errors, 0, 'load errors recorded in report'); +} +try { + const user = await prisma.user.create({ + data: { + username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Synthetic benchmark', + }, + }); + userId = user.id; + token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 3600000), + }, + }); + const readChat = await newChat(); + for (let offset = 0; offset < seedMessages; offset += 1000) { + await prisma.message.createMany({ + data: Array.from( + { length: Math.min(1000, seedMessages - offset) }, + (_, i) => ({ + chatId: readChat, + senderId: userId, + text: `Synthetic message ${offset + i}`, + createdAt: new Date(1700000000000 + offset + i), + }), + ), + }); + } + await prisma.chatMessageEpoch.createMany({ + data: Array.from({ length: 16 }, (_, bucket) => ({ + chatId: readChat, + bucket, + epoch: BigInt( + Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0), + ), + })), + }); + const path = `/api/v1/chats/${readChat}/messages?limit=50`; + if (compareRedis) { + const cacheUrl = process.env.CACHE_URL; + delete process.env.CACHE_URL; + let withoutRedis; + try { + withoutRedis = await start(); + } finally { + process.env.CACHE_URL = cacheUrl; + } + assert.equal(withoutRedis.app.get(CacheService).redis, undefined); + report.methodology.comparison = { + modes: ['redis', 'no-redis'], + no_redis: + 'API started without CACHE_URL; all application Redis caches disabled', + epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes', + order: 'redis first in repeats 1/3, no-redis first in repeat 2', + processes: + 'both API instances and HTTP generator share one measured Node process', + }; + report.correctness.read_after_write = {}; + for (let repeat = 1; repeat <= repetitions; repeat++) { + const variants = [ + { name: 'redis', target: first }, + { name: 'no-redis', target: withoutRedis }, + ]; + if (repeat % 2 === 0) variants.reverse(); + for (const { name, target } of variants) { + await clearCache(); + await measure( + `cold-history-burst:${name}`, + repeat, + () => request('GET', path, undefined, target), + { total: coldBurst, concurrency: coldBurst }, + ); + const warmUntil = performance.now() + warmup * 1000; + while (performance.now() < warmUntil) + await request('GET', path, undefined, target); + await measure(`warm-history:${name}`, repeat, () => + request('GET', path, undefined, target), + ); + const writeChat = await newChat(); + const writePath = `/api/v1/chats/${writeChat}/messages`; + assert.equal( + (await request('GET', writePath, undefined, target)).items.length, + 0, + ); + await measure( + `writes-single-chat:${name}`, + repeat, + (i) => + request( + 'POST', + writePath, + { text: `Concurrent write ${i}` }, + target, + ), + { total: writesPerRepeat }, + ); + assert.equal( + await prisma.message.count({ where: { chatId: writeChat } }), + writesPerRepeat, + ); + assert.equal( + await messageEpoch(prisma, writeChat), + BigInt(writesPerRepeat), + ); + const page = await request('GET', writePath, undefined, target); + assert.equal(page.items.length, 50); + assert.equal(new Set(page.items.map((message) => message.id)).size, 50); + report.correctness.read_after_write[`${name}:${repeat}`] = true; + } + } + } else { + const cache = first.app.get(CacheService); + const remember = cache.remember.bind(cache); + for (let repeat = 1; repeat <= repetitions; repeat++) { + await clearCache(); + await request('GET', `/api/v1/chats/${readChat}`); + await measure('cold-history-burst', repeat, () => request('GET', path), { + total: coldBurst, + concurrency: coldBurst, + }); + const warmUntil = performance.now() + warmup * 1000; + while (performance.now() < warmUntil) await request('GET', path); + await measure('warm-history', repeat, () => request('GET', path)); + if (historyControl) { + cache.remember = (namespace, parameters, ttl, load) => + namespace.startsWith('messages:') + ? load() + : remember(namespace, parameters, ttl, load); + const directUntil = performance.now() + warmup * 1000; + while (performance.now() < directUntil) await request('GET', path); + await measure('history-cache-bypassed', repeat, () => + request('GET', path), + ); + cache.remember = remember; + } + const writeChat = await newChat(); + await request('GET', `/api/v1/chats/${writeChat}`); + await measure( + 'writes-single-chat', + repeat, + (i) => + request('POST', `/api/v1/chats/${writeChat}/messages`, { + text: `Concurrent write ${i}`, + }), + { total: writesPerRepeat }, + ); + assert.equal( + await prisma.message.count({ where: { chatId: writeChat } }), + writesPerRepeat, + ); + } + const second = await start(); + const faultChat = await newChat(); + const faultPath = `/api/v1/chats/${faultChat}/messages`; + assert.equal( + (await request('GET', faultPath, undefined, second)).items.length, + 0, + ); + const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`); + assert.ok(keysBefore.length); + proxy.setMode('cut'); + await sleep(100); + const written = await request('POST', faultPath, { + text: 'Committed while Redis is unavailable', + }); + const during = await request('GET', faultPath, undefined, second); + proxy.setMode('online'); + for (let i = 0; i < 60; i++) { + if (apps.every((app) => app.get(CacheService).redis?.status === 'ready')) + break; + await sleep(100); + } + assert.ok( + apps.every((app) => app.get(CacheService).redis?.status === 'ready'), + ); + const recovered = await request('GET', faultPath, undefined, second); + report.correctness.redis_reconnect = { + old_keys_retained: + (await observer.exists(...keysBefore)) === keysBefore.length, + read_after_write_during_outage: during.items.some( + (x) => x.id === written.id, + ), + read_after_write_after_reconnect_other_replica: recovered.items.some( + (x) => x.id === written.id, + ), + }; + await clearCache(); + const cold = await request('GET', faultPath, undefined, second); + report.correctness.empty_cache_recovery = cold.items.some( + (x) => x.id === written.id, + ); + const raceChat = await newChat(); + const racePath = `/api/v1/chats/${raceChat}/messages`; + const service = first.app.get( + require(`${process.cwd()}/dist/Chat/chat.service`).ChatService, + ); + const actualPrisma = service.prisma; + let release, began; + const started = new Promise((resolve) => { + began = resolve; + }); + const held = new Promise((resolve) => { + release = resolve; + }); + service.prisma = actualPrisma.$extends({ + query: { + message: { + async findMany({ args, query }) { + const rows = await query(args); + if (args.where?.chatId === raceChat) { + began(); + await held; + } + return rows; + }, + }, + }, + }); + const pending = request('GET', racePath); + let fresh; + try { + await Promise.race([ + started, + sleep(10000).then(() => { + throw new Error('race load did not start'); + }), + ]); + proxy.setMode('cut'); + await sleep(100); + fresh = await request( + 'POST', + racePath, + { text: 'New message after old SELECT' }, + second, + ); + proxy.setMode('online'); + for (let i = 0; i < 60; i++) { + if ( + apps.every((app) => app.get(CacheService).redis?.status === 'ready') + ) + break; + await sleep(100); + } + assert.ok( + apps.every((app) => app.get(CacheService).redis?.status === 'ready'), + ); + } finally { + release(); + service.prisma = actualPrisma; + } + await pending; + report.correctness.delayed_old_fill = ( + await request('GET', racePath, undefined, second) + ).items.some((x) => x.id === fresh.id); + if (stage === 'after') { + assert.equal( + report.correctness.redis_reconnect + .read_after_write_after_reconnect_other_replica, + true, + ); + assert.equal(report.correctness.delayed_old_fill, true); + assert.equal(report.correctness.empty_cache_recovery, true); + } + } + report.completed = true; +} finally { + proxy.setMode('online'); + try { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + if (userId) await prisma.user.delete({ where: { id: userId } }); + await clearCache(); + report.cleanup = 'only experiment chats/user/cache prefix removed'; + } finally { + for (const app of apps) await app.close(); + observer.disconnect(); + await proxy.close(); + await writeFile( + process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`, + JSON.stringify(report, null, 2) + '\n', + ); + } +} +console.log( + JSON.stringify({ + stage, + correctness: report.correctness, + completed: report.completed, + }), +); diff --git a/test/cache-environment.ps1 b/test/cache-environment.ps1 new file mode 100644 index 0000000..3223737 --- /dev/null +++ b/test/cache-environment.ps1 @@ -0,0 +1,25 @@ +$ErrorActionPreference = 'Stop' +$dockerInfo = docker info --format '{{json .}}' | ConvertFrom-Json +if ($LASTEXITCODE -ne 0) { throw 'docker info failed' } +$processor = Get-CimInstance Win32_Processor | Select-Object -First 1 +$computer = Get-CimInstance Win32_ComputerSystem +$containers = docker inspect mtusi2026lab2-fork-backend-1 mtusi2026lab2-fork-postgres-1 mtusi2026lab2-fork-redis-1 | ConvertFrom-Json +if ($LASTEXITCODE -ne 0) { throw 'docker inspect failed' } +$metadata = [ordered]@{ + recorded_at = [DateTime]::UtcNow.ToString('o') + host_cpu = $processor.Name + host_physical_cores = $processor.NumberOfCores + host_logical_processors = $processor.NumberOfLogicalProcessors + host_ram_bytes = $computer.TotalPhysicalMemory + docker_version = $dockerInfo.ServerVersion + docker_cpu_count = $dockerInfo.NCPU + docker_ram_bytes = $dockerInfo.MemTotal + docker_kernel = $dockerInfo.KernelVersion + containers = @($containers | ForEach-Object { + [ordered]@{ name = $_.Name; image = $_.Image; memory_limit_bytes = $_.HostConfig.Memory; nano_cpus_limit = $_.HostConfig.NanoCpus; cpu_quota = $_.HostConfig.CpuQuota } + }) + before_image = 'sha256:075170aed76a42aa98b435cfb918d83879259b7e17177275d5274e328198e119' + before_image_local_tag = 'lab2-cache-epochs:before' +} +$metadataPath = Join-Path (Get-Location).Path 'evidence/cache-epochs-environment.json' +[System.IO.File]::WriteAllText($metadataPath, ($metadata | ConvertTo-Json -Depth 6) + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false)) diff --git a/test/cache-epochs-optimization-summary.mjs b/test/cache-epochs-optimization-summary.mjs new file mode 100644 index 0000000..4a89670 --- /dev/null +++ b/test/cache-epochs-optimization-summary.mjs @@ -0,0 +1,91 @@ +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; + +const reports = await Promise.all( + ['after', 'optimized'].map(async (name) => + JSON.parse(await readFile(`evidence/cache-epochs-${name}.json`, 'utf8')), + ), +); +assert.ok(reports.every((report) => report.completed)); +assert.deepEqual(reports[0].methodology, reports[1].methodology); +const median = (values) => + [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; +const labels = ['Отдельная транзакция', 'Один CTE']; +const lines = [ + '# Оптимизация эпохи кэша в PostgreSQL', + '', + `Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`, + '', + 'Сравнение сохранённого замера прежней реализации и нового замера после оптимизации. В обоих вариантах эпоха хранится в PostgreSQL; новый вариант объединяет проверку членства, UPDATE эпохи и INSERT сообщения одним запросом. При чтении истории убрано получение неиспользуемых метаданных чата.', + '', + 'Медианы трёх повторов. Перцентили — медианы перцентилей повторов, без объединения выборок.', + '', + '| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | BEGIN/COMMIT на HTTP | CPU, % ядра | RSS, MiB | Ошибки |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', +]; +for (const workload of new Set( + reports[0].samples.map((sample) => sample.workload), +)) { + for (const [index, report] of reports.entries()) { + const samples = report.samples.filter( + (sample) => sample.workload === workload, + ); + assert.equal(samples.length, 3); + const get = (select) => median(samples.map(select)).toFixed(3); + const controls = (sample) => + Object.entries(sample.sql_calls) + .filter(([key]) => /^(BEGIN|COMMIT):/.test(key)) + .reduce((total, [, count]) => total + count, 0) / + (sample.successes + sample.errors); + lines.push( + `| ${workload} | ${labels[index]} | ${get((sample) => sample.successful_rps)} | ${get((sample) => sample.latency_ms.p50)} | ${get((sample) => sample.latency_ms.p95)} | ${get((sample) => sample.latency_ms.p99)} | ${get((sample) => sample.sql_per_request)} | ${get(controls)} | ${get((sample) => sample.process_cpu_percent_one_core)} | ${get((sample) => sample.peak_process_rss_mib)} | ${samples.reduce((total, sample) => total + sample.errors, 0)} |`, + ); + } +} +lines.push( + '', + '| Проверка | Отдельная транзакция | Один CTE |', + '| --- | --- | --- |', +); +for (const [label, select] of [ + [ + 'Чтение после записи при отказе Redis', + (report) => + report.correctness.redis_reconnect.read_after_write_during_outage, + ], + [ + 'Чтение другой репликой после восстановления Redis', + (report) => + report.correctness.redis_reconnect + .read_after_write_after_reconnect_other_replica, + ], + [ + 'Старые ключи сохранились в Redis', + (report) => report.correctness.redis_reconnect.old_keys_retained, + ], + [ + 'Восстановление с пустым кэшем', + (report) => report.correctness.empty_cache_recovery, + ], + [ + 'Задержанная загрузка старой истории', + (report) => report.correctness.delayed_old_fill, + ], +]) { + lines.push( + `| ${label} | ${reports.map((report) => (select(report) ? 'PASS' : 'FAIL')).join(' | ')} |`, + ); +} +lines.push( + '', + 'Методика: тот же `test/cache-benchmark.mjs`, 10 000 сообщений, limit=50, конкурентность 16, 3 с прогрева и 10 с чтения, 512 записей в новый чат, три повтора. Реальный HTTP по loopback в отдельных benchmark-контейнерах; общий PostgreSQL/Redis. Токены и пользователи создаются напрямую, хеширование паролей не измеряется. BEGIN/COMMIT исключены из SQL/HTTP; запрос WITH учитывается как один вызов pg.Client.query.', + '', + 'CPU/RSS включают API, генератор HTTP и инструментацию. Docker-host разделяется с другими работающими сервисами. Старый и новый замеры сделаны в разное время; изменение фоновой нагрузки и состояния host ограничивает сравнение RPS/latency. Замеры не доказывают максимальную ёмкость или SLA. UPDATE общей строки эпохи по-прежнему сериализует записи одного чата.', + '', + 'Сырые результаты: [до оптимизации](cache-epochs-after.json), [после оптимизации](cache-epochs-optimized.json). Хеши измеренных модулей сохранены в JSON. Исторические замеры до введения эпох остаются в `cache-epochs-summary.md`.', +); +await writeFile( + 'evidence/cache-epochs-optimization-summary.md', + lines.join('\n') + '\n', +); +console.log(lines.join('\n')); diff --git a/test/cache-epochs.mjs b/test/cache-epochs.mjs new file mode 100644 index 0000000..cac3aa1 --- /dev/null +++ b/test/cache-epochs.mjs @@ -0,0 +1,200 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; +import { faultProxy } from './helpers/fault-proxy.mjs'; +import { messageEpoch } from './helpers/message-epoch.mjs'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { CacheService } = require('../dist/cache/cache.service'); +const { ChatService } = require('../dist/Chat/chat.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +test( + 'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback', + { timeout: 90000 }, + async (t) => { + assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL); + const oldUrl = process.env.CACHE_URL; + const oldPrefix = process.env.CACHE_PREFIX; + const prefix = `test:epochs:${randomUUID()}`; + const proxy = await faultProxy(oldUrl, 6379); + const observer = new Redis(oldUrl); + process.env.CACHE_URL = proxy.url; + process.env.CACHE_PREFIX = prefix; + const apps = []; + let prisma, userId; + const chatIds = []; + t.after(async () => { + proxy.setMode('online'); + try { + if (prisma) { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + if (userId) await prisma.user.delete({ where: { id: userId } }); + } + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + for (const app of apps) await app.close(); + observer.disconnect(); + await proxy.close(); + process.env.CACHE_URL = oldUrl; + if (oldPrefix === undefined) delete process.env.CACHE_PREFIX; + else process.env.CACHE_PREFIX = oldPrefix; + } + }); + for (let i = 0; i < 2; i++) { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + apps.push(app); + exposeRequestIdHeader(app); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + require('nestjs-pino').PinoLogger.root.level = 'silent'; + } + prisma = apps[0].get(PrismaService); + const user = await prisma.user.create({ + data: { + username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Synthetic epochs', + }, + }); + userId = user.id; + const token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + const call = async (replica, method, url, payload) => { + const response = await apps[replica] + .getHttpAdapter() + .getInstance() + .inject({ + method, + url, + headers: { + authorization: `Bearer ${token}`, + ...(method === 'POST' && /\/messages$/.test(url) + ? { 'idempotency-key': randomUUID() } + : {}), + }, + ...(payload ? { payload } : {}), + }); + return { status: response.statusCode, body: response.json() }; + }; + const createChat = async () => { + const response = await call(0, 'POST', '/api/v1/chats', { + title: 'Epoch test', + member_ids: [userId], + }); + assert.equal(response.status, 201); + chatIds.push(response.body.id); + return response.body.id; + }; + assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0); + proxy.setMode('cut'); + await pause(100); + const chatId = await createChat(); + proxy.setMode('online'); + const reconnect = async () => { + for (let i = 0; i < 60; i++) { + if (apps.every((app) => app.get(CacheService).redis.status === 'ready')) + return; + await pause(100); + } + assert.fail('Redis did not reconnect'); + }; + await reconnect(); + assert.deepEqual( + (await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id), + [chatId], + ); + assert.equal( + (await prisma.user.findUniqueOrThrow({ where: { id: userId } })) + .chatsEpoch, + 1n, + ); + + const path = `/api/v1/chats/${chatId}/messages`; + assert.equal((await call(1, 'GET', path)).body.items.length, 0); + const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`); + assert.ok(oldKeys.length); + proxy.setMode('cut'); + await pause(100); + const posted = await call(0, 'POST', path, { + text: 'Write during Redis outage', + }); + assert.equal(posted.status, 201); + proxy.setMode('online'); + await reconnect(); + assert.equal( + await observer.exists(...oldKeys), + oldKeys.length, + 'old cache keys must still exist', + ); + assert.deepEqual( + (await call(1, 'GET', path)).body.items.map((x) => x.id), + [posted.body.id], + ); + + const service = apps[0].get(ChatService); + const original = service.prisma; + service.prisma = original.$extends({ + query: { + $queryRaw({ args, query }) { + const index = args.values.indexOf('Must roll back'); + if (index !== -1) args.values[index] = 'x'.repeat(4001); + return query(args); + }, + }, + }); + try { + assert.equal( + (await call(0, 'POST', path, { text: 'Must roll back' })).status, + 500, + ); + } finally { + service.prisma = original; + } + assert.equal(await prisma.message.count({ where: { chatId } }), 1); + assert.equal(await messageEpoch(prisma, chatId), 1n); + + const writes = await Promise.all( + Array.from({ length: 20 }, (_, i) => + call(i % 2, 'POST', path, { text: `Concurrent ${i}` }), + ), + ); + assert.ok(writes.every((x) => x.status === 201)); + assert.equal(await prisma.message.count({ where: { chatId } }), 21); + assert.equal(await messageEpoch(prisma, chatId), 21n); + const history = (await call(1, 'GET', path)).body.items; + assert.equal(new Set(history.map((x) => x.id)).size, 21); + assert.ok( + writes.every((x) => history.some((message) => message.id === x.body.id)), + ); + }, +); diff --git a/test/cache-metrics-summary.mjs b/test/cache-metrics-summary.mjs new file mode 100644 index 0000000..def3db5 --- /dev/null +++ b/test/cache-metrics-summary.mjs @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; +const before = JSON.parse( + await readFile('evidence/cache-epochs-before.json', 'utf8'), +); +const after = JSON.parse( + await readFile('evidence/cache-epochs-after.json', 'utf8'), +); +assert.ok(before.completed && after.completed); +assert.deepEqual(before.methodology, after.methodology); +const median = (values) => + values.sort((a, b) => a - b)[Math.floor(values.length / 2)]; +const lines = [ + '# Эпохи кэша — сравнение 2026-10-13', + '', + 'Медианы трёх повторов. p50/p95/p99 — медианы перцентилей отдельных повторов, не перцентили объединённой выборки.', + '', + '| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | CPU процесса, % ядра | Пиковый RSS, MiB | Ошибки |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', +]; +for (const workload of [...new Set(before.samples.map((x) => x.workload))]) { + for (const report of [before, after]) { + const samples = report.samples.filter((x) => x.workload === workload); + assert.equal(samples.length, 3); + const get = (fn) => median(samples.map(fn)).toFixed(3); + lines.push( + `| ${workload} | ${report.stage} | ${get((x) => x.successful_rps)} | ${get((x) => x.latency_ms.p50)} | ${get((x) => x.latency_ms.p95)} | ${get((x) => x.latency_ms.p99)} | ${get((x) => x.sql_per_request)} | ${get((x) => x.process_cpu_percent_one_core)} | ${get((x) => x.peak_process_rss_mib)} | ${samples.reduce((sum, x) => sum + x.errors, 0)} |`, + ); + } +} +lines.push('', '| Проверка | До | После |', '| --- | --- | --- |'); +for (const [label, select] of [ + [ + 'Чтение после успешной записи при недоступном Redis', + (x) => x.correctness.redis_reconnect.read_after_write_during_outage, + ], + [ + 'Чтение другой репликой после восстановления Redis со старыми ключами', + (x) => + x.correctness.redis_reconnect + .read_after_write_after_reconnect_other_replica, + ], + [ + 'Старые ключи действительно сохранились', + (x) => x.correctness.redis_reconnect.old_keys_retained, + ], + ['Восстановление с пустым кэшем', (x) => x.correctness.empty_cache_recovery], + [ + 'Задержанная загрузка старой истории не отравляет следующий GET', + (x) => x.correctness.delayed_old_fill, + ], +]) + lines.push( + `| ${label} | ${select(before) ? 'PASS' : 'FAIL'} | ${select(after) ? 'PASS' : 'FAIL'} |`, + ); +lines.push( + '', + 'Методика: `test/cache-benchmark.mjs`, сырой результат: `cache-epochs-before.json` и `cache-epochs-after.json`. Конфигурация: `cache-epochs-environment.json`.', + '', + 'Реальный HTTP по loopback, 10 000 синтетических сообщений, первая страница limit=50. На повтор: холодная волна 32 запросов; прогрев 3 с; чтение с конкурентностью 16 в течение 10 с; прогрев 3 с и такой же контроль с отключённым кэшем истории; 512 записей в новый чат с конкурентностью 16. Три повтора. SQL считается на уровне pg.Client.query; BEGIN/COMMIT/ROLLBACK/SET вынесены в raw sql_calls и исключены из SQL/HTTP. Пароли не измеряются: пользователь и сессия создаются напрямую, токен остаётся в памяти.', + '', + 'CPU/RSS относятся к процессу эксперимента, включающему API, генератор HTTP и инструментацию. Docker Desktop: 4 CPU, около 2.80 GiB RAM, отдельные лимиты сервисов не заданы. Замеры closed-loop, короткие и последовательные before → after; разброс и прогрев ограничивают выводы о latency/RPS. Это не тест максимальной ёмкости или SLA. Первый холодный повтор также включает прогрев runtime.', + '', + 'Первичные короткие замеры (1 с прогрева, 5 с чтений, 128 записей) сохранены отдельно в cache-epochs-before-short.json и cache-epochs-after-short.json. Из-за большого разброса сделано одно повторное сравнение с увеличенным прогревом и нагрузкой; основные результаты получены в отдельных контейнерах benchmark из соответствующих образов. PostgreSQL/Redis и остальные контейнеры разделяют ресурсы Docker-host. Другие проекты на этом host не останавливались. Формулировки provenance в основных JSON уточнены после перехода от docker exec к отдельному контейнеру; численные измерения не менялись.', + '', + 'До изменений использован фактически запущенный образ (локальный тег lab2-cache-epochs:before), после — пересобранный проект. Старый образ ещё использовал OR для курсорной пагинации; измеряемая первая страница не содержит курсора, поэтому этот путь в нагрузочном сравнении не выполняется. Хеши измеренных модулей записаны в JSON.', + '', + 'Эпоха истории читается JOIN-ом вместе с проверкой членства; эпоха списка чатов — JOIN-ом с проверкой сессии. Для отдельного запроса дополнительного чтения эпохи нет. Однако raw JOIN проверки членства не использует прежний batching findUnique в Prisma: под конкуренцией суммарный SQL/HTTP может вырасти. Запись добавляет UPDATE чата и транзакционные команды; обновления одного чата сериализуются. Старые ключи недоступны новым читателям и удаляются по TTL.', + '', + 'Корректность зависит от основного PostgreSQL и обновления эпох во всех путях изменения данных. Прямые SQL-изменения в обход приложения требуют увеличения соответствующей эпохи. После восстановления PostgreSQL из старого backup следует сменить CACHE_PREFIX или очистить кэш, чтобы исключить повторное использование эпох. Сохранность Docker-host и высокая доступность PostgreSQL не исследовались. Cache stampede не устраняется этим изменением.', +); +await writeFile('evidence/cache-epochs-summary.md', lines.join('\n') + '\n'); +console.log(lines.slice(0, 14).join('\n')); diff --git a/test/cache-strategy-summary.mjs b/test/cache-strategy-summary.mjs new file mode 100644 index 0000000..e7b2cc7 --- /dev/null +++ b/test/cache-strategy-summary.mjs @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; + +const resultPrefix = process.argv[2] ?? 'cache-strategy'; +assert.match(resultPrefix, /^[a-z0-9-]+$/); + +const reports = await Promise.all( + ['full', 'no-redis-no-epochs'].map(async (name) => + JSON.parse(await readFile(`evidence/${resultPrefix}-${name}.json`, 'utf8')), + ), +); +const provenance = JSON.parse( + await readFile(`evidence/${resultPrefix}-experiment.json`, 'utf8'), +); +assert.ok(reports.every((report) => report.completed)); +assert.deepEqual(reports[0].methodology, reports[1].methodology); +assert.equal(provenance.exit_code, 0); +assert.ok(provenance.sources_unchanged); +assert.ok(reports[1].correctness.epochs_unchanged); +assert.ok(reports[1].correctness.no_redis_client); +const median = (values) => + [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; +const labels = ['Redis и эпохи', 'Без Redis и эпох']; +const aggregates = []; +const settings = reports[0].methodology; +const lines = [ + '# Полная стратегия кэша и вариант без Redis и эпох', + '', + `Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`, + '', + 'Сначала выполнен свежий прогон текущего образа с Redis, PostgreSQL-эпохами и записью сообщения через CTE. Затем шесть модулей приложения заменены временными версиями без поддержания эпох: убраны изменения счётчиков и чтения эпох в проверках сессии/членства. Объединённые проверки сессии, публичного профиля и доступа к истории сохранены. Сообщение вставляется одним INSERT ... SELECT с проверкой членства, без обновления счётчиков. Второй API запущен без CACHE_URL: все кэши приложения отключены. Исходники рабочего проекта и запущенный backend не изменены.', + '', + `Параметры обоих прогонов совпадают: ${settings.repetitions} повтора, конкурентность ${settings.concurrency}, ${settings.seed_messages} сообщений, limit=50, прогрев ${settings.warmup_seconds} с и чтение ${settings.duration_seconds} с, ${settings.writes_per_repeat} записей в новый чат. RPS = общее число успешных запросов / общее время повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее отдельных p95, не перцентиль объединённой выборки. SQL/HTTP исключает команды транзакций.`, + '', + '| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | Медиана RPS | Ошибки |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', +]; +for (const [workload, label] of [ + ['cold-history-burst', 'Холодная волна истории'], + ['warm-history', 'Прогретая история'], + ['history-cache-bypassed', 'Контроль без кэша истории'], + ['writes-single-chat', 'Запись в один чат'], +]) { + if ( + reports.every( + (report) => + !report.samples.some((sample) => sample.workload === workload), + ) + ) + continue; + for (const [index, report] of reports.entries()) { + const samples = report.samples.filter( + (sample) => sample.workload === workload, + ); + assert.equal(samples.length, settings.repetitions); + const sum = (select) => + samples.reduce((total, sample) => total + select(sample), 0); + const successes = sum((sample) => sample.successes); + const errors = sum((sample) => sample.errors); + const aggregate = { + workload, + variant: labels[index], + successes, + errors, + mean_rps: successes / sum((sample) => sample.elapsed_seconds), + mean_latency_ms: + sum((sample) => sample.latency_ms.mean * sample.successes) / successes, + mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length, + sql_per_request: + sum((sample) => sample.sql_queries) / (successes + errors), + median_rps: median(samples.map((sample) => sample.successful_rps)), + repeat_rps: samples.map((sample) => sample.successful_rps), + }; + aggregates.push(aggregate); + lines.push( + `| ${label} | ${labels[index]} | ${aggregate.mean_rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.median_rps.toFixed(3)} | ${errors} |`, + ); + } +} +lines.push( + '', + `Учтено успешных HTTP-запросов: ${reports.map((report, index) => `${labels[index]} — ${report.samples.reduce((sum, sample) => sum + sample.successes, 0)}`).join('; ')}.`, + '', + ...['warm-history', 'writes-single-chat'].map((workload) => { + const pair = aggregates.filter( + (aggregate) => aggregate.workload === workload, + ); + return `${workload}: отношение среднего RPS полной стратегии к варианту без стратегии — ${(pair[0].mean_rps / pair[1].mean_rps).toFixed(3)}. RPS по повторам: ${pair.map((aggregate) => `${aggregate.variant}: ${aggregate.repeat_rps.join(' / ')}`).join('; ')}.`; + }), + '', + settings.history_control === false + ? 'Дополнительный контроль с частично отключённым кэшем в этом прогоне пропущен. Сравниваются только полная стратегия и отсутствие Redis/поддержания эпох.' + : 'В контрольном сценарии у первого API отключён только кэш истории; кэш профиля продолжает работать. У второго API отключены все кэши. Поэтому этот контроль не измеряет отдельно чистую стоимость эпох.', + '', + `Проверки полной стратегии: чтение после записи при отказе Redis, чтение другой репликой после восстановления со старыми ключами, восстановление с пустым кэшем, задержанная загрузка старой истории. Проверки варианта без стратегии: приложение не создало Redis-клиент, эпохи синтетических пользователей/чатов остались равны 0 после всех записей, ${settings.writes_per_repeat} сообщений записаны в каждом новом чате, история читается после записи без повторяющихся ID.`, + '', + `Оба прогона используют один базовый образ; изменённые модули временной версии скомпилированы TypeScript без полной пересборки контейнера и подключены отдельными bind mounts. Хеши исходников и JS сохранены в [метаданных эксперимента](${resultPrefix}-experiment.json). Колонки эпох остаются в схеме; это исключает миграцию и позволяет сохранить рабочую БД. SQL-проверки профиля и членства во втором варианте не используют эпохи. Временные файлы удалены, хеши рабочих исходников проверены.`, + '', + 'Ограничения: прогоны последовательные, Docker-host разделяется с другими сервисами; большой разброс ограничивает выводы о скорости. CPU/RSS процесса включают API, HTTP-генератор и инструментацию; CPU PostgreSQL отдельно не измерен. Синтетические сессии обходят KDF. Холодная волна означает пустой кэш приложения, а не холодный PostgreSQL. Отдельные цены Redis и эпох этим сравнением не разделяются: меняется вся стратегия. Данные рабочей БД и volumes сохранены, удалены только фикстуры эксперимента.', + '', + 'Повторение:', + '', + '```powershell', + `docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/${resultPrefix}-full.json --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`, + `$env:EXPERIMENT_PREFIX='${resultPrefix}'`, + `node test/no-epochs-experiment.cjs --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`, + `node test/cache-strategy-summary.mjs ${resultPrefix}`, + '```', + '', + `Сырые результаты: [полная стратегия](${resultPrefix}-full.json), [без Redis и эпох](${resultPrefix}-no-redis-no-epochs.json).`, +); +await writeFile(`evidence/${resultPrefix}-summary.md`, lines.join('\n') + '\n'); +await writeFile( + `evidence/${resultPrefix}-averages.json`, + JSON.stringify(aggregates, null, 2) + '\n', +); +console.log(lines.slice(0, 18).join('\n')); diff --git a/test/cache.mjs b/test/cache.mjs new file mode 100644 index 0000000..d648410 --- /dev/null +++ b/test/cache.mjs @@ -0,0 +1,292 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { CacheService } = require('../dist/cache/cache.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +async function isolatedCache(t) { + assert.ok(process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance'); + const previous = process.env.CACHE_PREFIX; + const prefix = `test:cache:${randomUUID()}`; + process.env.CACHE_PREFIX = prefix; + const observer = new Redis(process.env.CACHE_URL, { + maxRetriesPerRequest: 0, + connectTimeout: 1000, + commandTimeout: 1000, + }); + observer.on('error', () => {}); + await observer.ping(); + t.after(async () => { + try { + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + observer.disconnect(); + if (previous === undefined) delete process.env.CACHE_PREFIX; + else process.env.CACHE_PREFIX = previous; + } + }); + return { prefix, observer }; +} + +test('Redis cache: hits, expiry, invalidation race and unavailable-server fallback', async (t) => { + await isolatedCache(t); + const cache = new CacheService(); + t.after(() => cache.onModuleDestroy()); + await cache.onModuleInit(); + let reads = 0; + const load = async () => ({ value: ++reads }); + assert.deepEqual(await cache.remember('hits', 'page', 30, load), { + value: 1, + }); + assert.deepEqual(await cache.remember('hits', 'page', 30, load), { + value: 1, + }); + assert.equal(reads, 1); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 2, + }); + await new Promise((resolve) => setTimeout(resolve, 1100)); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 3, + }); + await cache.invalidate('expiry'); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 4, + }); + + let resolveOld; + let began; + const started = new Promise((resolve) => { + began = resolve; + }); + const pending = cache.remember('race', 'page', 30, () => { + began(); + return new Promise((resolve) => { + resolveOld = resolve; + }); + }); + await started; + await cache.invalidate('race'); + resolveOld({ value: 'old' }); + await pending; + assert.deepEqual( + await cache.remember('race', 'page', 30, async () => ({ value: 'new' })), + { value: 'new' }, + ); + assert.deepEqual( + await cache.remember('race', 'page', 30, async () => { + throw new Error('cache miss'); + }), + { value: 'new' }, + ); + await assert.rejects( + cache.remember('db-error', 'page', 30, async () => { + throw new Error('database failure'); + }), + /database failure/, + ); + + const previous = process.env.CACHE_URL; + process.env.CACHE_URL = 'redis://127.0.0.1:1/0'; + const offline = new CacheService(); + process.env.CACHE_URL = previous; + t.after(() => offline.onModuleDestroy()); + await offline.onModuleInit(); + assert.deepEqual( + await offline.remember('offline', 'page', 30, async () => ({ + value: 'database', + })), + { value: 'database' }, + ); + await offline.invalidate('offline'); +}); + +test('API cache: fewer SQL reads, write invalidation, permissions, logout and Redis fallback', async (t) => { + assert.ok(process.env.DATABASE_URL); + const { prefix, observer } = await isolatedCache(t); + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + const prisma = app.get(PrismaService); + const userIds = []; + const chatIds = []; + t.after(async () => { + try { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + await prisma.user.deleteMany({ where: { id: { in: userIds } } }); + } finally { + await app.close(); + } + }); + const actors = []; + for (let i = 0; i < 2; i++) { + const user = await prisma.user.create({ + data: { + username: `cache_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Cache test', + }, + }); + userIds.push(user.id); + const token = randomBytes(32).toString('base64url'); + const session = await prisma.session.create({ + data: { + userId: user.id, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + actors.push({ id: user.id, token, sessionId: session.id }); + } + const [alice, outsider] = actors; + const request = (method, url, payload, actor = alice) => + server.inject({ + method, + url, + headers: { + authorization: `Bearer ${actor.token}`, + ...(method === 'POST' && /\/messages$/.test(url) + ? { 'idempotency-key': randomUUID() } + : {}), + }, + ...(payload === undefined ? {} : { payload }), + }); + const counts = new Map(); + const tracked = prisma.$extends({ + query: { + $allModels: { + $allOperations({ model, operation, args, query }) { + const key = `${model}:${operation}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + return query(args); + }, + }, + }, + }); + for (const name of [ + 'User/user.service', + 'Chat/chat.service', + 'Auth/auth.service', + ]) { + const exported = require(`../dist/${name}`); + const Service = Object.values(exported)[0]; + app.get(Service).prisma = tracked; + } + const spy = (model, operation) => ({ + mock: { callCount: () => counts.get(`${model}:${operation}`) ?? 0 }, + }); + const profileReads = spy('User', 'findUnique'); + assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200); + assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200); + assert.equal(profileReads.mock.callCount(), 0); + + const chatListReads = spy('Chat', 'findMany'); + assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0); + assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0); + assert.equal(chatListReads.mock.callCount(), 1); + const created = await request('POST', '/api/v1/chats', { + title: 'Cached chat', + member_ids: [alice.id], + }); + assert.equal(created.statusCode, 201); + const chatId = created.json().id; + chatIds.push(chatId); + assert.deepEqual( + (await request('GET', '/api/v1/chats')).json().items.map((x) => x.id), + [chatId], + ); + assert.equal(chatListReads.mock.callCount(), 2); + + const metadataReads = spy('Chat', 'findUnique'); + const messageReads = spy('Message', 'findMany'); + const path = `/api/v1/chats/${chatId}/messages`; + assert.equal((await request('GET', path)).json().items.length, 0); + assert.equal((await request('GET', path)).json().items.length, 0); + assert.equal(metadataReads.mock.callCount(), 0); + assert.equal(messageReads.mock.callCount(), 1); + const message = await request('POST', path, { + text: 'Invalidate cached empty history', + }); + assert.equal(message.statusCode, 201); + assert.deepEqual( + (await request('GET', path)).json().items.map((x) => x.id), + [message.json().id], + ); + assert.equal(messageReads.mock.callCount(), 2); + assert.equal( + (await request('GET', path, undefined, outsider)).statusCode, + 404, + ); + await prisma.chatMember.delete({ + where: { chatId_userId: { chatId, userId: alice.id } }, + }); + assert.equal((await request('GET', path)).statusCode, 404); + await prisma.chatMember.create({ + data: { chatId, userId: alice.id, position: 0 }, + }); + await prisma.session.update({ + where: { id: outsider.sessionId }, + data: { revokedAt: new Date() }, + }); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, outsider)).statusCode, + 401, + ); + + const keys = []; + let cursor = '0'; + do { + const [next, batch] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + keys.push(...batch); + } while (cursor !== '0'); + assert.ok(keys.length >= 4, 'Redis must contain the cached responses'); + for (const key of keys) { + const value = await observer.get(key); + assert.ok( + !value.includes('password_hash') && !value.includes('passwordHash'), + ); + for (const actor of actors) assert.ok(!value.includes(actor.token)); + } + app.get(CacheService).onModuleDestroy(); + assert.equal((await request('GET', path)).statusCode, 200); + assert.equal(messageReads.mock.callCount(), 3); + assert.equal( + (await request('POST', path, { text: 'Write while Redis is unavailable' })) + .statusCode, + 201, + ); + assert.equal((await request('GET', path)).json().items.length, 2); +}); diff --git a/test/check_migration_compat.py b/test/check_migration_compat.py new file mode 100644 index 0000000..11da40e --- /dev/null +++ b/test/check_migration_compat.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Run the lab3 migration while a lab2 API uses an isolated temporary database.""" + +import base64 +import json +from pathlib import Path +import secrets +import subprocess +import sys +import threading +import time +import urllib.error +from urllib.parse import quote +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parents[1] +NETWORK = 'mtusi2026lab3-fork_default' +OLD_API = 'mtusi2026lab2-fork-backend:latest' +OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest' +NEW_API = 'mtusi2026lab3-fork-api:latest' +NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest' + + +def docker(*args, timeout=60): + result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, + text=True, timeout=timeout, check=False) + if result.returncode: + raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}') + return result.stdout.strip() + + +def request(base, method, path, body=None, token=None, basic=None, key=None): + headers = {'Accept': 'application/json'} + if token: + headers['Authorization'] = 'Bearer ' + token + if basic: + headers['Authorization'] = 'Basic ' + base64.b64encode( + (basic[0] + ':' + basic[1]).encode()).decode() + if key: + headers['Idempotency-Key'] = key + data = None + if body is not None: + data = json.dumps(body).encode() + headers['Content-Type'] = 'application/json' + req = urllib.request.Request(base + path, data=data, headers=headers, + method=method) + try: + with urllib.request.urlopen(req, timeout=3) as response: + raw = response.read() + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as exc: + raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None + + +def wait_http(base): + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + try: + if request(base, 'GET', '/health/ready')[0] == 200: + return + except RuntimeError: + pass + time.sleep(0.2) + raise RuntimeError('temporary API did not become ready') + + +def publish_port(name): + value = docker('port', name, '8080/tcp') + return 'http://127.0.0.1:' + value.rsplit(':', 1)[1] + + +def make_user(base, label): + username = 'compat_' + label + '_' + uuid.uuid4().hex[:8] + password = 'T3st-' + uuid.uuid4().hex + status, user = request(base, 'POST', '/api/v1/users', + {'username': username, 'display_name': label, + 'password': password}) + assert status == 201 + status, session = request(base, 'POST', '/api/v1/auth/sessions', + basic=(username, password)) + assert status == 201 + return user, session['token'] + + +def message(base, token, chat_id, text, key=None): + status, item = request(base, 'POST', + f'/api/v1/chats/{chat_id}/messages', + {'text': text}, token=token, key=key) + assert status == 201 + return item + + +def history(base, token, chat_id): + items = [] + cursor = None + for _ in range(100): + path = f'/api/v1/chats/{chat_id}/messages?limit=100' + if cursor: + path += '&cursor=' + quote(cursor, safe='') + status, page = request(base, 'GET', path, token=token) + assert status == 200 + items.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + return items + raise RuntimeError('history pagination exceeded safety bound') + + +def run(): + suffix = uuid.uuid4().hex[:10] + names = {label: 'lab3-compat-' + suffix + '-' + label + for label in ('pg', 'redis', 'old', 'new')} + password = secrets.token_urlsafe(18) + db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat' + cache_url = f'redis://{names["redis"]}:6379/0' + created = [] + try: + print('Starting isolated PostgreSQL and Redis...', flush=True) + docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK, + '--tmpfs', '/var/lib/postgresql/data:rw,mode=0700', + '-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app', + '-e', 'POSTGRES_PASSWORD=' + password, + 'postgres:17.6-bookworm') + created.append(names['pg']) + docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK, + 'redis:7.4.5-alpine', 'redis-server', '--save', '', + '--appendonly', 'no') + created.append(names['redis']) + deadline = time.monotonic() + 30 + while True: + try: + docker('exec', names['pg'], 'pg_isready', '-U', 'app', + '-d', 'compat', timeout=5) + break + except RuntimeError: + if time.monotonic() >= deadline: + raise RuntimeError('temporary PostgreSQL did not start') + time.sleep(0.3) + docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url, + OLD_MIGRATE, timeout=90) + print('Lab2 schema applied.', flush=True) + before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d', + 'compat', '-tAc', + "SELECT to_regclass('public.message_idempotency') IS NOT NULL") + assert before_table == 'f', 'new table exists before migration' + base_env = ['-e', 'DATABASE_URL=' + db_url, + '-e', 'CACHE_URL=' + cache_url, + '-e', 'CACHE_PREFIX=compat:' + suffix, + '-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080', + '-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres'] + docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK, + '-p', '127.0.0.1::8080', *base_env, OLD_API) + created.append(names['old']) + old_base = publish_port(names['old']) + wait_http(old_base) + print('Lab2 API ready; preparing synthetic chat.', flush=True) + alice, token = make_user(old_base, 'alice') + bob, _ = make_user(old_base, 'bob') + status, chat = request(old_base, 'POST', '/api/v1/chats', + {'title': 'Migration compatibility', + 'member_ids': [alice['id'], bob['id']]}, + token=token) + assert status == 201 + pre = message(old_base, token, chat['id'], 'before migration') + + stop = threading.Event() + events = [] + + def old_writer(): + index = 0 + while not stop.is_set(): + at = time.monotonic() + try: + item = message(old_base, token, chat['id'], + f'during migration {index}') + events.append((at, True, item['id'])) + except (RuntimeError, AssertionError): + events.append((at, False, None)) + index += 1 + stop.wait(0.3) + + thread = threading.Thread(target=old_writer, daemon=True) + thread.start() + time.sleep(0.5) + started = time.monotonic() + try: + print('Applying lab3 migration while lab2 API writes...', flush=True) + docker('run', '--rm', '--network', NETWORK, + '-e', 'DATABASE_URL=' + db_url, + NEW_MIGRATE, timeout=90) + finally: + migrated = time.monotonic() + time.sleep(0.7) + stop.set() + thread.join(timeout=5) + after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d', + 'compat', '-tAc', + "SELECT to_regclass('public.message_idempotency') IS NOT NULL") + assert after_table == 't', 'new table was not created' + assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration' + assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration' + assert all(ok for _, ok, _ in events), 'old API write failed near migration' + after = message(old_base, token, chat['id'], 'after migration') + print('Lab2 API still reads and writes; starting lab3 API.', flush=True) + + docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK, + '-p', '127.0.0.1::8080', *base_env, + '-e', 'INSTANCE_ID=compat-new', NEW_API) + created.append(names['new']) + new_base = publish_port(names['new']) + wait_http(new_base) + print('Lab3 API ready; checking mixed-version reads and writes.', flush=True) + status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token) + assert status == 200 and profile['id'] == alice['id'], 'session not shared' + key = uuid.uuid4().hex + new_message = message(new_base, token, chat['id'], 'from new version', key) + replay = message(new_base, token, chat['id'], 'from new version', key) + assert replay == new_message, 'new API idempotent replay changed response' + old_after_new = message(old_base, token, chat['id'], 'from old version') + for version, base in (('old', old_base), ('new', new_base)): + items = history(base, token, chat['id']) + ids = [item['id'] for item in items] + for label, item in (('pre', pre), ('after', after), + ('new', new_message), ('old-after-new', old_after_new)): + assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message' + result = { + 'old_image': OLD_API, 'new_image': NEW_API, + 'migration_image': NEW_MIGRATE, + 'migration_seconds': round(migrated - started, 3), + 'old_writes_near_migration': len(events), + 'old_write_errors': sum(not ok for _, ok, _ in events), + 'old_and_new_read_each_others_messages': True, + 'existing_session_accepted_by_new_api': True, + 'new_api_idempotent_replay': True, + 'temporary_database': True, + } + print(json.dumps(result, indent=2)) + print('PASS: additive migration compatible with running lab2 API') + finally: + print('Stopping isolated test containers...', flush=True) + for name in reversed(created): + try: + docker('stop', name, timeout=30) + except (RuntimeError, subprocess.TimeoutExpired): + print(f'WARNING: stop temporary container {name} manually', + file=sys.stderr) + + +if __name__ == '__main__': + try: + run() + except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc: + print(f'FAIL: {exc}', file=sys.stderr) + raise SystemExit(1) from None diff --git a/test/check_shutdown.py b/test/check_shutdown.py new file mode 100644 index 0000000..0b09eec --- /dev/null +++ b/test/check_shutdown.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM.""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +PROBE = r''' +import json, sys, threading, time, urllib.error, urllib.request +url = sys.argv[1] +live_url = url.replace('/health/ready', '/health/live') +phase = 'before' +events = [] +def commands(): + global phase + for line in sys.stdin: + phase = line.strip() + if phase == 'end': break +def status(address): + try: + with urllib.request.urlopen(address, timeout=1) as response: + return str(response.status) + except urllib.error.HTTPError as error: + return str(error.code) + except (urllib.error.URLError, TimeoutError, OSError): + return 'connection_error' +threading.Thread(target=commands, daemon=True).start() +print('READY', flush=True) +while phase != 'end': + current = phase + ready = status(url) + live = status(live_url) + events.append({'phase': current, 'ready': ready, 'live': live}) + time.sleep(0.05) +print(json.dumps(events), flush=True) +''' + + +def command(*args, timeout=45): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, + timeout=timeout, check=False) + if result.returncode: + raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})') + return result.stdout.strip() + + +def both_ready(base_url, expected): + seen = set() + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response: + seen.add(response.headers['X-Instance-Id']) + except (urllib.error.URLError, TimeoutError, OSError): + pass + if seen == expected: + return + time.sleep(0.1) + raise RuntimeError(f'both replicas did not recover: {seen}') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + args = parser.parse_args() + ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines() + if len(ids) != 2: + raise RuntimeError('shutdown probe requires two API replicas') + expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids} + target = ids[0] + networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target)) + network, settings = next(iter(networks.items())) + address = settings['IPAddress'] + nginx = command('docker', 'compose', 'ps', '-q', 'nginx') + both_ready(args.base_url, expected) + probe = subprocess.Popen( + ['docker', 'run', '--rm', '-i', '--network', network, + 'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE, + f'http://{address}:8080/health/ready'], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + stop = None + + def phase(value): + probe.stdin.write(value + '\n') + probe.stdin.flush() + + try: + if probe.stdout.readline().strip() != 'READY': + raise RuntimeError('direct readiness probe did not start') + time.sleep(1) + phase('stopping') + started = time.monotonic() + stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + withdrawn_at = None + withdrawn_while_running = False + while stop.poll() is None and time.monotonic() - started < 35: + upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10) + if f'server {address}:8080 ' not in upstreams: + withdrawn_at = time.monotonic() - started + withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true' + if withdrawn_while_running: + phase('withdrawn') + break + time.sleep(0.1) + stop.communicate(timeout=40) + if stop.returncode: + raise RuntimeError('docker stop failed') + stop_seconds = time.monotonic() - started + exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target)) + phase('stopped') + time.sleep(1) + phase('end') + output, _ = probe.communicate(timeout=15) + if probe.returncode: + raise RuntimeError('direct probe failed') + events = json.loads(output.strip()) + summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events) + for value in sorted({e['ready'] for e in events if e['phase'] == name})} + for name in ('before', 'stopping', 'withdrawn', 'stopped')} + draining = [e for e in events if e['ready'] == '503'] + first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None) + if summary['before'].get('200', 0) < 5: + raise RuntimeError('readiness was not healthy before SIGTERM') + if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]): + raise RuntimeError('expected readiness 200 -> 503 with no late 200') + # These are separate requests; transport failures can straddle HTTP + # close or the client's deadline. Do not confuse them with unhealthy + # HTTP responses. Record them separately rather than hiding them. + if not draining or not any(e['live'] == '200' for e in draining): + raise RuntimeError('liveness must remain 200 while readiness is 503') + if any(e['live'] not in {'200', 'connection_error'} for e in draining): + raise RuntimeError('liveness returned an unhealthy HTTP status during draining') + if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events): + raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive') + if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0): + raise RuntimeError('replica did not close HTTP after draining') + if exit_code == 137: + raise RuntimeError('container required SIGKILL instead of graceful shutdown') + print(json.dumps({'stop_seconds': round(stop_seconds, 3), + 'lb_withdrawn_seconds': round(withdrawn_at, 3), + 'lb_withdrawn_before_http_close': True, + 'readiness_sequence': [200, 503, 'connection_error'], + 'readiness_503_samples': len(draining), + 'liveness_200_while_draining': sum(e['live'] == '200' for e in draining), + 'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining), + 'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2)) + print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close') + finally: + if probe.poll() is None: + probe.kill() + probe.communicate() + if stop is not None and stop.poll() is None: + stop.communicate(timeout=40) + command('docker', 'start', target) + both_ready(args.base_url, expected) + + +if __name__ == '__main__': + try: + main() + except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc: + print(f'FAIL: {exc}', file=sys.stderr) + raise SystemExit(1) from None diff --git a/test/compose.benchmark.yaml b/test/compose.benchmark.yaml new file mode 100644 index 0000000..445d2d5 --- /dev/null +++ b/test/compose.benchmark.yaml @@ -0,0 +1,12 @@ +services: + benchmark: + image: ${BENCHMARK_IMAGE:-mtusi2026lab2-fork-backend} + working_dir: /app + environment: + NODE_ENV: production + DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat} + CACHE_URL: redis://redis:6379/0 + volumes: + - ./test:/app/test:ro + - ./evidence:/results + command: ['node', 'test/cache-benchmark.mjs', '${BENCHMARK_STAGE:-after}', '/results/cache-epochs-${BENCHMARK_STAGE:-after}.json'] diff --git a/test/compose.diagnostics.yaml b/test/compose.diagnostics.yaml new file mode 100644 index 0000000..ab4f627 --- /dev/null +++ b/test/compose.diagnostics.yaml @@ -0,0 +1,5 @@ +services: + api: + command: ['sh', '-c', 'INSTANCE_ID=$$(hostname) exec node --require /app/diagnostics/pool.cjs dist/main.js'] + volumes: + - ./scripts/diagnostics/pool.cjs:/app/diagnostics/pool.cjs:ro diff --git a/test/diagnostic-summary.py b/test/diagnostic-summary.py new file mode 100644 index 0000000..02a0ed3 --- /dev/null +++ b/test/diagnostic-summary.py @@ -0,0 +1,35 @@ +import importlib.util +from pathlib import Path +import unittest +import tempfile + +spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).resolve().parents[1] / "scripts/run-scaling-diagnostics.py") +diagnostics = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diagnostics) + + +class SummaryTest(unittest.TestCase): + def test_cumulative_counters_become_interval_rates(self): + summary = diagnostics.resource_summary([ + {"time_ms": 1000, "cpu_usage_usec": 100, "cpu_nr_periods": 10, + "cpu_nr_throttled": 1, "cpu_throttled_usec": 100, "memory_bytes": 1024**2}, + {"time_ms": 3000, "cpu_usage_usec": 1000100, "cpu_nr_periods": 30, + "cpu_nr_throttled": 6, "cpu_throttled_usec": 500100, "memory_bytes": 2*1024**2}, + ]) + self.assertEqual(summary["mean_cpu_cores"], 0.5) + self.assertEqual(summary["throttled_period_fraction"], 0.25) + self.assertEqual(summary["throttled_seconds"], 0.5) + self.assertEqual(summary["peak_memory_mib"], 2) + + def test_short_sample_does_not_invent_rates(self): + self.assertEqual(diagnostics.resource_summary([]), {"samples": 0}) + + def test_postgres_multiline_json_and_incomplete_tail(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "postgres.jsonl" + path.write_text('{"kind":"postgres",\n"activity":[\n{"state":"active"}\n]}\n{"kind":"cgroup"}\n{"unfinished":', encoding="utf-8") + self.assertEqual([r["kind"] for r in diagnostics.rows(path)], ["postgres", "cgroup"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/draining.mjs b/test/draining.mjs new file mode 100644 index 0000000..2a1ffaf --- /dev/null +++ b/test/draining.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { setImmediate, setTimeout } from 'node:timers/promises'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { Module } = require('@nestjs/common'); +const { NestFactory } = require('@nestjs/core'); +const { AppController } = require('../dist/app.controller'); +const { AppService } = require('../dist/app.service'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { createFastifyAdapter } = require('../dist/common/http/fastify-adapter'); + +async function fixture(t, query = async () => []) { + const state = { queries: 0, destroyed: false }; + const database = { + async $queryRaw() { + state.queries += 1; + return query(); + }, + onModuleDestroy() { + state.destroyed = true; + }, + }; + class FixtureModule {} + Module({ + controllers: [AppController], + providers: [AppService, { provide: PrismaService, useValue: database }], + })(FixtureModule); + const adapter = createFastifyAdapter(30); + const app = await NestFactory.create(FixtureModule, adapter, { logger: false }); + t.after(async () => { + state.release?.(); + await app.close(); + }); + return { app, adapter, state, controller: app.get(AppController) }; +} + +test('draining keeps liveness and dependencies alive until active HTTP work completes', { timeout: 30000 }, async (t) => { + const { app, adapter, state, controller } = await fixture(t); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + // Always release the request, including when an assertion fails. + state.release = () => release.resolve(); + adapter.getInstance().get('/slow-work', async () => { + entered.resolve(); + await release.promise; + assert.equal(state.destroyed, false, 'database was destroyed during active work'); + return { completed: true }; + }); + await app.listen(0, '127.0.0.1'); + t.diagnostic('HTTP listener started'); + const { port } = app.getHttpServer().address(); + await controller.ready(); + const response = fetch(`http://127.0.0.1:${port}/slow-work`, { signal: AbortSignal.timeout(5000) }); + await entered.promise; + t.diagnostic('Slow request started'); + let closed = false; + const closing = app.close('SIGTERM').then(() => { closed = true; }); + await setImmediate(); + assert.equal(adapter.isDraining, true); + assert.equal(adapter.beforeClose(), adapter.beforeClose(), 'shutdown must be idempotent'); + await assert.rejects(controller.ready(), (error) => error.getStatus() === 503); + assert.equal(state.queries, 1, 'draining readiness must not acquire a DB connection'); + assert.deepEqual(controller.live(), { status: 'ok' }); + await setTimeout(80); + assert.equal(closed, false, 'server closed before the active request completed'); + assert.equal(state.destroyed, false, 'dependencies closed before the active request'); + release.resolve(); + t.diagnostic('Slow request released'); + assert.deepEqual(await (await response).json(), { completed: true }); + await closing; + t.diagnostic('Application shutdown completed'); + assert.equal(state.destroyed, true); +}); + +test('readiness already querying the DB cannot return 200 after draining begins', { timeout: 10000 }, async (t) => { + const query = Promise.withResolvers(); + const { app, adapter, controller } = await fixture(t, () => query.promise); + await app.init(); + const pending = assert.rejects(controller.ready(), (error) => error.getStatus() === 503); + const closing = app.close('SIGTERM'); + await setImmediate(); + assert.equal(adapter.isDraining, true); + query.resolve([]); + await pending; + await closing; +}); diff --git a/test/epoch-variants-benchmark.mjs b/test/epoch-variants-benchmark.mjs new file mode 100644 index 0000000..52dbb55 --- /dev/null +++ b/test/epoch-variants-benchmark.mjs @@ -0,0 +1,475 @@ +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import Module from 'node:module'; +import { readFile, writeFile } from 'node:fs/promises'; +import { performance } from 'node:perf_hooks'; +import { faultProxy } from './helpers/fault-proxy.mjs'; + +const require = createRequire('/app/package.json'); +const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8')); +const pg = require('pg'); +let counts = {}; +const originalQuery = pg.Client.prototype.query; +pg.Client.prototype.query = function (query, ...args) { + const sql = typeof query === 'string' ? query : query.text; + const verb = sql.trim().split(/\s+/)[0].toUpperCase(); + counts[verb] = (counts[verb] ?? 0) + 1; + return originalQuery.call(this, query, ...args); +}; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('/app/dist/app.module'); +const { PrismaService } = require('/app/dist/prisma/prisma.service'); +const { CacheService } = require('/app/dist/cache/cache.service'); +const { ChatService } = require('/app/dist/Chat/chat.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('/app/dist/common/http/fastify-adapter'); +const exportedClasses = { + 'Auth/auth.service.ts': 'AuthService', + 'User/user.service.ts': 'UserService', + 'User/user.guard.ts': 'UserGuard', + 'Chat/chat.service.ts': 'ChatService', + 'Chat/chat.controller.ts': 'ChatController', +}; +const originals = {}; +const variants = {}; +for (const [relative, name] of Object.entries(exportedClasses)) { + const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`; + originals[name] = require(filename)[name]; +} +for (const variant of manifest.variants) { + variants[variant.name] = {}; + for (const [relative, name] of Object.entries(exportedClasses)) { + const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`; + const source = await readFile( + `/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`, + 'utf8', + ); + const loaded = new Module(filename); + loaded.filename = filename; + loaded.paths = Module._nodeModulePaths('/app'); + loaded._compile(source, filename); + variants[variant.name][name] = loaded.exports[name]; + } +} +function activate(name) { + for (const [className, replacement] of Object.entries(variants[name])) { + const target = originals[className].prototype; + for (const method of Object.getOwnPropertyNames(replacement.prototype)) { + if (method === 'constructor') continue; + const descriptor = Object.getOwnPropertyDescriptor( + replacement.prototype, + method, + ); + const previous = target[method]; + if ( + typeof descriptor.value === 'function' && + typeof previous === 'function' + ) { + for (const key of Reflect.getMetadataKeys(previous)) + Reflect.defineMetadata( + key, + Reflect.getMetadata(key, previous), + descriptor.value, + ); + } + Object.defineProperty(target, method, descriptor); + } + } +} + +const Redis = require('ioredis').default; +const prefix = `experiment:epoch-variants:${randomUUID()}`; +const proxy = await faultProxy(process.env.CACHE_URL, 6379); +const observer = new Redis(process.env.CACHE_URL); +process.env.CACHE_URL = proxy.url; +const apps = {}; +const chatIds = []; +const userIds = []; +let prisma, token; +const report = { + recorded_at: new Date().toISOString(), + methodology: { + concurrency: 32, + duration_seconds: 60, + warmup_seconds: 10, + repetitions: 3, + seed_messages: 100000, + writes_per_repeat: 4096, + order: [ + ['baseline', 'profile-join', 'access-join', 'striped'], + ['striped', 'access-join', 'profile-join', 'baseline'], + ['profile-join', 'striped', 'baseline', 'access-join'], + ], + requests: + 'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes', + resources: + 'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time', + sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size', + }, + variants: manifest.variants, + samples: [], + correctness: {}, + completed: false, +}; +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(name, method, url, body, bearer = token) { + const response = await fetch(`${apps[name].url}${url}`, { + method, + headers: { + authorization: `Bearer ${bearer}`, + ...(body ? { 'content-type': 'application/json' } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + signal: AbortSignal.timeout(10000), + }); + const json = await response.json(); + return { status: response.status, json }; +} +async function call(name, method, url, body) { + const result = await request(name, method, url, body); + assert.equal(result.status, method === 'POST' ? 201 : 200); + return result.json; +} +async function clearCache() { + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); +} +async function newChat(userId) { + const chat = await prisma.chat.create({ + data: { + title: 'Variant benchmark', + members: { create: { userId, position: 0 } }, + }, + }); + chatIds.push(chat.id); + return chat.id; +} +async function epoch(name, chatId) { + if (name !== 'striped') + return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } })) + .messagesEpoch; + const [row] = await prisma.$queryRawUnsafe( + `SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`, + chatId, + ); + return row.epoch; +} +async function measure(name, workload, repeat, operation, total) { + counts = {}; + const latencies = []; + let errors = 0, + index = 0; + const cpu = process.cpuUsage(); + const start = performance.now(); + await Promise.all( + Array.from({ length: 32 }, async () => { + while ( + total === undefined ? performance.now() - start < 60000 : index < total + ) { + const item = index++; + const began = performance.now(); + try { + await operation(item); + latencies.push(performance.now() - began); + } catch { + errors++; + } + } + }), + ); + const seconds = (performance.now() - start) / 1000; + const used = process.cpuUsage(cpu); + latencies.sort((a, b) => a - b); + const calls = { ...counts }; + const queries = Object.entries(calls) + .filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb)) + .reduce((sum, [, count]) => sum + count, 0); + const sample = { + variant: name, + workload, + repeat, + successes: latencies.length, + errors, + elapsed_seconds: +seconds.toFixed(3), + rps: +(latencies.length / seconds).toFixed(3), + mean_latency_ms: +( + latencies.reduce((sum, value) => sum + value, 0) / latencies.length + ).toFixed(3), + p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3), + sql_queries: queries, + sql_per_request: +(queries / (latencies.length + errors)).toFixed(4), + sql_calls: calls, + cpu_percent_one_core: +( + (used.user + used.system) / + (seconds * 10000) + ).toFixed(2), + rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2), + }; + report.samples.push(sample); + console.log(JSON.stringify(sample)); + assert.equal(errors, 0, 'HTTP workload errors'); +} +async function reconnect() { + proxy.setMode('online'); + for (let attempt = 0; attempt < 80; attempt++) { + if ( + Object.values(apps).every( + ({ app }) => app.get(CacheService).redis.status === 'ready', + ) + ) + return; + await pause(100); + } + assert.fail('Redis reconnect timeout'); +} +async function correctness(name, userId, otherToken) { + activate(name); + await clearCache(); + const replica = Object.keys(apps).find((candidate) => candidate !== name); + const chatId = await newChat(userId); + const url = `/api/v1/chats/${chatId}/messages`; + assert.equal( + ( + await request( + name, + 'GET', + url, + undefined, + randomBytes(32).toString('base64url'), + ) + ).status, + 401, + ); + assert.equal( + (await request(name, 'GET', url, undefined, otherToken)).status, + 404, + ); + assert.equal( + (await request(name, 'POST', url, { text: 'Forbidden' }, otherToken)) + .status, + 404, + ); + assert.equal( + (await request(name, 'GET', '/api/v1/chats/invalid/messages')).status, + 400, + ); + assert.equal( + ( + await request( + name, + 'GET', + '/api/v1/chats/invalid/messages', + undefined, + randomBytes(32).toString('base64url'), + ) + ).status, + 401, + ); + assert.equal((await call(replica, 'GET', url)).items.length, 0); + const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`); + assert.ok(keys.length); + proxy.setMode('cut'); + await pause(100); + const posted = await call(name, 'POST', url, { text: 'During outage' }); + assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id); + await reconnect(); + assert.equal(await observer.exists(...keys), keys.length); + assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id); + const service = apps[name].app.get(ChatService); + const actual = service.prisma; + service.prisma = actual.$extends({ + query: { + $queryRaw({ args, query }) { + const index = args.values.indexOf('Must roll back'); + if (index !== -1) args.values[index] = 'x'.repeat(4001); + return query(args); + }, + }, + }); + try { + assert.equal( + (await request(name, 'POST', url, { text: 'Must roll back' })).status, + 500, + ); + } finally { + service.prisma = actual; + } + assert.equal(await epoch(name, chatId), 1n); + assert.equal(await prisma.message.count({ where: { chatId } }), 1); + const raceChat = await newChat(userId); + const raceUrl = `/api/v1/chats/${raceChat}/messages`; + let began, release; + const started = new Promise((resolve) => { + began = resolve; + }); + const held = new Promise((resolve) => { + release = resolve; + }); + service.prisma = actual.$extends({ + query: { + message: { + async findMany({ args, query }) { + const rows = await query(args); + if (args.where?.chatId === raceChat) { + began(); + await held; + } + return rows; + }, + }, + }, + }); + const pending = call(name, 'GET', raceUrl); + try { + await Promise.race([ + started, + pause(10000).then(() => { + throw new Error('Race load timeout'); + }), + ]); + const fresh = await call(replica, 'POST', raceUrl, { + text: 'After old SELECT', + }); + release(); + await pending; + assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id); + } finally { + release(); + service.prisma = actual; + } + await clearCache(); + assert.equal((await call(name, 'GET', url)).items[0].id, posted.id); + report.correctness[name] = { + access_statuses: true, + redis_outage_and_retained_keys: true, + rollback: true, + delayed_old_fill: true, + empty_cache: true, + }; + console.log(JSON.stringify({ correctness: name, result: 'PASS' })); +} + +try { + for (const variant of manifest.variants) { + activate(variant.name); + process.env.CACHE_PREFIX = `${prefix}:${variant.name}`; + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + apps[variant.name] = { app }; + exposeRequestIdHeader(app); + await app.listen(0, '127.0.0.1'); + require('nestjs-pino').PinoLogger.root.level = 'silent'; + apps[variant.name].url = await app.getUrl(); + } + prisma = apps.baseline.app.get(PrismaService); + const user = await prisma.user.create({ + data: { + username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Variant user', + }, + }); + userIds.push(user.id); + token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId: user.id, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 3600000), + }, + }); + const other = await prisma.user.create({ + data: { + username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Other user', + }, + }); + userIds.push(other.id); + const otherToken = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId: other.id, + tokenHash: createHash('sha256').update(otherToken).digest('hex'), + expiresAt: new Date(Date.now() + 3600000), + }, + }); + for (const name of Object.keys(apps)) + await correctness(name, user.id, otherToken); + const readChat = await newChat(user.id); + for (let offset = 0; offset < 100000; offset += 1000) { + await prisma.message.createMany({ + data: Array.from({ length: 1000 }, (_, i) => ({ + chatId: readChat, + senderId: user.id, + text: `Synthetic message ${offset + i}`, + createdAt: new Date(1700000000000 + offset + i), + })), + }); + } + await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } }); + await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat); + for (const [index, order] of report.methodology.order.entries()) { + for (const name of order) { + activate(name); + await clearCache(); + const url = `/api/v1/chats/${readChat}/messages?limit=50`; + const warmUntil = performance.now() + 10000; + while (performance.now() < warmUntil) await call(name, 'GET', url); + await measure(name, 'warm-history', index + 1, () => + call(name, 'GET', url), + ); + const writeChat = await newChat(user.id); + const writeUrl = `/api/v1/chats/${writeChat}/messages`; + assert.equal((await call(name, 'GET', writeUrl)).items.length, 0); + await measure( + name, + 'writes-single-chat', + index + 1, + (i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }), + 4096, + ); + assert.equal( + await prisma.message.count({ where: { chatId: writeChat } }), + 4096, + ); + assert.equal(await epoch(name, writeChat), 4096n); + assert.equal((await call(name, 'GET', writeUrl)).items.length, 50); + } + } + report.completed = true; +} finally { + await reconnect(); + if (prisma) { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + await prisma.user.deleteMany({ where: { id: { in: userIds } } }); + } + await clearCache(); + for (const { app } of Object.values(apps)) await app.close(); + observer.disconnect(); + await proxy.close(); + await writeFile( + '/results/epoch-variants.json', + JSON.stringify(report, null, 2) + '\n', + ); +} +console.log( + JSON.stringify({ + completed: report.completed, + correctness: report.correctness, + }), +); diff --git a/test/epoch-variants-experiment.cjs b/test/epoch-variants-experiment.cjs new file mode 100644 index 0000000..77fed41 --- /dev/null +++ b/test/epoch-variants-experiment.cjs @@ -0,0 +1,261 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { randomUUID, createHash } = require('node:crypto'); +const { spawn } = require('node:child_process'); +const ts = require('typescript'); +require('dotenv').config({ quiet: true }); + +const root = path.resolve(__dirname, '..'); +const statePath = path.join(root, '.epoch-variants-state.json'); +const files = [ + 'Auth/auth.service.ts', + 'User/user.service.ts', + 'User/user.guard.ts', + 'Chat/chat.service.ts', + 'Chat/chat.controller.ts', +]; +const digest = (value) => createHash('sha256').update(value).digest('hex'); +const command = process.argv[2]; + +async function main() { + let state; + if (command === 'init') { + assert.ok(!fs.existsSync(statePath)); + assert.ok( + /messagesEpoch\s+BigInt/.test( + fs.readFileSync(path.join(root, 'prisma/schema.prisma'), 'utf8'), + ), + 'Historical comparison requires the pre-bucket schema. Original sources are saved in evidence/epoch-variants-sources.json', + ); + state = { + directory: fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-epoch-variants-')), + schema: 'epoch_variants_' + randomUUID().replaceAll('-', ''), + variants: [], + }; + await database(async (client) => { + await client.query(`CREATE SCHEMA "${state.schema}"`); + await client.query( + `CREATE TABLE "${state.schema}".chat_message_epochs (chat_id uuid NOT NULL REFERENCES public.chats(id) ON DELETE CASCADE, bucket smallint NOT NULL CHECK (bucket BETWEEN 0 AND 15), epoch bigint NOT NULL DEFAULT 0, PRIMARY KEY (chat_id, bucket))`, + ); + }); + capture(state, 'baseline'); + } else { + state = JSON.parse(fs.readFileSync(statePath, 'utf8')); + } + if (command === 'capture') capture(state, process.argv[3]); + if (command === 'capture-striped') capture(state, 'striped', true); + if (command === 'apply-striped') { + const sources = {}; + for (const variant of state.variants) { + sources[variant.name] = Object.fromEntries( + files.map((relative) => [ + relative, + fs.readFileSync( + path.join(state.directory, variant.name, relative), + 'utf8', + ), + ]), + ); + } + fs.writeFileSync( + path.join(root, 'evidence/epoch-variants-sources.json'), + JSON.stringify(sources, null, 2) + '\n', + ); + for (const relative of ['Auth/auth.service.ts', 'Chat/chat.service.ts']) { + const source = sources.striped[relative].replaceAll( + `"${state.schema}".chat_message_epochs`, + '"public"."chat_message_epochs"', + ); + fs.writeFileSync(path.join(root, 'src', relative), source); + } + console.log( + 'Applied striped epochs to application sources; saved all source snapshots', + ); + } + if (['init', 'capture', 'capture-striped'].includes(command)) { + fs.writeFileSync(statePath, JSON.stringify(state, null, 2)); + console.log( + 'Captured:', + state.variants.map((variant) => variant.name).join(', '), + ); + return; + } + if (command === 'run') { + assert.equal(state.variants.length, 4); + fs.writeFileSync( + path.join(state.directory, 'manifest.json'), + JSON.stringify(state, null, 2), + ); + const override = path.join(state.directory, 'compose.json'); + fs.writeFileSync( + override, + JSON.stringify({ + services: { + benchmark: { + volumes: [ + { + type: 'bind', + source: state.directory, + target: '/variants', + read_only: true, + }, + ], + }, + }, + }), + ); + const args = [ + 'compose', + '-f', + 'compose.yaml', + '-f', + 'test/compose.benchmark.yaml', + '-f', + override, + 'run', + '--rm', + '--no-deps', + '--pull', + 'never', + 'benchmark', + 'node', + 'test/epoch-variants-benchmark.mjs', + ]; + const code = await new Promise((resolve, reject) => { + const child = spawn('docker', args, { cwd: root, stdio: 'inherit' }); + child.once('error', reject); + child.once('exit', resolve); + }); + fs.writeFileSync( + path.join(root, 'evidence/epoch-variants-code.json'), + JSON.stringify( + { + variants: state.variants, + schema: state.schema, + exit_code: code, + implementation: + 'TypeScript source snapshots; variant methods activated sequentially before HTTP workloads. Separate cache prefixes and API instances, shared fixtures and Node process.', + }, + null, + 2, + ), + ); + process.exitCode = code ?? 1; + } + if (command === 'seed-read-fixture') { + await database(async (client) => { + const result = await client.query( + `SELECT c.id FROM public.chats c JOIN public.messages m ON m.chat_id = c.id WHERE c.title = 'Variant benchmark' GROUP BY c.id HAVING COUNT(*) = 100000`, + ); + assert.equal(result.rows.length, 1); + const chatId = result.rows[0].id; + await client.query( + 'UPDATE public.chats SET messages_epoch = 100000 WHERE id = $1::uuid', + [chatId], + ); + await client.query( + `INSERT INTO "${state.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, + [chatId], + ); + console.log('Seeded read fixture with 16 nonzero epoch buckets'); + }); + } + if (command === 'cleanup') { + await database((client) => + client.query(`DROP SCHEMA "${state.schema}" CASCADE`), + ); + const resolved = path.resolve(state.directory); + assert.equal(path.dirname(resolved), path.resolve(os.tmpdir())); + assert.ok(path.basename(resolved).startsWith('lab2-epoch-variants-')); + fs.rmSync(resolved, { recursive: true, force: true }); + fs.unlinkSync(statePath); + console.log('Removed experiment schema and temporary source snapshots'); + } +} + +function capture(state, name, striped = false) { + assert.match(name, /^[a-z-]+$/); + assert.ok(!state.variants.some((variant) => variant.name === name)); + const modules = {}; + const table = `"${state.schema}".chat_message_epochs`; + for (const relative of files) { + let source = fs + .readFileSync(path.join(root, 'src', relative), 'utf8') + .replace(/\r\n/g, '\n'); + if ( + striped && + ['Auth/auth.service.ts', 'Chat/chat.service.ts'].includes(relative) + ) { + source = source.replaceAll( + 'c.messages_epoch', + `COALESCE((SELECT SUM(e.epoch) FROM ${table} AS e WHERE e.chat_id = c.id), 0)::bigint`, + ); + if (relative === 'Chat/chat.service.ts') { + const start = source.indexOf(' const [message] ='); + const end = source.indexOf(' if (!message)', start); + assert.ok(start !== -1 && end > start); + source = + source.slice(0, start) + + ' const messageId = randomUUID();\n' + + ' const bucket = parseInt(messageId.slice(0, 2), 16) % 16;\n' + + ' const [message] = await this.prisma.$queryRaw(Prisma.sql`\n' + + ` WITH updated AS (\n INSERT INTO ${table} AS e (chat_id, bucket, epoch)\n` + + ' SELECT cm.chat_id, ${bucket}::smallint, 1 FROM chat_members AS cm\n' + + ' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' + + ' ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = e.epoch + 1\n' + + ' RETURNING chat_id\n )\n' + + ' INSERT INTO messages (id, chat_id, sender_id, text)\n' + + ' SELECT ${messageId}::uuid, updated.chat_id, ${userId}::uuid, ${text} FROM updated\n' + + ' RETURNING id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt"\n' + + ' `);\n' + + source.slice(end); + } + } + const compiled = ts.transpileModule(source, { + compilerOptions: { + target: ts.ScriptTarget.ES2023, + module: ts.ModuleKind.CommonJS, + experimentalDecorators: true, + emitDecoratorMetadata: true, + }, + reportDiagnostics: true, + }); + assert.ok( + !compiled.diagnostics.some( + (diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error, + ), + ); + const destination = path.join( + state.directory, + name, + relative.replace(/\.ts$/, '.js'), + ); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.writeFileSync(destination, compiled.outputText); + fs.writeFileSync(destination.replace(/\.js$/, '.ts'), source); + modules[relative] = { + source_sha256: digest(source), + js_sha256: digest(compiled.outputText), + }; + } + state.variants.push({ name, modules, striped }); +} + +async function database(operation) { + const client = new (require('pg').Client)({ + connectionString: process.env.DATABASE_URL, + }); + await client.connect(); + try { + return await operation(client); + } finally { + await client.end(); + } +} + +main().catch((error) => { + console.error(error.message); + process.exitCode = 1; +}); diff --git a/test/epoch-variants-summary.mjs b/test/epoch-variants-summary.mjs new file mode 100644 index 0000000..4055449 --- /dev/null +++ b/test/epoch-variants-summary.mjs @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; + +const report = JSON.parse( + await readFile('evidence/epoch-variants.json', 'utf8'), +); +assert.ok(report.completed); +const labels = { + baseline: 'Текущая схема', + 'profile-join': 'Профиль из JOIN', + 'access-join': 'Профиль + единая проверка доступа', + striped: 'Единая проверка + 16 счётчиков', +}; +const aggregates = []; +const lines = [ + '# Сравнение трёх оптимизаций эпох и текущей схемы', + '', + `Дата UTC: ${report.recorded_at}.`, + '', + 'Все режимы используют Redis. Варианты последовательно добавляют оптимизации: профиль из запроса сессии; объединённую проверку сессии, членства и эпохи; 16 счётчиков истории, увеличиваемых независимо, с суммой как версией кэша. Порядок вариантов меняется между тремя повторами. Чтения: 100 000 сообщений, limit=50, конкурентность 32, прогрев 10 с, измерение 60 с. Запись: 4096 сообщений в отдельный новый чат на вариант/повтор. Только один workload активен одновременно.', + '', + 'RPS рассчитан по общему числу успешных запросов и времени повторов. Средняя задержка взвешена по числу запросов. p95 — среднее p95 отдельных повторов, а не перцентиль объединённой выборки. SQL/HTTP исключает BEGIN/COMMIT/ROLLBACK/SET.', + '', + '| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | RPS по повторам | Ошибки |', + '| --- | --- | ---: | ---: | ---: | ---: | --- | ---: |', +]; +for (const workload of ['warm-history', 'writes-single-chat']) { + for (const [name, label] of Object.entries(labels)) { + const samples = report.samples.filter( + (sample) => sample.variant === name && sample.workload === workload, + ); + assert.equal(samples.length, 3); + const sum = (select) => + samples.reduce((total, sample) => total + select(sample), 0); + const successes = sum((sample) => sample.successes); + const errors = sum((sample) => sample.errors); + const aggregate = { + variant: name, + workload, + successes, + errors, + rps: successes / sum((sample) => sample.elapsed_seconds), + mean_latency_ms: + sum((sample) => sample.mean_latency_ms * sample.successes) / successes, + mean_p95_ms: sum((sample) => sample.p95_ms) / samples.length, + sql_per_request: + sum((sample) => sample.sql_queries) / (successes + errors), + repeat_rps: samples.map((sample) => sample.rps), + }; + aggregates.push(aggregate); + lines.push( + `| ${workload} | ${label} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.repeat_rps.map((value) => value.toFixed(1)).join(' / ')} | ${errors} |`, + ); + assert.ok(Object.values(report.correctness[name]).every(Boolean)); + } +} +lines.push( + '', + 'Все варианты прошли проверки 401/404/400, доступа постороннего пользователя, чтения после записи при недоступном Redis и после восстановления со старыми ключами, отката счётчика при ошибке INSERT, задержанного заполнения старого кэша и работы с пустым кэшем. После каждого нагрузочного блока проверены число сообщений и значение эпохи 4096.', + '', + 'Четыре API и HTTP-генератор работают в одном Node-процессе, каждый API имеет отдельный Redis-prefix и пул Prisma прежнего размера. Методы переключаются между скомпилированными снимками исходников перед каждым workload; скрипт сохраняет метаданные маршрутов. Idle-экземпляры присутствуют во всех режимах. CPU/RSS включает генератор и все экземпляры. Другие сервисы Docker-host продолжают работать. Сессии создаются напрямую, KDF не измеряется. Порядок меняется, но три повтора и фоновые нагрузки всё ещё ограничивают выводы об устойчивых коэффициентах ускорения.', + '', + 'Распределённые счётчики находятся в отдельной временной схеме. Рабочая схема приложения не переключается на них. В нагрузочном чате 16 ненулевых счётчиков; стоимость их суммирования входит в запрос доступа. Первое чтение baseline выполнено до заполнения счётчиков временной схемы; данные истории неизменны, счётчики подготовлены до первого измерения striped. Это изменение подготовки фикстуры добавлено в скрипт для следующих запусков.', + '', + 'Сырые результаты: [epoch-variants.json](epoch-variants.json). Хеши исходников вариантов: [epoch-variants-code.json](epoch-variants-code.json).', +); +await writeFile('evidence/epoch-variants-summary.md', lines.join('\n') + '\n'); +await writeFile( + 'evidence/epoch-variants-averages.json', + JSON.stringify(aggregates, null, 2) + '\n', +); +console.log(lines.slice(0, 19).join('\n')); diff --git a/test/explain-history.mjs b/test/explain-history.mjs new file mode 100644 index 0000000..f2560fa --- /dev/null +++ b/test/explain-history.mjs @@ -0,0 +1,195 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const require = createRequire(import.meta.url); +const { Client } = require('pg'); +assert.ok(process.env.DATABASE_URL, 'Set DATABASE_URL for the project PostgreSQL'); +const output = resolve('evidence'); +const schema = `explain_history_${randomUUID().replaceAll('-', '')}`; +const table = `"${schema}"."messages"`; +const client = new Client({ + connectionString: process.env.DATABASE_URL, + connectionTimeoutMillis: 5000, + query_timeout: 60000, + options: '-c statement_timeout=60000 -c lock_timeout=2000', +}); +const seed = ` +WITH base AS ( + SELECT n, CASE WHEN (n - 1) % 6 < 2 THEN 0 ELSE (n - 1) % 6 - 1 END AS chat_no + FROM generate_series(1, 60000) AS n +), ranked AS ( + SELECT n, chat_no, row_number() OVER (PARTITION BY chat_no ORDER BY n) AS chat_seq + FROM base +) +INSERT INTO ${table} (id, chat_id, sender_id, text, created_at) +SELECT md5('message:' || n)::uuid, md5('chat:' || chat_no)::uuid, + md5('sender:' || (n % 20))::uuid, + rpad('Synthetic message ' || n || ' ', 256, '.'), + '2026-01-01T00:00:00Z'::timestamptz + ((chat_seq - 1) / 10) * interval '1 second' +FROM ranked ORDER BY n`; +const columns = 'id, chat_id, sender_id, text, created_at'; +const firstSql = `SELECT ${columns} FROM ${table} +WHERE chat_id = $1::uuid ORDER BY created_at ASC, id ASC LIMIT $2`; +const orSql = `SELECT ${columns} FROM ${table} +WHERE chat_id = $1::uuid + AND (created_at > $2::timestamptz OR (created_at = $2::timestamptz AND id > $3::uuid)) +ORDER BY created_at ASC, id ASC LIMIT $4`; +const tupleSql = `SELECT ${columns} FROM ${table} +WHERE chat_id = $1::uuid AND (created_at, id) > ($2::timestamptz, $3::uuid) +ORDER BY created_at ASC, id ASC LIMIT $4`; + +function nodes(plan) { + return [plan, ...(plan.Plans ?? []).flatMap(nodes)]; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} + +function inspectDocker() { + const template = '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"server_version":{{json .ServerVersion}},"os":{{json .OperatingSystem}},"arch":{{json .Architecture}}}'; + const result = spawnSync('docker', ['info', '--format', template], { + encoding: 'utf8', timeout: 10000, windowsHide: true, + }); + try { + if (result.status !== 0) return null; + const info = JSON.parse(result.stdout); + const ids = spawnSync('docker', ['compose', 'ps', '--quiet', 'postgres'], { + encoding: 'utf8', timeout: 10000, windowsHide: true, + }); + if (ids.status === 0 && ids.stdout.trim()) { + const limitsTemplate = '{"image":{{json .Config.Image}},"memory_limit_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}},"cpu_period":{{.HostConfig.CpuPeriod}},"cpuset_cpus":{{json .HostConfig.CpusetCpus}}}'; + const limits = spawnSync('docker', ['inspect', '--format', limitsTemplate, ids.stdout.trim()], { + encoding: 'utf8', timeout: 10000, windowsHide: true, + }); + if (limits.status === 0) info.postgres_container = JSON.parse(limits.stdout); + } + return info; + } catch { + return null; + } +} + +await client.connect(); +const runs = []; +const raw = []; +const signatures = new Map(); +let metadata; +let queries; +let cleanup = false; +let sourceIndexes; +try { + sourceIndexes = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows; + await client.query('BEGIN'); + await client.query(`CREATE SCHEMA "${schema}"`); + await client.query(`CREATE TABLE ${table} (LIKE public.messages INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE)`); + await client.query(seed); + await client.query(`ANALYZE ${table}`); + const distribution = (await client.query(`SELECT chat_id, count(*)::integer AS messages FROM ${table} GROUP BY chat_id ORDER BY messages DESC, chat_id`)).rows; + assert.equal(distribution.reduce((sum, row) => sum + row.messages, 0), 60000); + const chatId = distribution.find((row) => row.messages === 20000).chat_id; + const cursor = async (offset) => (await client.query( + `SELECT id, created_at FROM ${table} WHERE chat_id = $1::uuid ORDER BY created_at, id OFFSET $2 LIMIT 1`, + [chatId, offset], + )).rows[0]; + const middle = await cursor(9999); + const late = await cursor(17999); + queries = [ + { name: 'first_page', sql: firstSql, parameters: [chatId, 51] }, + { name: 'middle_page_or', sql: orSql, parameters: [chatId, middle.created_at.toISOString(), middle.id, 51] }, + { name: 'late_page_or', sql: orSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] }, + { name: 'late_page_tuple', sql: tupleSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] }, + ]; + const settings = (await client.query( + 'SELECT name, setting, unit FROM pg_settings WHERE name = ANY($1::text[]) ORDER BY name', + [['server_version', 'shared_buffers', 'work_mem', 'random_page_cost', 'effective_cache_size', 'max_parallel_workers_per_gather', 'jit', 'block_size']], + )).rows; + metadata = { + recorded_at: new Date().toISOString(), + command: 'node test/explain-history.mjs', + node_version: process.version, + docker: inspectDocker(), + settings, + source_table_indexes_before: sourceIndexes, + dataset: { total_messages: 60000, target_chat_messages: 20000, chats: 5, distribution, text_length: 256, timestamp_ties: 10, insertion_order: 'interleaved chats' }, + method: { warmup_runs_per_query_per_phase: 1, measured_runs: 5, page_size: 50, fetched_rows: 51, cursor_positions: [10000, 18000], cache: 'warm PostgreSQL buffers; Redis is bypassed', cleanup: 'ROLLBACK the experiment schema; live tables and indexes unchanged', planner: 'default scan settings; no forced index or disabled sequential scans', schema_difference: 'copied columns/checks/defaults; foreign keys omitted equally in all phases', tuple_variant: 'current cursor SQL in ChatService.listMessages; OR is the pre-fix baseline' }, + }; + const phases = [ + { name: 'no_indexes', ddl: [] }, + { name: 'without_history_index', ddl: [`CREATE UNIQUE INDEX messages_pkey ON ${table} (id)`, `CREATE INDEX messages_sender_id_idx ON ${table} (sender_id)`] }, + { name: 'with_history_index', ddl: [`CREATE INDEX messages_chat_id_created_at_id_idx ON ${table} (chat_id, created_at, id)`] }, + ]; + for (const phase of phases) { + for (const ddl of phase.ddl) await client.query(ddl); + await client.query(`ANALYZE ${table}`); + const indexes = (await client.query('SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = $1 ORDER BY indexname', [schema])).rows; + const phaseRuns = new Map(queries.map((query) => [query.name, []])); + for (const query of queries) { + const rows = (await client.query(query.sql, query.parameters)).rows; + assert.equal(rows.length, 51); + const signature = createHash('sha256').update(JSON.stringify(rows)).digest('hex'); + if (signatures.has(query.name)) assert.equal(signature, signatures.get(query.name)); + else signatures.set(query.name, signature); + if (query.name === 'late_page_tuple') assert.equal(signature, signatures.get('late_page_or')); + await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters); + } + for (let repetition = 1; repetition <= 5; repetition++) { + for (const query of queries) { + const plan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters)).rows[0]['QUERY PLAN'][0]; + phaseRuns.get(query.name).push(plan); + } + } + for (const query of queries) { + const plans = phaseRuns.get(query.name); + const medianMs = median(plans.map((plan) => plan['Execution Time'])); + const representative = plans.find((plan) => plan['Execution Time'] === medianMs); + const flat = nodes(representative.Plan); + const scan = flat.find((node) => node['Node Type'].includes('Scan')); + runs.push({ + phase: phase.name, query: query.name, indexes, sql: query.sql, parameters: query.parameters, + result_sha256: signatures.get(query.name), execution_ms: plans.map((plan) => plan['Execution Time']), + median_execution_ms: medianMs, representative_plan: representative, + scan_type: scan?.['Node Type'], index_name: scan?.['Index Name'] ?? null, + rows_removed_by_filter: scan?.['Rows Removed by Filter'] ?? 0, + scanned_rows: ((scan?.['Actual Rows'] ?? 0) + (scan?.['Rows Removed by Filter'] ?? 0)) * (scan?.['Actual Loops'] ?? 1), + sort: flat.some((node) => node['Node Type'] === 'Sort'), + shared_buffer_hits: representative.Plan['Shared Hit Blocks'] ?? 0, + shared_buffer_reads: representative.Plan['Shared Read Blocks'] ?? 0, + raw_plans: plans, + }); + const textPlan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS) ${query.sql}`, query.parameters)).rows.map((row) => row['QUERY PLAN']).join('\n'); + raw.push(`=== ${phase.name} / ${query.name} ===\nIndexes: ${indexes.map((index) => index.indexname).join(', ') || '(none)'}\nSQL:\n${query.sql}\nParameters: ${JSON.stringify(query.parameters)}\n\n${textPlan}\n`); + console.log(`${phase.name} / ${query.name}: median ${medianMs.toFixed(3)} ms; ${scan?.['Node Type']}; scanned ${runs.at(-1).scanned_rows}; sort=${runs.at(-1).sort}`); + } + } +} finally { + try { + await client.query('ROLLBACK'); + if (metadata) { + metadata.source_table_indexes_after = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows; + assert.deepEqual(metadata.source_table_indexes_after, sourceIndexes); + const leftovers = (await client.query('SELECT count(*)::integer AS count FROM pg_namespace WHERE nspname = $1', [schema])).rows[0].count; + assert.equal(leftovers, 0); + metadata.experiment_schema_absent = true; + } + cleanup = true; + } finally { + await client.end(); + } +} +assert.equal(cleanup, true); +await mkdir(output, { recursive: true }); +await writeFile(resolve(output, 'history-index-explain.json'), JSON.stringify({ ...metadata, cleanup_completed: cleanup, runs }, null, 2) + '\n'); +await writeFile(resolve(output, 'history-index-plans.txt'), raw.join('\n')); +const labels = { no_indexes: 'Без индексов', without_history_index: 'PK + sender_id, без индекса истории', with_history_index: 'С индексом истории' }; +const tableRows = runs.map((row) => `| ${labels[row.phase]} | ${row.query} | ${row.median_execution_ms.toFixed(3)} | ${row.scan_type} | ${row.scanned_rows} | ${row.sort ? 'да' : 'нет'} | ${row.shared_buffer_hits} / ${row.shared_buffer_reads} |`).join('\n'); +const speedup = (query) => runs.find((row) => row.query === query && row.phase === 'no_indexes').median_execution_ms / runs.find((row) => row.query === query && row.phase === 'with_history_index').median_execution_ms; +const summary = `# EXPLAIN истории сообщений: с индексами и без\n\nДата UTC: ${metadata.recorded_at}. PostgreSQL ${metadata.settings.find((setting) => setting.name === 'server_version').setting}.\n\nКоманда: \`node test/explain-history.mjs\` (нужен \`DATABASE_URL\`; PostgreSQL проекта с применёнными миграциями).\n\n60 000 синтетических сообщений в пяти чатах; в целевом чате 20 000. Чаты перемежаются при вставке, текст 256 символов, по 10 сообщений имеют одинаковый created_at. Страница API: 50 сообщений + 1 для next_cursor. Курсоры после 10 000-го и 18 000-го сообщений. Запросы первой страницы и с OR соответствуют прежнему \`ChatService.listMessages\`; tuple — исправленное условие курсора, используемое в текущем API. Скрипт сравнивает прежний и исправленный запросы; сам код API не меняет.\n\n| Индексы | Запрос | Медиана, мс | Scan | Проверено строк | Sort | Buffers hit/read |\n| --- | --- | ---: | --- | ---: | --- | ---: |\n${tableRows}\n\nПервая страница с индексом быстрее варианта без индексов в ${speedup('first_page').toFixed(1)} раза на этом наборе и стенде. Само наличие PK и индекса sender_id не заменяет индекс истории. Составной B-tree \`(chat_id, created_at, id)\` соответствует равенству chat_id и порядку истории; наличие Sort и границы Index Cond видны в сохранённых планах. ORDER BY нужен и при наличии индекса.\n\nПоздняя страница с OR: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_or').scanned_rows} проверенных строк. Эквивалентное сравнение пары: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_tuple').scanned_rows} строк. Это показывает, удалось ли передать границу курсора в Index Cond или она осталась Filter. Совпадение результатов всех фаз и OR/tuple проверено SHA-256 от полных 51 строк.\n\nМетодика: отдельная таблица с колонками текущей public.messages в случайной схеме; одинаковые данные во всех фазах; ANALYZE после загрузки и изменений индексов; 1 прогрев + 5 измерений каждого запроса в каждой фазе, медиана PostgreSQL Execution Time. Планы не принуждались настройками enable_seqscan/enable_indexscan. Фаза с индексом сохраняет PK и sender_id. Все создание таблиц, данных и индексов выполнены внутри транзакции; ROLLBACK удалил экспериментальную схему. Рабочие таблицы и индексы не изменялись.\n\nОграничения: это прогретый SQL-тест без HTTP, Redis, аутентификации и конкуренции; не cold-cache benchmark, не измерение RPS и не доказательство latency всего API. FK не копировались, поскольку сравнивается чтение; это одинаково во всех фазах. Измерения зависят от ресурсов и нагрузки Docker-host. Лимиты и настройки PostgreSQL записаны в JSON; текстовые планы сняты отдельным запуском, поэтому их времена могут отличаться от медиан JSON.\n\nФайлы: [полные текстовые планы](history-index-plans.txt), [JSON, пять повторов, настройки и результаты](history-index-explain.json).\n`; +await writeFile(resolve(output, 'history-index-summary.md'), summary); +console.log(`Saved EXPLAIN evidence to ${output}; experiment schema rolled back.`); diff --git a/test/failover-target.py b/test/failover-target.py new file mode 100644 index 0000000..f1133cf --- /dev/null +++ b/test/failover-target.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Check failover of a named Compose API replica without changing public tests.""" + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--target-index", type=int, choices=(1, 2), required=True) + args = parser.parse_args() + + ids = subprocess.run( + ["docker", "compose", "ps", "--all", "-q", "api"], + cwd=ROOT, capture_output=True, text=True, check=True, + ).stdout.splitlines() + check(len(ids) == 2, "targeted failover requires two API replicas") + target = subprocess.run( + ["docker", "inspect", "--format", "{{.Config.Hostname}}", + ids[args.target_index - 1]], + cwd=ROOT, capture_output=True, text=True, check=True, + ).stdout.strip() + expected = { + subprocess.run( + ["docker", "inspect", "--format", "{{.Config.Hostname}}", item], + cwd=ROOT, capture_output=True, text=True, check=True, + ).stdout.strip() + for item in ids + } + + client = Client(args.base_url, timeout=10) + alice, bob, _, chat = client.bootstrap() + before = client.message(alice, chat, "Before targeted failover")["json"] + observed = { + client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[ + "headers" + ]["X-Instance-Id"] + for _ in range(40) + } + check(observed == expected, "both replicas must answer before failure") + client.timeout = 2 + + hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + env = dict(os.environ, TARGET_INSTANCE_ID=target) + started = time.monotonic() + result = subprocess.run( + [str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=10, check=False, + ) + check(result.returncode == 0, "stop-one hook failed") + + key = uuid.uuid4().hex + text = "Targeted failover " + key + failures = 0 + while True: + try: + written = client.message(alice, chat, text, key=key) + break + except Failure: + failures += 1 + check(time.monotonic() - started <= 10, "write recovery exceeded 10 seconds") + time.sleep(0.1) + recovery = time.monotonic() - started + check(recovery <= 10, "write recovery exceeded 10 seconds") + check(written["headers"]["X-Instance-Id"] != target, + "stopped replica answered the write") + + reads = 0 + first_read = None + while reads < 10: + try: + response = client.call("GET", "/api/v1/users/me", actor=alice, + schema="User") + except Failure: + failures += 1 + check(time.monotonic() - started <= 10, + "read recovery exceeded 10 seconds") + continue + if first_read is None: + first_read = time.monotonic() - started + check(first_read <= 10, "read recovery exceeded 10 seconds") + check(response["headers"]["X-Instance-Id"] != target, + "stopped replica still serves traffic") + reads += 1 + messages = client.call( + "GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", + actor=bob, schema="MessagePage", + )["json"]["items"] + check(before in messages, "confirmed pre-failure message disappeared") + check(sum(message["text"] == text for message in messages) == 1, + "uncertain retry created a duplicate") + print(f"PASS: replica {args.target_index} stopped; write_recovery={recovery:.3f}s; " + f"read_recovery={first_read:.3f}s; transient_failures={failures}") + + +if __name__ == "__main__": + main() diff --git a/test/full-restart.py b/test/full-restart.py new file mode 100644 index 0000000..0c2d813 --- /dev/null +++ b/test/full-restart.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import argparse +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run +from resilience import history + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + parser.add_argument('--ca-file') + parser.add_argument('--action-script', required=True) + args = parser.parse_args() + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = client.bootstrap() + original = client.message(alice, chat, 'Сохранность после полного перезапуска')['json'] + check(history(client, bob, chat) == [original], 'pre-restart history mismatch') + hook = Path(args.action_script).resolve() + check(hook.is_file(), 'restart hook not found') + print('Restarting backend, PostgreSQL and Redis; hook output suppressed.', flush=True) + try: + proc = subprocess.run([str(hook)], cwd=ROOT, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=90) + except subprocess.TimeoutExpired: + raise Failure('restart hook exceeded 90 seconds') from None + check(proc.returncode == 0, 'restart hook failed') + deadline = time.monotonic() + 90 + while True: + try: + client.call('GET', '/health/ready', schema='Health') + break + except Failure: + check(time.monotonic() < deadline, 'API did not recover within 90 seconds') + time.sleep(0.5) + for actor in (alice, bob, eve): + check(client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'] + == actor['user'], 'user or existing session changed after restart') + check(client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')['json'] + == chat, 'chat or membership changed after restart') + check(history(client, bob, chat) == [original], 'confirmed message lost, changed or duplicated') + client.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve, schema='Error') + followup = client.message(bob, chat, 'Запись после восстановления')['json'] + check(history(client, alice, chat) == [original, followup], 'post-restart write/read mismatch') + client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + client.call('GET', '/api/v1/users/me', 401, actor=alice, schema='Error') + print('PASS: users, chat, membership, confirmed message and all existing sessions survived; ' + 'access control, new writes and logout work after restart.') + + +if __name__ == '__main__': + run(main) diff --git a/test/grade4.py b/test/grade4.py new file mode 100644 index 0000000..cc9283c --- /dev/null +++ b/test/grade4.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Lab 3 grade 4: measured replica failover or sequential replacement.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 +from resilience import history # noqa: E402 + +def compose(*args, timeout=60): + try: + result = subprocess.run( + ["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args], + cwd=ROOT, capture_output=True, + text=True, timeout=timeout, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None + check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs") + return result.stdout.strip() + + +def docker(*args, timeout=60): + try: + result = subprocess.run( + ["docker", *args], cwd=ROOT, capture_output=True, + text=True, timeout=timeout, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None + check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs") + return result.stdout.strip() + + +def inspect(container_id, field): + try: + result = subprocess.run( + ["docker", "inspect", "--format", field, container_id], + cwd=ROOT, capture_output=True, text=True, timeout=10, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure(f"docker inspect failed: {type(exc).__name__}") from None + check(result.returncode == 0, "docker inspect failed") + return result.stdout.strip() + + +def containers(service): + ids = compose("ps", "--all", "-q", service).splitlines() + check(bool(ids), f"{service} container is missing") + return ids + + +def container(service): + ids = containers(service) + check(len(ids) == 1, f"expected one {service} container") + return ids[0] + + +def instance(container_id): + return inspect(container_id, "{{.Config.Hostname}}") + + +def healthy(container_id, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if inspect(container_id, "{{.State.Health.Status}}") == "healthy": + return + time.sleep(0.5) + raise Failure(f"API container did not become healthy in {timeout} seconds") + + +def both_observed(client, actor, expected, timeout=30): + seen = set() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = client.call("GET", "/api/v1/users/me", actor=actor, + schema="User") + seen.add(response["headers"]["X-Instance-Id"]) + except Failure: + pass + if seen == expected: + return + time.sleep(0.1) + raise Failure(f"both replicas were not observed; saw {sorted(seen)}") + + +class Traffic: + def __init__(self, base_url, actor, chat, concurrency): + self.base_url = base_url + self.actor = actor + self.chat = chat + self.concurrency = concurrency + self.stop = threading.Event() + self.lock = threading.Lock() + self.events = [] + self.confirmed = set() + self.attempted = set() + self.pool = ThreadPoolExecutor(max_workers=concurrency) + + def start(self): + self.futures = [self.pool.submit(self.worker, i) + for i in range(self.concurrency)] + + def worker(self, worker_id): + client = Client(self.base_url, timeout=2) + index = 0 + pending = None + while not self.stop.is_set(): + write = worker_id % 2 == 0 + if write and pending is None: + key = uuid.uuid4().hex + pending = (key, f"grade4-{worker_id}-{index}-{key}") + index += 1 + with self.lock: + self.attempted.add(pending[1]) + try: + if write: + response = client.message(self.actor, self.chat, + pending[1], key=pending[0]) + else: + response = client.call("GET", "/api/v1/users/me", + actor=self.actor, schema="User") + event = (time.monotonic(), "write" if write else "read", True, + response["headers"]["X-Instance-Id"]) + if write: + with self.lock: + self.confirmed.add(pending[1]) + pending = None + except Failure: + event = (time.monotonic(), "write" if write else "read", False, None) + with self.lock: + self.events.append(event) + self.stop.wait(0.05) + + def finish(self): + self.stop.set() + try: + for future in self.futures: + future.result(timeout=5) + finally: + self.pool.shutdown(wait=True) + + def snapshot(self): + with self.lock: + return list(self.events) + + +def metrics(events, started, ended, target, action_done, scenario): + window = [event for event in events if started <= event[0] <= ended] + check(bool(window), "no traffic was recorded during the action") + counts = {} + for kind in ("read", "write"): + records = [event for event in window if event[1] == kind] + successes = [event for event in records if event[2]] + check(bool(successes), f"no successful {kind}s during the action") + survivor = next((event[0] for event in successes + if event[0] >= started and event[3] != target), None) + check(survivor is not None, f"no {kind} served by surviving replica") + after_action = next((event[0] for event in successes + if event[0] >= action_done), None) + check(after_action is not None, f"no {kind} succeeded after the action") + counts[kind] = { + "successful": len(successes), + "errors": len(records) - len(successes), + "successful_rps": round(len(successes) / (ended - started), 3), + "first_survivor_seconds": round(survivor - started, 3), + "first_success_after_action_seconds": round(after_action - started, 3), + } + after = [event for event in window if event[0] >= action_done + 1 and event[2]] + check(bool(after), "no successful traffic after the action") + if scenario == "failover": + check(all(event[3] != target for event in after), + "stopped replica still served traffic after the action") + target_times = [event[0] for event in window + if event[2] and event[3] == target] + counts["last_target_response_seconds"] = ( + round(max(target_times) - started, 3) if target_times else None + ) + for kind in ("read", "write"): + check(counts[kind]["first_success_after_action_seconds"] <= 10, + f"{kind} did not recover within 10 seconds") + return counts + + +def verify_history(client, actor, chat, original, traffic): + messages = history(client, actor, chat) + counts = Counter(message["text"] for message in messages) + check(any(message == original for message in messages), + "confirmed pre-action message disappeared") + for text in traffic.attempted: + check(counts[text] <= 1, "an uncertain retry created a duplicate") + for text in traffic.confirmed: + check(counts[text] == 1, "confirmed write disappeared") + + +def resource_limits(): + expected = { + "nginx": (500_000_000, 128 * 1024**2), + "migrate": (1_000_000_000, 512 * 1024**2), + "postgres": (1_000_000_000, 512 * 1024**2), + "redis": (500_000_000, 256 * 1024**2), + } + ids = containers("api") + check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas") + api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids)) + observed = {"api": []} + for container_id in ids: + actual = ( + int(inspect(container_id, "{{.HostConfig.NanoCpus}}")), + int(inspect(container_id, "{{.HostConfig.Memory}}")), + ) + check(actual == api_limits, "API resource limits differ from fixed budget") + observed["api"].append({"instance": instance(container_id), + "cpus": actual[0] / 1e9, + "memory_mib": actual[1] // 1024**2}) + for service, limits in expected.items(): + container_id = container(service) + actual = ( + int(inspect(container_id, "{{.HostConfig.NanoCpus}}")), + int(inspect(container_id, "{{.HostConfig.Memory}}")), + ) + check(actual == limits, f"{service} resource limits differ from compose.yaml") + observed[service] = {"cpus": actual[0] / 1e9, + "memory_mib": actual[1] // 1024**2} + return observed + + +def run(args): + check(2 <= args.concurrency <= 8, "concurrency must be 2..8") + check(3 <= args.observe_seconds <= 60, + "observe-seconds must be 3..60") + result = {"scenario": args.scenario, "resources": resource_limits(), + "actions": []} + if args.scenario == "resources": + return result + targets = containers("api") + check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile") + + client = Client(args.base_url, timeout=10) + alice, bob, _, chat = client.bootstrap() + original = client.message(alice, chat, "Before grade 4 actions")["json"] + both_observed(client, alice, {instance(item) for item in targets}) + migrate_id = container("migrate") + traffic = Traffic(args.base_url, alice, chat, args.concurrency) + traffic.start() + try: + for before_id in targets: + target = instance(before_id) + healthy(before_id) + before_ids = set(containers("api")) + both_observed(client, alice, {instance(item) for item in before_ids}) + time.sleep(1) + started = time.monotonic() + if args.scenario == "failover": + docker("stop", "--time", "30", before_id, timeout=40) + check(inspect(before_id, "{{.State.Status}}") == "exited", + "target API container was not stopped") + else: + docker("stop", "--time", "30", before_id, timeout=40) + docker("rm", before_id) + compose("up", "-d", "--no-build", "--no-deps", + "--scale", "api=2", "api", timeout=90) + replacements = set(containers("api")) - before_ids + check(len(replacements) == 1, "one API container was not replaced") + healthy(next(iter(replacements))) + action_done = time.monotonic() + time.sleep(args.observe_seconds) + ended = time.monotonic() + item = {"target": target, + "action_seconds": round(action_done - started, 3), + "window_seconds": round(ended - started, 3), + "traffic": metrics(traffic.snapshot(), started, ended, + target, action_done, args.scenario)} + if args.scenario == "failover": + docker("start", before_id) + healthy(before_id) + both_observed(client, alice, + {instance(item) for item in containers("api")}) + result["actions"].append(item) + check(container("migrate") == migrate_id, + "migration container changed during API replacement") + finally: + traffic.finish() + # Restore both API replicas after a failed test; never remove volumes. + try: + compose("up", "-d", "--no-build", "--no-deps", + "--scale", "api=2", "api") + except Failure: + print("WARNING: restore API replicas manually", file=sys.stderr) + verify_history(client, bob, chat, original, traffic) + result["confirmed_writes"] = len(traffic.confirmed) + result["attempted_unique_writes"] = len(traffic.attempted) + result["migration_container_unchanged"] = True + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scenario", choices=("resources", "failover", "rolling"), + required=True) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--observe-seconds", type=float, default=10) + parser.add_argument("--output", help="optional JSON result path without tokens") + args = parser.parse_args() + try: + result = run(args) + output = json.dumps(result, ensure_ascii=False, indent=2) + if args.output: + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(output + "\n", encoding="utf-8") + print(output) + print("PASS: grade 4 scenario") + except (Failure, ValueError, OSError) as exc: + print(f"FAIL: {exc}", file=sys.stderr) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/test/helpers/fault-proxy.mjs b/test/helpers/fault-proxy.mjs new file mode 100644 index 0000000..223dbf4 --- /dev/null +++ b/test/helpers/fault-proxy.mjs @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import net from 'node:net'; + +export async function faultProxy(connectionUrl, defaultPort) { + const upstreamUrl = new URL(connectionUrl); + const host = upstreamUrl.hostname.replace(/^\[|\]$/g, ''); + const port = Number(upstreamUrl.port || defaultPort); + let mode = 'online'; + const sockets = new Set(); + const track = (socket) => { + sockets.add(socket); + socket.on('error', () => {}); + socket.on('close', () => sockets.delete(socket)); + return socket; + }; + const server = net.createServer((client) => { + track(client); + if (mode === 'cut') { + client.destroy(); + return; + } + const upstream = track(net.connect({ host, port })); + client.on('data', (chunk) => { + if (mode === 'online' && !upstream.destroyed) upstream.write(chunk); + }); + upstream.on('data', (chunk) => { + if (mode === 'online' && !client.destroyed) client.write(chunk); + }); + client.on('close', () => upstream.destroy()); + upstream.on('close', () => client.destroy()); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const relayUrl = new URL(connectionUrl); + relayUrl.hostname = '127.0.0.1'; + relayUrl.port = String(server.address().port); + return { + url: relayUrl.toString(), + setMode(nextMode) { + assert.ok(['online', 'cut', 'hang'].includes(nextMode)); + mode = nextMode; + if (mode === 'cut') { + for (const socket of sockets) socket.destroy(); + } + }, + async close() { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + }, + }; +} diff --git a/test/helpers/message-epoch.mjs b/test/helpers/message-epoch.mjs new file mode 100644 index 0000000..1e2f49a --- /dev/null +++ b/test/helpers/message-epoch.mjs @@ -0,0 +1,7 @@ +export async function messageEpoch(prisma, chatId) { + const [row] = await prisma.$queryRawUnsafe( + 'SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM public.chat_message_epochs WHERE chat_id = $1::uuid', + chatId, + ); + return row.epoch; +} diff --git a/test/idempotency.mjs b/test/idempotency.mjs new file mode 100644 index 0000000..88b8992 --- /dev/null +++ b/test/idempotency.mjs @@ -0,0 +1,146 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +test( + 'message idempotency is shared by two API instances', + { timeout: 120000 }, + async (t) => { + assert.ok( + process.env.DATABASE_URL, + 'Set DATABASE_URL and apply migrations', + ); + const apps = []; + let db; + let chatId; + const userIds = []; + t.after(async () => { + if (db && chatId) { + await db.$executeRawUnsafe( + 'DELETE FROM message_idempotency WHERE chat_id = $1::uuid', + chatId, + ); + await db.chat.deleteMany({ where: { id: chatId } }); + } + if (db && userIds.length) { + await db.user.deleteMany({ where: { id: { in: userIds } } }); + } + for (const app of apps) await app.close(); + }); + + for (let index = 0; index < 2; index++) { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + }); + exposeRequestIdHeader(app); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + apps.push(app); + } + db = apps[0].get(PrismaService); + const servers = apps.map((app) => app.getHttpAdapter().getInstance()); + const call = (index, method, url, token, payload, key) => + servers[index].inject({ + method, + url, + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(key === undefined ? {} : { 'idempotency-key': key }), + }, + ...(payload === undefined ? {} : { payload }), + }); + const register = async (label) => { + const username = `idem_${label}_${randomUUID().replaceAll('-', '').slice(0, 12)}`; + const password = `T3st-${randomUUID()}`; + const response = await call(0, 'POST', '/api/v1/users', undefined, { + username, + display_name: label, + password, + }); + assert.equal(response.statusCode, 201, response.body); + const user = response.json(); + userIds.push(user.id); + const login = await servers[0].inject({ + method: 'POST', + url: '/api/v1/auth/sessions', + headers: { + authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`, + }, + }); + assert.equal(login.statusCode, 201, login.body); + return { user, token: login.json().token }; + }; + + const owner = await register('owner'); + const outsider = await register('outsider'); + const created = await call(0, 'POST', '/api/v1/chats', owner.token, { + title: 'Idempotency test', + member_ids: [owner.user.id], + }); + assert.equal(created.statusCode, 201, created.body); + chatId = created.json().id; + const path = `/api/v1/chats/${chatId}/messages`; + const key = randomUUID(); + const body = { text: 'One message' }; + + assert.equal( + (await call(0, 'POST', path, owner.token, body)).statusCode, + 400, + ); + assert.equal( + (await call(0, 'POST', path, owner.token, body, 'bad key')).statusCode, + 400, + ); + const first = await call(0, 'POST', path, owner.token, body, key); + assert.equal(first.statusCode, 201, first.body); + const replay = await call(1, 'POST', path, owner.token, body, key); + assert.equal(replay.statusCode, 201, replay.body); + assert.deepEqual(replay.json(), first.json()); + assert.equal( + (await call(1, 'POST', path, owner.token, { text: 'Changed' }, key)) + .statusCode, + 409, + ); + assert.equal( + (await call(1, 'POST', path, outsider.token, body, key)).statusCode, + 404, + ); + + const concurrentKey = randomUUID(); + const concurrent = await Promise.all( + Array.from({ length: 12 }, (_, index) => + call( + index % 2, + 'POST', + path, + owner.token, + { text: 'Concurrent' }, + concurrentKey, + ), + ), + ); + for (const response of concurrent) { + assert.equal(response.statusCode, 201, response.body); + assert.deepEqual(response.json(), concurrent[0].json()); + } + assert.equal(await db.message.count({ where: { chatId } }), 2); + const [record] = await db.$queryRawUnsafe( + 'SELECT expires_at AS "expiresAt" FROM message_idempotency WHERE user_id = $1::uuid AND chat_id = $2::uuid AND "key" = $3', + owner.user.id, + chatId, + concurrentKey, + ); + assert.ok(record.expiresAt.getTime() - Date.now() >= 23 * 3600 * 1000); + }, +); diff --git a/test/message-pagination.mjs b/test/message-pagination.mjs new file mode 100644 index 0000000..bc92b1e --- /dev/null +++ b/test/message-pagination.mjs @@ -0,0 +1,137 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +test('message keyset pagination: timestamp ties, chat isolation and deleted cursor row', async (t) => { + assert.ok(process.env.DATABASE_URL); + const previousCacheUrl = process.env.CACHE_URL; + process.env.CACHE_URL = ''; + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + const prisma = app.get(PrismaService); + let userId; + const chatIds = []; + t.after(async () => { + try { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + if (userId) await prisma.user.delete({ where: { id: userId } }); + } finally { + await app.close(); + if (previousCacheUrl === undefined) delete process.env.CACHE_URL; + else process.env.CACHE_URL = previousCacheUrl; + } + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + const user = await prisma.user.create({ + data: { + username: `page_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Synthetic pagination test', + }, + }); + userId = user.id; + const token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + for (let i = 0; i < 2; i++) { + const chat = await prisma.chat.create({ + data: { + title: `Pagination ${i}`, + members: { create: { userId, position: 0 } }, + }, + }); + chatIds.push(chat.id); + } + const [chatId, otherChatId] = chatIds; + const rows = Array.from({ length: 140 }, (_, i) => ({ + id: randomUUID(), + chatId, + senderId: userId, + text: `Message ${i}`, + createdAt: new Date(Date.UTC(2026, 0, 1) + Math.floor(i / 10) * 1000), + })); + await prisma.message.createMany({ + data: [ + ...rows, + ...rows.map((row) => ({ + ...row, + id: randomUUID(), + chatId: otherChatId, + text: 'Other chat', + })), + ], + }); + const expected = [...rows] + .sort( + (a, b) => + a.createdAt - b.createdAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ) + .map((row) => ({ + id: row.id, + chat_id: chatId, + sender_id: userId, + text: row.text, + created_at: row.createdAt.toISOString(), + })); + const request = (cursor, selectedChat = chatId) => + server.inject({ + method: 'GET', + url: `/api/v1/chats/${selectedChat}/messages?limit=7${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, + headers: { authorization: `Bearer ${token}` }, + }); + const seen = []; + let cursor; + let firstCursor; + do { + const response = await request(cursor); + assert.equal(response.statusCode, 200); + const page = response.json(); + assert.ok(page.items.length <= 7); + seen.push(...page.items); + if (!firstCursor) firstCursor = page.next_cursor; + cursor = page.next_cursor; + assert.ok( + seen.length <= rows.length, + 'pagination must terminate without duplicates', + ); + } while (cursor); + assert.deepEqual(seen, expected); + const anchor = JSON.parse( + Buffer.from(firstCursor, 'base64url').toString('utf8'), + ); + await prisma.message.delete({ where: { id: anchor.id } }); + const afterDeletion = await request(firstCursor); + assert.equal(afterDeletion.statusCode, 200); + assert.deepEqual(afterDeletion.json().items, expected.slice(7, 14)); + assert.equal((await request(firstCursor, otherChatId)).statusCode, 400); + const pastEnd = Buffer.from( + JSON.stringify({ + scope: `messages:${chatId}`, + id: randomUUID(), + created_at: '2027-01-01T00:00:00.000Z', + }), + ).toString('base64url'); + const empty = await request(pastEnd); + assert.equal(empty.statusCode, 200); + assert.deepEqual(empty.json(), { items: [], next_cursor: null }); +}); diff --git a/test/nginx-readiness-parallel.sh b/test/nginx-readiness-parallel.sh new file mode 100755 index 0000000..8c3924d --- /dev/null +++ b/test/nginx-readiness-parallel.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# A barrier makes sequential probes fail: all four requests must start together. +set -eu +sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh +. /tmp/readiness-under-test.sh +mkdir -p /tmp/readiness-barrier +cycle=0 +getent() { + for ip in 1 2 3 4; do printf '192.0.2.%s STREAM api\n' "$ip"; done +} +wget() { + for arg do + case "$arg" in http://*) address=${arg#http://}; address=${address%%:*} ;; esac + done + touch "/tmp/readiness-barrier/$address" + attempt=0 + while [ "$attempt" -lt 50 ]; do + if [ "$(find /tmp/readiness-barrier -type f | wc -l)" -eq 4 ]; then return 0; fi + attempt=$((attempt + 1)) + command sleep 0.01 + done + return 1 +} +nginx() { return 0; } +sleep() { + if [ "$cycle" -eq 2 ]; then + count=$(grep -c 'server 192.0.2.' "$upstreams" || true) + [ "$count" -eq 4 ] || { printf 'FAIL: readiness probes did not start in parallel\n'; exit 1; } + printf 'PASS: four concurrent probes admitted after two successful rounds\n' + exit 0 + fi + rm -f /tmp/readiness-barrier/* + cycle=$((cycle + 1)) +} +check_ready diff --git a/test/nginx-readiness.sh b/test/nginx-readiness.sh new file mode 100755 index 0000000..5c334da --- /dev/null +++ b/test/nginx-readiness.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Run in a disposable nginx container; never against a running load balancer. +set -eu + +sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh +. /tmp/readiness-under-test.sh + +cycle=0 +reloads=0 +getent() { printf '192.0.2.1 STREAM api\n'; } +wget() { + case "$cycle" in + 1|2|4|7|8) return 0 ;; + *) return 1 ;; + esac +} +nginx() { + if [ "$1" = '-s' ]; then reloads=$((reloads + 1)); fi +} +sleep() { + # Check the previous cycle before advancing to the next probe. + if [ "$cycle" -gt 0 ]; then + case "$cycle" in + 2|3|4|5|8) expected=1 ;; + *) expected=0 ;; + esac + actual=0 + if grep -q 'server 192.0.2.1:8080 ' "$upstreams"; then actual=1; fi + if [ "$actual" -ne "$expected" ]; then + printf 'FAIL: cycle %s, expected readiness %s, got %s\n' "$cycle" "$expected" "$actual" >&2 + exit 1 + fi + fi + cycle=$((cycle + 1)) + if [ "$cycle" -eq 9 ]; then + [ "$reloads" -eq 3 ] || exit 1 + printf 'PASS: admission, isolated failure, recovery, removal and re-admission; 3 reloads\n' + exit 0 + fi +} + +check_ready diff --git a/test/no-epochs-experiment.cjs b/test/no-epochs-experiment.cjs new file mode 100644 index 0000000..8867ff1 --- /dev/null +++ b/test/no-epochs-experiment.cjs @@ -0,0 +1,268 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { spawn, execFileSync } = require('node:child_process'); +const ts = require('typescript'); + +const root = path.resolve(__dirname, '..'); +const resultPrefix = process.env.EXPERIMENT_PREFIX ?? 'cache-strategy'; +assert.match(resultPrefix, /^[a-z0-9-]+$/); +const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-no-epochs-')); +const hashes = {}; +const volumes = []; +const replace = (source, from, to) => { + assert.ok(source.includes(from), `Missing replacement: ${from}`); + assert.equal( + source.indexOf(from), + source.lastIndexOf(from), + `Ambiguous replacement: ${from}`, + ); + return source.replace(from, to); +}; +const section = (source, from, to, replacement) => { + const start = source.indexOf(from); + const end = source.indexOf(to, start); + assert.ok(start !== -1 && end > start, `Missing section: ${from}`); + return source.slice(0, start) + replacement + source.slice(end); +}; +const digest = (value) => createHash('sha256').update(value).digest('hex'); + +const transforms = { + 'Chat/chat.service.ts': (source) => { + source = section( + source, + ' await tx.$executeRaw(', + ' return created;', + '', + ); + source = replace(source, ' epoch: string,\n', ''); + source = replace(source, ' epoch,\n', ''); + source = section( + source, + ' const [membership] =', + ' if (!membership)', + ' const [membership] = await this.prisma.$queryRaw<{ id: string }[]>(Prisma.sql`\n' + + ' SELECT cm.chat_id AS id FROM chat_members AS cm\n' + + ' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' + + ' `);\n', + ); + source = section( + source, + ' const [message] =', + ' if (!message)', + ' const [message] = await this.prisma.$queryRaw(Prisma.sql`\n' + + ' INSERT INTO messages (id, chat_id, sender_id, text)\n' + + ' SELECT ${randomUUID()}::uuid, cm.chat_id, ${userId}::uuid, ${text}\n' + + ' FROM chat_members AS cm\n' + + ' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' + + ' RETURNING id, chat_id AS "chatId", sender_id AS "senderId",\n' + + ' text, created_at AS "createdAt"\n' + + ' `);\n', + ); + return replace(source, ' chat.messagesEpoch.toString(),\n', ''); + }, + 'Auth/auth.service.ts': (source) => { + source = source.replace( + /COALESCE\(\(SELECT SUM\(e\.epoch\)[\s\S]*?0\)::bigint AS "messagesEpoch"/g, + '0::bigint AS "messagesEpoch"', + ); + return source + .replaceAll('u.chats_epoch', '0::bigint') + .replaceAll('c.messages_epoch', '0::bigint'); + }, + 'User/user.guard.ts': (source) => { + source = replace(source, ' currentChatsEpoch: string;\n', ''); + return replace( + source, + ' request.currentChatsEpoch = session.chatsEpoch;\n', + '', + ); + }, + 'Chat/chat.controller.ts': (source) => + replace(source, ' request.currentChatsEpoch,\n', ''), + 'cache/cache.service.ts': (source) => { + source = replace(source, ' epoch?: string,\n', ''); + return section( + source, + ' if (epoch !== undefined)', + ' const versionKey =', + '', + ); + }, + 'User/user.service.ts': (source) => { + source = replace( + source, + 'const user = await this.prisma.user.findUnique({ where: { id } });', + 'const user = await this.prisma.user.findUnique({ where: { id }, select: { id: true, username: true, displayName: true, createdAt: true } });', + ); + return source; + }, +}; + +try { + for (const [relative, transform] of Object.entries(transforms)) { + const original = fs.readFileSync(path.join(root, 'src', relative), 'utf8'); + const source = transform(original.replace(/\r\n/g, '\n')); + assert.ok( + !/messages_epoch|chats_epoch|chat_message_epochs/.test(source), + relative, + ); + const compiled = ts.transpileModule(source, { + compilerOptions: { + target: ts.ScriptTarget.ES2023, + module: ts.ModuleKind.CommonJS, + experimentalDecorators: true, + emitDecoratorMetadata: true, + }, + reportDiagnostics: true, + }); + assert.equal( + compiled.diagnostics.filter( + (item) => item.category === ts.DiagnosticCategory.Error, + ).length, + 0, + ); + const sourcePath = path.join(directory, relative); + const compiledPath = sourcePath.replace(/\.ts$/, '.js'); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync(sourcePath, source); + fs.writeFileSync(compiledPath, compiled.outputText); + hashes[relative] = { + original_source: digest(original), + experiment_source: digest(source), + experiment_js: digest(compiled.outputText), + }; + volumes.push({ + type: 'bind', + source: compiledPath, + target: `/app/dist/${relative.replace(/\.ts$/, '.js')}`, + read_only: true, + }); + } + let benchmark = fs + .readFileSync(path.join(root, 'test/cache-benchmark.mjs'), 'utf8') + .replace(/\r\n/g, '\n'); + benchmark = replace( + benchmark, + 'process.env.CACHE_URL = proxy.url;', + 'delete process.env.CACHE_URL;', + ); + benchmark = section( + benchmark, + ' await prisma.chatMessageEpoch.createMany(', + ' const path =', + '', + ); + benchmark = section( + benchmark, + ' const second = await start();', + ' report.completed = true;', + ' assert.equal(first.app.get(CacheService).redis, undefined);\n' + + ' for (const chatId of chatIds) {\n' + + ' assert.equal(await messageEpoch(prisma, chatId), 0n);\n' + + ' }\n' + + ' assert.equal((await prisma.user.findUniqueOrThrow({ where: { id: userId } })).chatsEpoch, 0n);\n' + + ' for (const chatId of chatIds.slice(1)) {\n' + + ' const page = await request("GET", `/api/v1/chats/${chatId}/messages`);\n' + + ' assert.equal(page.items.length, 50);\n' + + ' assert.equal(new Set(page.items.map(message => message.id)).size, 50);\n' + + ' }\n' + + ' report.correctness = { no_redis_client: true, epochs_unchanged: true, read_after_write: true };\n' + + ' }\n', + ); + const benchmarkPath = path.join(directory, 'cache-benchmark.mjs'); + fs.writeFileSync(benchmarkPath, benchmark); + volumes.push({ + type: 'bind', + source: benchmarkPath, + target: '/experiment/cache-benchmark.mjs', + read_only: true, + }); + volumes.push({ + type: 'bind', + source: path.join(root, 'test/helpers'), + target: '/experiment/helpers', + read_only: true, + }); + const override = path.join(directory, 'compose.no-epochs.yaml'); + fs.writeFileSync( + override, + JSON.stringify({ services: { benchmark: { volumes } } }, null, 2), + ); + const args = [ + 'compose', + '-f', + 'compose.yaml', + '-f', + 'test/compose.benchmark.yaml', + '-f', + override, + 'run', + '--rm', + '--no-deps', + '--pull', + 'never', + 'benchmark', + 'node', + '/experiment/cache-benchmark.mjs', + 'after', + `/results/${resultPrefix}-no-redis-no-epochs.json`, + ...process.argv.slice(2), + ]; + const metadata = { + recorded_at: new Date().toISOString(), + image: execFileSync( + 'docker', + ['image', 'inspect', 'mtusi2026lab2-fork-backend', '--format', '{{.Id}}'], + { encoding: 'utf8' }, + ).trim(), + modified_sources: hashes, + command: ['docker', ...args], + implementation: + 'Separate temporary source copies compiled with TypeScript and mounted over six application modules. Workspace sources and running backend remain unchanged.', + schema: + 'Epoch storage remains in the database; experiment API paths do not update counters or read authoritative epoch values.', + redis: + 'Application starts without CACHE_URL. Observer/proxy are retained only for benchmark setup and cleanup.', + full_result: `${resultPrefix}-full.json`, + experiment_result: `${resultPrefix}-no-redis-no-epochs.json`, + }; + const child = spawn('docker', args, { + cwd: root, + stdio: 'inherit', + env: process.env, + }); + child.once('error', finish); + child.once('exit', (code, signal) => finish(null, code, signal)); + let finished = false; + function finish(error, code, signal) { + if (finished) return; + finished = true; + metadata.exit_code = error ? 1 : code; + metadata.signal = signal; + metadata.sources_unchanged = Object.entries(hashes).every( + ([relative, hash]) => + digest(fs.readFileSync(path.join(root, 'src', relative), 'utf8')) === + hash.original_source, + ); + fs.writeFileSync( + path.join(root, `evidence/${resultPrefix}-experiment.json`), + JSON.stringify(metadata, null, 2) + '\n', + ); + cleanup(); + if (error) console.error(error.message); + process.exitCode = error ? 1 : (code ?? 1); + } +} catch (error) { + cleanup(); + throw error; +} + +function cleanup() { + const resolved = path.resolve(directory); + assert.equal(path.dirname(resolved), path.resolve(os.tmpdir())); + assert.ok(path.basename(resolved).startsWith('lab2-no-epochs-')); + fs.rmSync(resolved, { recursive: true, force: true }); +} diff --git a/test/passwords.mjs b/test/passwords.mjs new file mode 100644 index 0000000..17c4eef --- /dev/null +++ b/test/passwords.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { argon2 } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { PasswordService } = require('../dist/Auth/password.service'); +const password = '🔐'.repeat(40) + ':Синтетический пароль\0'; + +test('Argon2id only: PHC format, independent derivation, unique salts, Unicode and invalid hashes', async () => { + const service = new PasswordService(); + const first = await service.hash(password); + const second = await service.hash(password); + assert.notEqual( + first, + second, + 'same password must have independently salted hashes', + ); + assert.match( + first, + /^\$argon2id\$v=19\$m=19456,t=2,p=1\$[A-Za-z0-9+/]{22}\$[A-Za-z0-9+/]{43}$/, + ); + assert.ok(!first.includes(password)); + const parts = first.split('$'); + const independent = await new Promise((resolve, reject) => + argon2( + 'argon2id', + { + message: password, + nonce: Buffer.from(parts[4], 'base64'), + memory: 19456, + passes: 2, + parallelism: 1, + tagLength: 32, + }, + (error, key) => (error ? reject(error) : resolve(key)), + ), + ); + assert.deepEqual(Buffer.from(parts[5], 'base64'), independent); + assert.equal(await service.verify(password, first), true); + assert.equal( + await service.verify(password.replace('🔐', '🔑'), first), + false, + ); + assert.equal( + await service.verify(password.slice(0, -1), first), + false, + 'NUL must not be silently truncated', + ); + for (const invalid of [ + null, + '', + 'scrypt$131072$8$1$unsupported$unsupported', + '$argon2id$v=19$m=999999999,t=999999999,p=1$x$x', + first.replace('v=19', 'v=16'), + first.replace('argon2id', 'argon2i'), + first + '$extra', + first.replace(parts[5], 'a'), + first.replace(parts[4], parts[4] + '='), + ]) { + assert.equal(await service.verify(password, invalid), false); + } +}); diff --git a/test/persistence.mjs b/test/persistence.mjs new file mode 100644 index 0000000..63cb47e --- /dev/null +++ b/test/persistence.mjs @@ -0,0 +1,349 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +async function start() { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + try { + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + return { app, server, prisma: app.get(PrismaService) }; + } catch (error) { + await app.close(); + throw error; + } +} + +test('PostgreSQL storage: constraints, pagination, access and application restart', async (t) => { + assert.ok( + process.env.DATABASE_URL, + 'Set DATABASE_URL and apply migrations before running this test', + ); + let context = await start(); + const userIds = []; + const chatIds = []; + t.after(async () => { + try { + await context.prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + await context.prisma.user.deleteMany({ where: { id: { in: userIds } } }); + } finally { + await context.app.close(); + } + }); + const suffix = randomUUID().replaceAll('-', '').slice(0, 16); + const password = '🔐'.repeat(40) + ':пароль с пробелами '; + const tokens = new Map(); + const request = (method, url, payload, userId) => + context.server.inject({ + method, + url, + ...(payload === undefined ? {} : { payload }), + headers: { + ...(userId ? { authorization: `Bearer ${tokens.get(userId)}` } : {}), + ...(method === 'POST' && /\/messages$/.test(url) + ? { 'idempotency-key': randomUUID() } + : {}), + }, + }); + + const registrations = await Promise.all( + Array.from({ length: 4 }, () => + request('POST', '/api/v1/users', { + username: `race_${suffix}`, + display_name: 'Race', + password, + }), + ), + ); + for (const response of registrations) { + if (response.statusCode === 201) userIds.push(response.json().id); + } + assert.deepEqual( + registrations.map((response) => response.statusCode).sort(), + [201, 409, 409, 409], + ); + const userId = userIds[0]; + const outsider = await request('POST', '/api/v1/users', { + username: `other_${suffix}`, + display_name: 'Other', + password, + }); + assert.equal(outsider.statusCode, 201); + userIds.push(outsider.json().id); + + const login = (username, secret = password, payload) => + context.server.inject({ + method: 'POST', + url: '/api/v1/auth/sessions', + headers: { + authorization: `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`, + }, + ...(payload === undefined ? {} : { payload }), + }); + for (const user of [ + registrations.find((r) => r.statusCode === 201).json(), + outsider.json(), + ]) { + assert.deepEqual(Object.keys(user).sort(), [ + 'created_at', + 'display_name', + 'id', + 'username', + ]); + const response = await login(user.username); + assert.equal(response.statusCode, 201); + assert.equal(response.json().token_type, 'Bearer'); + assert.match(response.json().token, /^[A-Za-z0-9_-]{43}$/); + assert.deepEqual(response.json().user, user); + tokens.set(user.id, response.json().token); + const stored = await context.prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + }); + assert.match(stored.passwordHash, /^\$argon2id\$v=19\$m=19456,t=2,p=1\$/); + assert.ok(!stored.passwordHash.includes(password)); + } + const sessions = await context.prisma.session.findMany({ where: { userId } }); + assert.equal(sessions.length, 1); + assert.match(sessions[0].tokenHash, /^[a-f0-9]{64}$/); + assert.notEqual(sessions[0].tokenHash, tokens.get(userId)); + const wrong = await login(`race_${suffix}`, 'wrong-password'); + const unknown = await login(`missing_${suffix}`, 'wrong-password'); + assert.equal(wrong.statusCode, 401); + assert.equal(unknown.statusCode, 401); + assert.equal(wrong.json().error.message, unknown.json().error.message); + assert.equal( + wrong.headers['www-authenticate'], + 'Basic realm="messenger", charset="UTF-8"', + ); + assert.equal((await login(`race_${suffix}`, password, {})).statusCode, 400); + for (const invalid of [ + undefined, + 'short', + ' '.repeat(12), + '🔐'.repeat(129), + ]) { + assert.equal( + ( + await request('POST', '/api/v1/users', { + username: `bad_${suffix}`, + display_name: 'Bad', + ...(invalid === undefined ? {} : { password: invalid }), + }) + ).statusCode, + 400, + ); + } + for (const authorization of [ + undefined, + 'Bearer unknown', + `Basic ${Buffer.from(`race_${suffix}:${password}`).toString('base64')}`, + ]) { + const response = await context.server.inject({ + method: 'GET', + url: '/api/v1/users/me', + headers: { + 'x-user-id': userId, + ...(authorization ? { authorization } : {}), + }, + }); + assert.equal(response.statusCode, 401); + assert.equal( + response.headers['www-authenticate'], + 'Bearer realm="messenger"', + ); + } + const identity = await context.server.inject({ + method: 'GET', + url: '/api/v1/users/me', + headers: { + authorization: `Bearer ${tokens.get(userId)}`, + 'x-user-id': outsider.json().id, + }, + }); + assert.equal(identity.json().id, userId); + + const ready = await request('GET', '/health/ready'); + assert.equal(ready.statusCode, 200); + + const invalidChat = await request( + 'POST', + '/api/v1/chats', + { + title: `invalid_${suffix}`, + member_ids: [userId, randomUUID()], + }, + userId, + ); + assert.equal(invalidChat.statusCode, 400); + assert.equal( + await context.prisma.chat.count({ where: { title: `invalid_${suffix}` } }), + 0, + ); + + const rolledBackId = randomUUID(); + chatIds.push(rolledBackId); + await assert.rejects( + context.prisma.$transaction(async (tx) => { + await tx.chat.create({ data: { id: rolledBackId, title: 'Rollback' } }); + await tx.chatMember.create({ + data: { chatId: rolledBackId, userId: randomUUID(), position: 0 }, + }); + }), + { code: 'P2003' }, + ); + assert.equal( + await context.prisma.chat.count({ where: { id: rolledBackId } }), + 0, + ); + + const createdChat = await request( + 'POST', + '/api/v1/chats', + { title: 'Persisted chat', member_ids: [userId] }, + userId, + ); + assert.equal(createdChat.statusCode, 201); + const chatId = createdChat.json().id; + chatIds.push(chatId); + const forbidden = await request( + 'GET', + `/api/v1/chats/${chatId}`, + undefined, + outsider.json().id, + ); + assert.equal(forbidden.statusCode, 404); + const unauthorized = await request('GET', `/api/v1/chats/${chatId}`); + assert.equal(unauthorized.statusCode, 401); + + const messageIds = []; + for (let i = 0; i < 5; i++) { + const message = await request( + 'POST', + `/api/v1/chats/${chatId}/messages`, + { text: `Message ${i}` }, + userId, + ); + assert.equal(message.statusCode, 201); + messageIds.push(message.json().id); + } + await context.prisma.message.updateMany({ + where: { chatId }, + data: { createdAt: new Date('2026-01-01T00:00:00.000Z') }, + }); + const first = await request( + 'GET', + `/api/v1/chats/${chatId}/messages?limit=2`, + undefined, + userId, + ); + assert.equal(first.statusCode, 200); + assert.equal(first.json().items.length, 2); + assert.ok(first.json().next_cursor); + + await context.app.close(); + context = await start(); + const me = await request('GET', '/api/v1/users/me', undefined, userId); + assert.equal(me.statusCode, 200); + const chat = await request( + 'GET', + `/api/v1/chats/${chatId}`, + undefined, + userId, + ); + assert.equal(chat.statusCode, 200); + assert.deepEqual(chat.json().member_ids, [userId]); + const listed = await request( + 'GET', + '/api/v1/chats?limit=1', + undefined, + userId, + ); + assert.deepEqual( + listed.json().items.map((item) => item.id), + [chatId], + ); + + const seen = first.json().items.map((message) => message.id); + let cursor = first.json().next_cursor; + while (cursor) { + const page = await request( + 'GET', + `/api/v1/chats/${chatId}/messages?limit=2&cursor=${cursor}`, + undefined, + userId, + ); + assert.equal(page.statusCode, 200); + assert.ok(page.json().items.length <= 2); + seen.push(...page.json().items.map((message) => message.id)); + cursor = page.json().next_cursor; + } + assert.deepEqual(seen, [...messageIds].sort()); + assert.equal(new Set(seen).size, 5); + + const logouts = await Promise.all( + Array.from({ length: 2 }, () => + request('DELETE', '/api/v1/auth/sessions/current', undefined, userId), + ), + ); + assert.deepEqual(logouts.map((r) => r.statusCode).sort(), [204, 401]); + assert.equal(logouts.find((r) => r.statusCode === 204).body, ''); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 401, + ); + assert.equal( + ( + await request( + 'DELETE', + '/api/v1/auth/sessions/current', + undefined, + userId, + ) + ).statusCode, + 401, + ); + await context.app.close(); + const originalTtl = process.env.SESSION_TTL_SECONDS; + process.env.SESSION_TTL_SECONDS = '2'; + try { + context = await start(); + } finally { + if (originalTtl === undefined) delete process.env.SESSION_TTL_SECONDS; + else process.env.SESSION_TTL_SECONDS = originalTtl; + } + const short = await login(`race_${suffix}`); + assert.equal(short.statusCode, 201); + tokens.set(userId, short.json().token); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 200, + ); + await context.app.close(); + context = await start(); + await new Promise((resolve) => + setTimeout( + resolve, + Math.max(0, Date.parse(short.json().expires_at) - Date.now()) + 100, + ), + ); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 401, + ); +}); diff --git a/test/pool-diagnostics.mjs b/test/pool-diagnostics.mjs new file mode 100644 index 0000000..6365e4d --- /dev/null +++ b/test/pool-diagnostics.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; +import test from 'node:test'; + +test('pool observer preserves promise/callback results, errors and release', async () => { + let timer; + let now = 0; + const records = []; + const released = () => {}; + const client = { release: released }; + const failure = new Error('synthetic'); + class FakePool { + totalCount = 2; + idleCount = 1; + waitingCount = 3; + fail = false; + connect(callback) { + now += 5; + if (callback) { + callback(this.fail ? failure : undefined, this.fail ? undefined : client, released); + return undefined; + } + return this.fail ? Promise.reject(failure) : Promise.resolve(client); + } + } + const modules = { + pg: { Pool: FakePool }, + 'node:fs': { + writeFileSync() {}, + appendFileSync(path, value) { records.push(JSON.parse(value)); }, + readFileSync(path) { return path.endsWith('memory.current') ? '1024' : 'usage_usec 10\n'; }, + }, + 'node:perf_hooks': { + performance: { now: () => now }, + monitorEventLoopDelay: () => ({ enable() {}, reset() {}, mean: 1, percentile: () => 1 }), + }, + }; + vm.runInNewContext(readFileSync('scripts/diagnostics/pool.cjs', 'utf8'), { + require: (name) => modules[name], + process: { pid: 1, env: { INSTANCE_ID: 'test' }, + memoryUsage: () => ({ rss: 2048 }), cpuUsage: () => ({ user: 1, system: 2 }) }, + setInterval(callback) { timer = callback; return { unref() {} }; }, + Date, + }); + const pool = new FakePool(); + assert.equal(await pool.connect(), client); + assert.equal(pool.connect((error, value, release) => { + assert.equal(error, undefined); + assert.equal(value, client); + assert.equal(release, released); + }), undefined); + pool.fail = true; + await assert.rejects(pool.connect(), (error) => error === failure); + pool.connect((error) => assert.equal(error, failure)); + now = 2000; + timer(); + assert.equal(records[0].pools[0].acquisitions, 4); + assert.equal(records[0].pools[0].errors, 2); + assert.equal(records[0].pools[0].acquire_ms, 20); + assert.equal(records[0].pools[0].interval_max_waiting, 3); + assert.equal(records[0].pools[0].interval_max_busy, 1); +}); diff --git a/test/redis-comparison-summary.mjs b/test/redis-comparison-summary.mjs new file mode 100644 index 0000000..1e4be20 --- /dev/null +++ b/test/redis-comparison-summary.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; + +const report = JSON.parse( + await readFile('evidence/redis-comparison.json', 'utf8'), +); +assert.ok(report.completed); +assert.ok(Object.values(report.correctness.read_after_write).every(Boolean)); +const lines = [ + '# Redis и работа без кэша приложения', + '', + `Дата UTC: ${report.recorded_at}. Node.js ${report.node}.`, + '', + 'Один прогон на одном образе: три повтора каждого режима с чередованием порядка. Второй экземпляр API запущен без CACHE_URL: отключены все кэши приложения, включая профиль пользователя. PostgreSQL-эпохи и запись одним CTE сохранены в обоих режимах. Контейнер Redis не останавливался; экземпляр без кэша не создаёт Redis-клиент.', + '', + 'RPS = суммарное число успешных запросов / суммарное время трёх повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее трёх отдельных p95, не перцентиль объединённой выборки. SQL/HTTP = суммарные SQL-вызовы / все HTTP-запросы; команды транзакций исключены.', + '', + '| Сценарий | Режим | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | CPU, % ядра | Ошибки |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', +]; +const aggregates = []; +for (const [workload, label] of [ + ['cold-history-burst', 'Холодная волна истории'], + ['warm-history', 'Прогретая история'], + ['writes-single-chat', 'Запись в один чат'], +]) { + for (const mode of ['redis', 'no-redis']) { + const samples = report.samples.filter( + (sample) => sample.workload === `${workload}:${mode}`, + ); + assert.equal(samples.length, 3); + const sum = (select) => + samples.reduce((total, sample) => total + select(sample), 0); + const successes = sum((sample) => sample.successes); + const errors = sum((sample) => sample.errors); + const seconds = sum((sample) => sample.elapsed_seconds); + const aggregate = { + workload, + mode, + successes, + errors, + rps: successes / seconds, + mean_latency_ms: + sum((sample) => sample.latency_ms.mean * sample.successes) / successes, + mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length, + sql_per_request: + sum((sample) => sample.sql_queries) / (successes + errors), + cpu_percent_one_core: + sum( + (sample) => + sample.process_cpu_percent_one_core * sample.elapsed_seconds, + ) / seconds, + }; + aggregates.push(aggregate); + lines.push( + `| ${label} | ${mode === 'redis' ? 'Redis' : 'Без Redis'} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.cpu_percent_one_core.toFixed(2)} | ${errors} |`, + ); + } +} +lines.push( + '', + 'В этом прогоне кэш полезен для прогретого чтения истории: выше пропускная способность, ниже средняя задержка и меньше SQL-вызовов. Для холодной волны и записи преимущество по времени не показано. Первый повтор с Redis значительно медленнее остальных; средние включают его, поэтому вывод о стабильной скорости записи ограничен. Это сравнение режимов кэша на текущей реализации, а не отдельное измерение цены PostgreSQL-эпох.', + '', + 'Нагрузка: 10 000 сообщений, первая страница limit=50, конкурентность 16, 3 с прогрева и 10 с чтения на режим/повтор, холодная волна 32 запроса, 512 записей в новый чат на режим/повтор. Реальный HTTP по loopback; синтетические пользователи и сессии обходят хеширование паролей. Проверены 512 сообщений и эпоха 512 после каждого сценария записи, а также чтение истории после записи.', + '', + 'Оба API и генератор HTTP работают в одном Node-процессе. CPU включает генератор, API и инструментацию; RSS общий и не показывает разницу памяти между режимами. PostgreSQL и Redis разделяют Docker-host с другими сервисами. Это короткая нагрузка, не предел производительности; холодные SQL-данные не имитируются. Уменьшение SQL/HTTP показывает снижение числа запросов, но не доказывает пропорциональное снижение CPU PostgreSQL. Цена самих эпох отдельно не измеряется.', + '', + 'Команда повторения:', + '', + '```powershell', + 'docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/redis-comparison.json --compare-redis', + 'node test/redis-comparison-summary.mjs', + '```', + '', + 'Сырые результаты и хеши измеренных модулей: [redis-comparison.json](redis-comparison.json).', +); +await writeFile( + 'evidence/redis-comparison-summary.md', + lines.join('\n') + '\n', +); +await writeFile( + 'evidence/redis-comparison-averages.json', + JSON.stringify(aggregates, null, 2) + '\n', +); +console.log(lines.slice(0, 16).join('\n')); diff --git a/test/resilience.mjs b/test/resilience.mjs new file mode 100644 index 0000000..35e043d --- /dev/null +++ b/test/resilience.mjs @@ -0,0 +1,444 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { performance } from 'node:perf_hooks'; +import test from 'node:test'; +import { faultProxy } from './helpers/fault-proxy.mjs'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); +const pause = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +test( + 'Auth and dependency failures remain bounded and fail closed', + { timeout: 120000 }, + async (t) => { + assert.ok( + process.env.DATABASE_URL, + 'Set DATABASE_URL and apply migrations', + ); + assert.ok( + process.env.CACHE_URL, + 'Set CACHE_URL to a running Redis instance', + ); + const original = { + DATABASE_URL: process.env.DATABASE_URL, + CACHE_URL: process.env.CACHE_URL, + CACHE_PREFIX: process.env.CACHE_PREFIX, + SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS, + }; + const prefix = `test:resilience:${randomUUID()}`; + const db = await faultProxy(original.DATABASE_URL, 5432); + const redis = await faultProxy(original.CACHE_URL, 6379); + Object.assign(process.env, { + DATABASE_URL: db.url, + CACHE_URL: redis.url, + CACHE_PREFIX: prefix, + SESSION_TTL_SECONDS: '2', + }); + const observer = new Redis(original.CACHE_URL, { + connectTimeout: 1000, + commandTimeout: 1000, + maxRetriesPerRequest: 0, + }); + observer.on('error', () => {}); + let app; + let prisma; + let userId; + let chatId; + t.after(async () => { + db.setMode('online'); + redis.setMode('online'); + try { + if (prisma) { + if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } }); + if (userId) await prisma.user.deleteMany({ where: { id: userId } }); + } + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + observer.disconnect(); + if (app) await app.close(); + await db.close(); + await redis.close(); + for (const [key, value] of Object.entries(original)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); + await observer.ping(); + app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + prisma = app.get(PrismaService); + let token; + const request = async ( + method, + url, + { + payload, + authorization = token ? `Bearer ${token}` : undefined, + headers = {}, + budget = 8000, + } = {}, + ) => { + const start = performance.now(); + let timer; + try { + const response = await Promise.race([ + server.inject({ + method, + url, + headers: { + ...headers, + ...(method === 'POST' && /\/messages$/.test(url) + ? { 'idempotency-key': randomUUID() } + : {}), + ...(authorization ? { authorization } : {}), + }, + ...(payload === undefined ? {} : { payload }), + }), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${method} ${url} exceeded ${budget}ms`)), + budget, + ); + }), + ]); + assert.ok( + performance.now() - start < budget, + 'request exceeded its deadline', + ); + return response; + } finally { + clearTimeout(timer); + } + }; + const errorResponse = (response, status, code, challenge) => { + assert.equal( + response.statusCode, + status, + `${response.statusCode}: ${response.body}`, + ); + const body = response.json(); + assert.deepEqual(Object.keys(body), ['error']); + assert.deepEqual(Object.keys(body.error).sort(), [ + 'code', + 'message', + 'request_id', + ]); + assert.equal(body.error.code, code); + assert.equal(body.error.request_id, response.headers['x-request-id']); + assert.ok(body.error.request_id); + assert.ok(body.error.message.length <= 200); + for (const secret of [ + original.DATABASE_URL, + original.CACHE_URL, + token, + ].filter(Boolean)) + assert.ok(!response.body.includes(secret)); + assert.ok( + !/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test( + response.body, + ), + 'error leaks implementation details', + ); + if (challenge) + assert.equal(response.headers['www-authenticate'], challenge); + }; + const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`; + const password = 'Synthetic-test-password:🔐123'; + const basic = (secret) => + `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`; + const login = (secret = password) => + request('POST', '/api/v1/auth/sessions', { + authorization: basic(secret), + }); + const registered = await request('POST', '/api/v1/users', { + payload: { username, display_name: 'Fault test', password }, + budget: 15000, + }); + assert.equal(registered.statusCode, 201); + userId = registered.json().id; + + await t.test('incorrect password is 401 with Basic challenge', async () => { + const wrong = await login('wrong-password'); + errorResponse( + wrong, + 401, + 'unauthorized', + 'Basic realm="messenger", charset="UTF-8"', + ); + assert.equal(await prisma.session.count({ where: { userId } }), 0); + }); + await t.test( + 'SESSION_TTL_SECONDS=2 expires without renewing on reads', + async () => { + const response = await login(); + assert.equal(response.statusCode, 201); + token = response.json().token; + const stored = await prisma.session.findFirstOrThrow({ + where: { userId }, + }); + assert.ok( + Math.abs( + stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000, + ) < 500, + ); + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + await pause( + Math.max(0, Date.parse(response.json().expires_at) - Date.now()) + + 100, + ); + errorResponse( + await request('GET', '/api/v1/users/me'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + }, + ); + await t.test( + 'logout revokes immediately, including with a cached profile', + async () => { + const response = await login(); + assert.equal(response.statusCode, 201); + token = response.json().token; + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + const logout = await request('DELETE', '/api/v1/auth/sessions/current'); + assert.equal(logout.statusCode, 204); + assert.equal(logout.body, ''); + errorResponse( + await request('GET', '/api/v1/users/me'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + }, + ); + token = randomBytes(32).toString('base64url'); + const { createHash } = await import('node:crypto'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + const chat = await request('POST', '/api/v1/chats', { + payload: { title: 'Fault isolation', member_ids: [userId] }, + }); + assert.equal(chat.statusCode, 201); + chatId = chat.json().id; + const history = `/api/v1/chats/${chatId}/messages`; + assert.equal( + (await request('POST', history, { payload: { text: 'Before fault' } })) + .statusCode, + 201, + ); + assert.equal((await request('GET', history)).json().items.length, 1); + + const anonymousChecks = async () => { + for (const authorization of ['', 'Bearer invalid', basic(password)]) { + const headers = { 'x-user-id': userId }; + errorResponse( + await request('GET', history, { authorization, headers }), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('POST', history, { + authorization, + headers, + payload: { text: 'Must not be stored' }, + }), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + } + }; + await t.test( + 'Redis disconnect: SQL fallback and anonymous callers remain unauthorized', + async () => { + redis.setMode('cut'); + await anonymousChecks(); + assert.equal( + (await request('GET', history, { budget: 3000 })).statusCode, + 200, + ); + assert.equal( + ( + await request('POST', history, { + payload: { text: 'Redis offline' }, + budget: 3000, + }) + ).statusCode, + 201, + ); + assert.equal( + (await request('GET', history, { budget: 3000 })).json().items.length, + 2, + ); + redis.setMode('online'); + await pause(1300); + }, + ); + await t.test('Redis hangs: command timeout bypasses cache', async () => { + redis.setMode('hang'); + try { + assert.equal( + (await request('GET', `${history}?limit=10`, { budget: 3000 })) + .statusCode, + 200, + ); + assert.equal( + ( + await request('POST', history, { + payload: { text: 'Redis blackhole' }, + budget: 3000, + }) + ).statusCode, + 201, + ); + await anonymousChecks(); + } finally { + redis.setMode('cut'); + redis.setMode('online'); + } + await pause(1300); + }); + await t.test( + 'PostgreSQL disconnect: bounded 503, no cached authorization bypass', + async () => { + db.setMode('cut'); + try { + await anonymousChecks(); + errorResponse( + await request('GET', '/api/v1/users/me'), + 503, + 'unavailable', + ); + errorResponse(await request('GET', history), 503, 'unavailable'); + errorResponse( + await request('POST', history, { payload: { text: 'DB offline' } }), + 503, + 'unavailable', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 503, + 'unavailable', + ); + errorResponse(await login(), 503, 'unavailable'); + errorResponse( + await request('GET', '/health/ready'), + 503, + 'unavailable', + ); + assert.equal((await request('GET', '/health/live')).statusCode, 200); + } finally { + db.setMode('online'); + } + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + }, + ); + await t.test( + 'PostgreSQL hangs on an established connection: client query timeout', + async () => { + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + db.setMode('hang'); + try { + await anonymousChecks(); + errorResponse(await request('GET', history), 503, 'unavailable'); + } finally { + db.setMode('cut'); + db.setMode('online'); + } + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + }, + ); + await t.test( + 'both services unavailable: fail closed and recover confirmed data', + async () => { + db.setMode('cut'); + redis.setMode('cut'); + try { + await anonymousChecks(); + errorResponse(await request('GET', history), 503, 'unavailable'); + errorResponse( + await request('GET', history, { + authorization: `Bearer ${randomBytes(32).toString('base64url')}`, + }), + 503, + 'unavailable', + ); + } finally { + db.setMode('online'); + redis.setMode('online'); + } + const recovered = await request('GET', `${history}?limit=100`); + assert.equal(recovered.statusCode, 200); + assert.deepEqual( + recovered.json().items.map((message) => message.text), + ['Before fault', 'Redis offline', 'Redis blackhole'], + ); + assert.equal(await prisma.message.count({ where: { chatId } }), 3); + assert.equal((await request('GET', '/health/ready')).statusCode, 200); + }, + ); + }, +); diff --git a/test/saturation-summary.py b/test/saturation-summary.py new file mode 100644 index 0000000..d61acfd --- /dev/null +++ b/test/saturation-summary.py @@ -0,0 +1,29 @@ +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location('saturation', Path(__file__).resolve().parents[1]/'scripts/run-saturation.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ClassificationTest(unittest.TestCase): + def classify(self, attempted=1000, dropped=0, errors=0, p95=500): + return module.classify({'measurement': {'scheduled_slots': 1000, + 'client_dropped_slots': dropped, 'all': {'attempted': attempted, 'errors': errors}}, + 'all_latency_ms': {'p95': p95}})['classification'] + + def test_delivered_healthy_workload_passes(self): + self.assertEqual(self.classify(), 'pass') + + def test_latency_or_errors_break_service_criterion(self): + self.assertEqual(self.classify(p95=1500), 'slo_fail') + self.assertEqual(self.classify(errors=20), 'slo_fail') + + def test_underfed_workload_is_not_a_server_capacity_measurement(self): + self.assertEqual(self.classify(attempted=800, dropped=200), 'generator_limited') + self.assertEqual(self.classify(attempted=800, dropped=200, p95=3000), 'generator_limited') + + +if __name__ == '__main__': + unittest.main()