test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab3 checks and retain one compact result without request logs or secrets."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def capture(command, timeout=180):
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout)
|
||||
|
||||
|
||||
def snapshot():
|
||||
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
|
||||
result.check_returncode()
|
||||
ids = result.stdout.split()
|
||||
inspected = capture(['docker', 'inspect', *ids], 30)
|
||||
inspected.check_returncode()
|
||||
rows = []
|
||||
for c in json.loads(inspected.stdout):
|
||||
service = c['Config']['Labels'].get('com.docker.compose.service')
|
||||
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
|
||||
continue
|
||||
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
|
||||
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
|
||||
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
|
||||
'health': c['State'].get('Health', {}).get('Status'),
|
||||
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
|
||||
'pool_max': pool, 'command': c['Config']['Cmd'],
|
||||
'ports': c['HostConfig'].get('PortBindings'),
|
||||
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
|
||||
for m in c['Mounts'] if m['Type'] == 'volume']})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
|
||||
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
|
||||
args = parser.parse_args()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if args.retry_failed:
|
||||
report = json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if report.get('source_sha256'):
|
||||
report['source_before_recheck_sha256'] = report['source_sha256']
|
||||
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
|
||||
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
|
||||
else:
|
||||
previous = (json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if args.output.exists() else None)
|
||||
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
|
||||
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
|
||||
if previous:
|
||||
report['previous_validation'] = {
|
||||
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
|
||||
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
|
||||
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
|
||||
for c in previous['checks']],
|
||||
}
|
||||
selected = None
|
||||
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||||
for name in ('src/main.ts', 'src/app.controller.ts',
|
||||
'src/common/http/fastify-adapter.ts',
|
||||
'infra/start-nginx.sh', 'scripts/stop-one.sh',
|
||||
'scripts/restart.sh', 'test/check_shutdown.py')}
|
||||
|
||||
def save():
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
def run(name, command, timeout=180, details=None, parse_json=False):
|
||||
if selected is not None and name not in selected and name != 'final-smoke':
|
||||
return None
|
||||
print(f'START {name}', flush=True)
|
||||
started = time.monotonic()
|
||||
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
|
||||
try:
|
||||
result = capture(command, timeout)
|
||||
item['exit_code'] = result.returncode
|
||||
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
|
||||
lines = (result.stdout + '\n' + result.stderr).splitlines()
|
||||
item['summary'] = [line for line in lines if line.startswith(
|
||||
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
|
||||
if details is not None and details.exists():
|
||||
item['details'] = json.loads(details.read_text(encoding='utf-8'))
|
||||
elif parse_json:
|
||||
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
|
||||
try:
|
||||
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
item.update(status='FAIL', error=type(exc).__name__)
|
||||
item['seconds'] = round(time.monotonic() - started, 3)
|
||||
previous = next((c for c in report['checks'] if c['name'] == name), None)
|
||||
if previous is not None:
|
||||
item['previous_attempts'] = previous.get('previous_attempts', []) + [
|
||||
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
|
||||
report['checks'][report['checks'].index(previous)] = item
|
||||
else:
|
||||
report['checks'].append(item)
|
||||
save()
|
||||
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
|
||||
for line in item.get('summary', [])[-3:]:
|
||||
print(line, flush=True)
|
||||
return item
|
||||
|
||||
def restore():
|
||||
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
|
||||
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
|
||||
result.check_returncode()
|
||||
time.sleep(6)
|
||||
|
||||
py = [sys.executable, '-X', 'utf8']
|
||||
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
|
||||
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
|
||||
save()
|
||||
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
|
||||
temp = Path(tmp)
|
||||
run('contract', py + ['scripts/check_contract.py'])
|
||||
run('lint', [pnpm, 'lint'])
|
||||
run('build', [pnpm, 'build'], 240)
|
||||
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
|
||||
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
|
||||
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
|
||||
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
|
||||
'-v', f'{ROOT / "infra"}:/source:ro',
|
||||
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
|
||||
'nginx:1.30.6-alpine', '/test.sh'], 60)
|
||||
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
|
||||
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
|
||||
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
|
||||
if selected is None or selected & {'persistence', 'public-failover'}:
|
||||
try:
|
||||
if selected is None or 'persistence' in selected:
|
||||
before = snapshot()
|
||||
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
|
||||
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
|
||||
after = snapshot()
|
||||
item['before'] = before
|
||||
item['after'] = after
|
||||
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
|
||||
same = {c['id']: c for c in after if c['service'] != 'migrate'}
|
||||
item['containers_restarted'] = restarted.keys() == same.keys() and all(
|
||||
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
|
||||
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
|
||||
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
|
||||
if not item['containers_restarted'] or not item['volumes_preserved']:
|
||||
item['status'] = 'FAIL'
|
||||
save()
|
||||
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
|
||||
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
|
||||
parse_json=True)
|
||||
finally:
|
||||
restore()
|
||||
for scenario in ('resources', 'failover', 'rolling'):
|
||||
result_file = temp / f'grade4-{scenario}.json'
|
||||
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
|
||||
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
|
||||
'--output', str(result_file)], 300, details=result_file)
|
||||
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
|
||||
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
|
||||
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
|
||||
if selected is None or 'migration-compatibility' in selected:
|
||||
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
|
||||
if available.returncode == 0:
|
||||
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
|
||||
else:
|
||||
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
|
||||
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
|
||||
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
report['final'] = snapshot()
|
||||
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
|
||||
report['finished'] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
|
||||
if not report['all_executed_checks_pass']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = {'backend', 'postgres', 'redis'}
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(
|
||||
['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
).split()
|
||||
if len(ids) != len(SERVICES):
|
||||
raise RuntimeError('Start backend, postgres and redis before this test')
|
||||
template = (
|
||||
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
||||
'"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"running":{{json .State.Running}},"mounts":{{json .Mounts}}}'
|
||||
)
|
||||
output = subprocess.check_output(
|
||||
['docker', 'inspect', '--format', template, *ids],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
)
|
||||
result = {}
|
||||
for line in output.splitlines():
|
||||
entry = json.loads(line)
|
||||
service = entry.pop('service')
|
||||
entry['volumes'] = [
|
||||
{'name': mount['Name'], 'destination': mount['Destination']}
|
||||
for mount in entry.pop('mounts') if mount['Type'] == 'volume'
|
||||
]
|
||||
result[service] = entry
|
||||
if set(result) != SERVICES or not all(v['running'] for v in result.values()):
|
||||
raise RuntimeError('All three services must be running')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json')
|
||||
args = parser.parse_args()
|
||||
before = snapshot()
|
||||
hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh')
|
||||
if args.scenario == 'public':
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence']
|
||||
else:
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')]
|
||||
command += ['--action-script', str(hook), '--base-url', args.base_url]
|
||||
if args.ca_file:
|
||||
command += ['--ca-file', args.ca_file]
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
|
||||
print(proc.stdout, end='')
|
||||
print(proc.stderr, end='', file=sys.stderr)
|
||||
if proc.returncode:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps({
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'result': 'FAIL', 'exit_code': proc.returncode,
|
||||
'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(),
|
||||
'before': before,
|
||||
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return proc.returncode
|
||||
after = snapshot()
|
||||
for service in sorted(SERVICES):
|
||||
old, new = before[service], after[service]
|
||||
if old['id'] != new['id'] or old['started_at'] == new['started_at']:
|
||||
raise RuntimeError(f'{service}: expected a restart of the same container')
|
||||
if not old['volumes'] or old['volumes'] != new['volumes']:
|
||||
raise RuntimeError(f'{service}: persistent volumes changed or are missing')
|
||||
evidence = {
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'elapsed_seconds': round(time.monotonic() - started, 3),
|
||||
'result': 'PASS', 'test_output': proc.stdout.strip(),
|
||||
'before': before, 'after': after,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: backend, postgres and redis restarted; named volumes preserved.')
|
||||
print(f'Evidence: {args.output}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
import { createRequire } from 'node:module';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
const require = createRequire(`${process.cwd()}/package.json`);
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
|
||||
const { PrismaService } = require(`${process.cwd()}/dist/prisma/prisma.service`);
|
||||
const { UserService } = require(`${process.cwd()}/dist/User/user.service`);
|
||||
const { PasswordService } = require(`${process.cwd()}/dist/Auth/password.service`);
|
||||
const { createFastifyAdapter } = require(`${process.cwd()}/dist/common/http/fastify-adapter`);
|
||||
process.env.CACHE_URL = '';
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false });
|
||||
await app.init();
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
const prisma = app.get(PrismaService);
|
||||
const passwords = app.get(PasswordService);
|
||||
const originalHash = passwords.hash.bind(passwords);
|
||||
const spans = [];
|
||||
passwords.hash = async (password) => {
|
||||
const start = performance.now();
|
||||
const value = await originalHash(password);
|
||||
spans.push({ phase: 'kdf', ms: Math.round(performance.now() - start) });
|
||||
return value;
|
||||
};
|
||||
app.get(UserService).prisma = prisma.$extends({ query: { user: { async create({ args, query }) {
|
||||
const start = performance.now();
|
||||
const value = await query(args);
|
||||
spans.push({ phase: 'sql_user_create', ms: Math.round(performance.now() - start) });
|
||||
return value;
|
||||
} } } });
|
||||
const ids = [];
|
||||
try {
|
||||
const health = await server.inject({ method: 'GET', url: '/health/ready' });
|
||||
console.log(JSON.stringify({ phase: 'health', status: health.statusCode }));
|
||||
for (let sample = 1; sample <= 3; sample++) {
|
||||
spans.length = 0;
|
||||
const start = performance.now();
|
||||
const response = await server.inject({ method: 'POST', url: '/api/v1/users', payload: {
|
||||
username: `warm_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
display_name: 'Synthetic startup diagnostic', password: 'Synthetic-diagnostic-password-123',
|
||||
} });
|
||||
if (response.statusCode === 201) ids.push(response.json().id);
|
||||
console.log(JSON.stringify({ sample, status: response.statusCode, total_ms: Math.round(performance.now() - start), spans }));
|
||||
}
|
||||
} finally {
|
||||
await prisma.user.deleteMany({ where: { id: { in: ids } } });
|
||||
await app.close();
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createServer } from 'node:http';
|
||||
import test from 'node:test';
|
||||
|
||||
test('scaling benchmark schedules reads and writes and reports distinct instances', async () => {
|
||||
let sequence = 0;
|
||||
const server = createServer(async (request, response) => {
|
||||
for await (const _ of request) {
|
||||
// Consume the request body before responding.
|
||||
}
|
||||
response.setHeader('Content-Type', 'application/json');
|
||||
response.setHeader('X-Instance-Id', 'mock1');
|
||||
let status = 200;
|
||||
let body = {};
|
||||
if (request.url === '/api/v1/users' && request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: `user-${++sequence}` };
|
||||
} else if (request.url === '/api/v1/auth/sessions') {
|
||||
status = 201;
|
||||
body = { token: 'synthetic-token' };
|
||||
} else if (request.url === '/api/v1/chats' && request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: 'chat-1' };
|
||||
} else if (request.url.startsWith('/api/v1/chats/chat-1/messages')) {
|
||||
if (request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: `message-${++sequence}` };
|
||||
} else {
|
||||
body = { items: [] };
|
||||
}
|
||||
} else {
|
||||
status = 404;
|
||||
}
|
||||
response.writeHead(status);
|
||||
response.end(JSON.stringify(body));
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
try {
|
||||
const port = server.address().port;
|
||||
const args = [
|
||||
'scripts/benchmark-scaling.mjs', '--base-url', `http://127.0.0.1:${port}`,
|
||||
'--label', 'mock', '--rate', '20', '--duration-seconds', '2',
|
||||
'--warmup-seconds', '1', '--seed-messages', '2',
|
||||
'--read-percent', '50', '--expected-instances', '1',
|
||||
];
|
||||
const child = spawn(process.execPath, args, { cwd: process.cwd() });
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => { stdout += chunk; });
|
||||
child.stderr.on('data', (chunk) => { stderr += chunk; });
|
||||
const code = await new Promise((resolve) => child.on('close', resolve));
|
||||
assert.equal(code, 0, stderr);
|
||||
const result = JSON.parse(stdout);
|
||||
assert.equal(result.measurement.scheduled_slots, 40);
|
||||
assert.equal(result.measurement.client_dropped_slots, 0);
|
||||
assert.equal(result.measurement.all.successful, 40);
|
||||
assert.equal(result.measurement.all.instance_counts.mock1, 40);
|
||||
assert.ok(result.measurement.read.successful > 0);
|
||||
assert.ok(result.measurement.write.successful > 0);
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,541 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(`${process.cwd()}/package.json`);
|
||||
const pg = require('pg');
|
||||
let counts = {};
|
||||
const originalQuery = pg.Client.prototype.query;
|
||||
pg.Client.prototype.query = function (query, ...args) {
|
||||
const sql = typeof query === 'string' ? query : query.text;
|
||||
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
|
||||
const table =
|
||||
['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) =>
|
||||
new RegExp(`\\b${name}\\b`).test(sql),
|
||||
) ?? 'other';
|
||||
const label = `${verb}:${table}`;
|
||||
counts[label] = (counts[label] ?? 0) + 1;
|
||||
return originalQuery.call(this, query, ...args);
|
||||
};
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
|
||||
const { PrismaService } = require(
|
||||
`${process.cwd()}/dist/prisma/prisma.service`,
|
||||
);
|
||||
const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`);
|
||||
const { createFastifyAdapter, exposeRequestIdHeader } = require(
|
||||
`${process.cwd()}/dist/common/http/fastify-adapter`,
|
||||
);
|
||||
const stage = process.argv[2];
|
||||
const compareRedis = process.argv.includes('--compare-redis');
|
||||
const historyControl = !process.argv.includes('--skip-history-control');
|
||||
function integerOption(name, fallback) {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
const value = index === -1 ? fallback : Number(process.argv[index + 1]);
|
||||
assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`);
|
||||
return value;
|
||||
}
|
||||
const concurrency = integerOption('concurrency', 16);
|
||||
const duration = integerOption('duration-seconds', 10);
|
||||
const warmup = integerOption('warmup-seconds', 3);
|
||||
const repetitions = integerOption('repetitions', 3);
|
||||
const seedMessages = integerOption('seed-messages', 10000);
|
||||
const coldBurst = integerOption('cold-burst', 32);
|
||||
const writesPerRepeat = integerOption('writes-per-repeat', 512);
|
||||
assert.ok(['before', 'after'].includes(stage));
|
||||
const prefix = `experiment:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
|
||||
const observer = new Redis(process.env.CACHE_URL);
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
const apps = [];
|
||||
async function start() {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
return { app, url: await app.getUrl() };
|
||||
}
|
||||
const first = await start();
|
||||
const prisma = first.app.get(PrismaService);
|
||||
let userId;
|
||||
const chatIds = [];
|
||||
const report = {
|
||||
stage,
|
||||
recorded_at: new Date().toISOString(),
|
||||
node: process.version,
|
||||
methodology: {
|
||||
requests:
|
||||
'real loopback HTTP; instrumented API process in measured backend image',
|
||||
sql: 'pg.Client.query calls; transaction control counted separately',
|
||||
concurrency,
|
||||
duration_seconds: duration,
|
||||
warmup_seconds: warmup,
|
||||
repetitions,
|
||||
seed_messages: seedMessages,
|
||||
cold_burst_requests: coldBurst,
|
||||
writes_per_repeat: writesPerRepeat,
|
||||
history_control: historyControl,
|
||||
resources:
|
||||
'process CPU/RSS includes HTTP generator and instrumentation; not container CPU',
|
||||
limitations:
|
||||
'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF',
|
||||
},
|
||||
dist_sha256: {},
|
||||
dependencies: {},
|
||||
samples: [],
|
||||
correctness: {},
|
||||
};
|
||||
for (const name of [
|
||||
'@nestjs/core',
|
||||
'@prisma/adapter-pg',
|
||||
'pg',
|
||||
'ioredis',
|
||||
'fastify',
|
||||
]) {
|
||||
try {
|
||||
report.dependencies[name] = require(`${name}/package.json`).version;
|
||||
} catch {
|
||||
report.dependencies[name] = 'package metadata not exported';
|
||||
}
|
||||
}
|
||||
for (const name of [
|
||||
'Chat/chat.service.js',
|
||||
'cache/cache.service.js',
|
||||
'Auth/auth.service.js',
|
||||
'User/user.guard.js',
|
||||
'Chat/chat.controller.js',
|
||||
]) {
|
||||
report.dist_sha256[name] = createHash('sha256')
|
||||
.update(await readFile(`${process.cwd()}/dist/${name}`))
|
||||
.digest('hex');
|
||||
}
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
async function clearCache() {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
}
|
||||
let token;
|
||||
async function request(method, path, body, target = first) {
|
||||
const response = await fetch(`${target.url}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(body ? { 'content-type': 'application/json' } : {}),
|
||||
...(method === 'POST' && /\/messages$/.test(path)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
const json = await response.json();
|
||||
assert.equal(
|
||||
response.status,
|
||||
method === 'POST' ? 201 : 200,
|
||||
`HTTP ${response.status}`,
|
||||
);
|
||||
return json;
|
||||
}
|
||||
async function newChat() {
|
||||
const chat = await prisma.chat.create({
|
||||
data: {
|
||||
title: 'Synthetic epoch benchmark',
|
||||
members: { create: { userId, position: 0 } },
|
||||
},
|
||||
});
|
||||
chatIds.push(chat.id);
|
||||
return chat.id;
|
||||
}
|
||||
async function measure(
|
||||
workload,
|
||||
repeat,
|
||||
operation,
|
||||
{
|
||||
duration: secondsToRun = duration,
|
||||
total,
|
||||
concurrency: clients = concurrency,
|
||||
} = {},
|
||||
) {
|
||||
counts = {};
|
||||
const cpu = process.cpuUsage();
|
||||
let peakRss = process.memoryUsage().rss;
|
||||
const timer = setInterval(() => {
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
}, 100);
|
||||
const latencies = [];
|
||||
let errors = 0,
|
||||
index = 0;
|
||||
const start = performance.now();
|
||||
await Promise.all(
|
||||
Array.from({ length: clients }, async () => {
|
||||
while (
|
||||
total === undefined
|
||||
? performance.now() - start < secondsToRun * 1000
|
||||
: index < total
|
||||
) {
|
||||
const item = index++;
|
||||
const began = performance.now();
|
||||
try {
|
||||
await operation(item);
|
||||
latencies.push(performance.now() - began);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
const seconds = (performance.now() - start) / 1000;
|
||||
clearInterval(timer);
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
const used = process.cpuUsage(cpu);
|
||||
latencies.sort((a, b) => a - b);
|
||||
const percentile = (p) =>
|
||||
latencies.length
|
||||
? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3)
|
||||
: null;
|
||||
const sql = Object.entries(counts)
|
||||
.filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key))
|
||||
.reduce((sum, [, n]) => sum + n, 0);
|
||||
const sample = {
|
||||
workload,
|
||||
repeat,
|
||||
concurrency: clients,
|
||||
elapsed_seconds: +seconds.toFixed(3),
|
||||
successes: latencies.length,
|
||||
errors,
|
||||
error_ratio: errors / (latencies.length + errors),
|
||||
successful_rps: +(latencies.length / seconds).toFixed(3),
|
||||
latency_ms: {
|
||||
mean: latencies.length
|
||||
? +(
|
||||
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
|
||||
).toFixed(3)
|
||||
: null,
|
||||
p50: percentile(0.5),
|
||||
p95: percentile(0.95),
|
||||
p99: percentile(0.99),
|
||||
},
|
||||
sql_queries: sql,
|
||||
sql_per_request: +(sql / (latencies.length + errors)).toFixed(4),
|
||||
sql_calls: { ...counts },
|
||||
process_cpu_percent_one_core: +(
|
||||
(used.user + used.system) /
|
||||
(seconds * 10000)
|
||||
).toFixed(2),
|
||||
peak_process_rss_mib: +(peakRss / 1048576).toFixed(2),
|
||||
};
|
||||
report.samples.push(sample);
|
||||
console.log(JSON.stringify(sample));
|
||||
assert.equal(errors, 0, 'load errors recorded in report');
|
||||
}
|
||||
try {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic benchmark',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 3600000),
|
||||
},
|
||||
});
|
||||
const readChat = await newChat();
|
||||
for (let offset = 0; offset < seedMessages; offset += 1000) {
|
||||
await prisma.message.createMany({
|
||||
data: Array.from(
|
||||
{ length: Math.min(1000, seedMessages - offset) },
|
||||
(_, i) => ({
|
||||
chatId: readChat,
|
||||
senderId: userId,
|
||||
text: `Synthetic message ${offset + i}`,
|
||||
createdAt: new Date(1700000000000 + offset + i),
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
await prisma.chatMessageEpoch.createMany({
|
||||
data: Array.from({ length: 16 }, (_, bucket) => ({
|
||||
chatId: readChat,
|
||||
bucket,
|
||||
epoch: BigInt(
|
||||
Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0),
|
||||
),
|
||||
})),
|
||||
});
|
||||
const path = `/api/v1/chats/${readChat}/messages?limit=50`;
|
||||
if (compareRedis) {
|
||||
const cacheUrl = process.env.CACHE_URL;
|
||||
delete process.env.CACHE_URL;
|
||||
let withoutRedis;
|
||||
try {
|
||||
withoutRedis = await start();
|
||||
} finally {
|
||||
process.env.CACHE_URL = cacheUrl;
|
||||
}
|
||||
assert.equal(withoutRedis.app.get(CacheService).redis, undefined);
|
||||
report.methodology.comparison = {
|
||||
modes: ['redis', 'no-redis'],
|
||||
no_redis:
|
||||
'API started without CACHE_URL; all application Redis caches disabled',
|
||||
epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes',
|
||||
order: 'redis first in repeats 1/3, no-redis first in repeat 2',
|
||||
processes:
|
||||
'both API instances and HTTP generator share one measured Node process',
|
||||
};
|
||||
report.correctness.read_after_write = {};
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
const variants = [
|
||||
{ name: 'redis', target: first },
|
||||
{ name: 'no-redis', target: withoutRedis },
|
||||
];
|
||||
if (repeat % 2 === 0) variants.reverse();
|
||||
for (const { name, target } of variants) {
|
||||
await clearCache();
|
||||
await measure(
|
||||
`cold-history-burst:${name}`,
|
||||
repeat,
|
||||
() => request('GET', path, undefined, target),
|
||||
{ total: coldBurst, concurrency: coldBurst },
|
||||
);
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil)
|
||||
await request('GET', path, undefined, target);
|
||||
await measure(`warm-history:${name}`, repeat, () =>
|
||||
request('GET', path, undefined, target),
|
||||
);
|
||||
const writeChat = await newChat();
|
||||
const writePath = `/api/v1/chats/${writeChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', writePath, undefined, target)).items.length,
|
||||
0,
|
||||
);
|
||||
await measure(
|
||||
`writes-single-chat:${name}`,
|
||||
repeat,
|
||||
(i) =>
|
||||
request(
|
||||
'POST',
|
||||
writePath,
|
||||
{ text: `Concurrent write ${i}` },
|
||||
target,
|
||||
),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
assert.equal(
|
||||
await messageEpoch(prisma, writeChat),
|
||||
BigInt(writesPerRepeat),
|
||||
);
|
||||
const page = await request('GET', writePath, undefined, target);
|
||||
assert.equal(page.items.length, 50);
|
||||
assert.equal(new Set(page.items.map((message) => message.id)).size, 50);
|
||||
report.correctness.read_after_write[`${name}:${repeat}`] = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const cache = first.app.get(CacheService);
|
||||
const remember = cache.remember.bind(cache);
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
await clearCache();
|
||||
await request('GET', `/api/v1/chats/${readChat}`);
|
||||
await measure('cold-history-burst', repeat, () => request('GET', path), {
|
||||
total: coldBurst,
|
||||
concurrency: coldBurst,
|
||||
});
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil) await request('GET', path);
|
||||
await measure('warm-history', repeat, () => request('GET', path));
|
||||
if (historyControl) {
|
||||
cache.remember = (namespace, parameters, ttl, load) =>
|
||||
namespace.startsWith('messages:')
|
||||
? load()
|
||||
: remember(namespace, parameters, ttl, load);
|
||||
const directUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < directUntil) await request('GET', path);
|
||||
await measure('history-cache-bypassed', repeat, () =>
|
||||
request('GET', path),
|
||||
);
|
||||
cache.remember = remember;
|
||||
}
|
||||
const writeChat = await newChat();
|
||||
await request('GET', `/api/v1/chats/${writeChat}`);
|
||||
await measure(
|
||||
'writes-single-chat',
|
||||
repeat,
|
||||
(i) =>
|
||||
request('POST', `/api/v1/chats/${writeChat}/messages`, {
|
||||
text: `Concurrent write ${i}`,
|
||||
}),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
}
|
||||
const second = await start();
|
||||
const faultChat = await newChat();
|
||||
const faultPath = `/api/v1/chats/${faultChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', faultPath, undefined, second)).items.length,
|
||||
0,
|
||||
);
|
||||
const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`);
|
||||
assert.ok(keysBefore.length);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
const written = await request('POST', faultPath, {
|
||||
text: 'Committed while Redis is unavailable',
|
||||
});
|
||||
const during = await request('GET', faultPath, undefined, second);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis?.status === 'ready'))
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
const recovered = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.redis_reconnect = {
|
||||
old_keys_retained:
|
||||
(await observer.exists(...keysBefore)) === keysBefore.length,
|
||||
read_after_write_during_outage: during.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
read_after_write_after_reconnect_other_replica: recovered.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
};
|
||||
await clearCache();
|
||||
const cold = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.empty_cache_recovery = cold.items.some(
|
||||
(x) => x.id === written.id,
|
||||
);
|
||||
const raceChat = await newChat();
|
||||
const racePath = `/api/v1/chats/${raceChat}/messages`;
|
||||
const service = first.app.get(
|
||||
require(`${process.cwd()}/dist/Chat/chat.service`).ChatService,
|
||||
);
|
||||
const actualPrisma = service.prisma;
|
||||
let release, began;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
service.prisma = actualPrisma.$extends({
|
||||
query: {
|
||||
message: {
|
||||
async findMany({ args, query }) {
|
||||
const rows = await query(args);
|
||||
if (args.where?.chatId === raceChat) {
|
||||
began();
|
||||
await held;
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const pending = request('GET', racePath);
|
||||
let fresh;
|
||||
try {
|
||||
await Promise.race([
|
||||
started,
|
||||
sleep(10000).then(() => {
|
||||
throw new Error('race load did not start');
|
||||
}),
|
||||
]);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
fresh = await request(
|
||||
'POST',
|
||||
racePath,
|
||||
{ text: 'New message after old SELECT' },
|
||||
second,
|
||||
);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready')
|
||||
)
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
} finally {
|
||||
release();
|
||||
service.prisma = actualPrisma;
|
||||
}
|
||||
await pending;
|
||||
report.correctness.delayed_old_fill = (
|
||||
await request('GET', racePath, undefined, second)
|
||||
).items.some((x) => x.id === fresh.id);
|
||||
if (stage === 'after') {
|
||||
assert.equal(
|
||||
report.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
true,
|
||||
);
|
||||
assert.equal(report.correctness.delayed_old_fill, true);
|
||||
assert.equal(report.correctness.empty_cache_recovery, true);
|
||||
}
|
||||
}
|
||||
report.completed = true;
|
||||
} finally {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
await clearCache();
|
||||
report.cleanup = 'only experiment chats/user/cache prefix removed';
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
await writeFile(
|
||||
process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`,
|
||||
JSON.stringify(report, null, 2) + '\n',
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
stage,
|
||||
correctness: report.correctness,
|
||||
completed: report.completed,
|
||||
}),
|
||||
);
|
||||
@@ -0,0 +1,25 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$dockerInfo = docker info --format '{{json .}}' | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) { throw 'docker info failed' }
|
||||
$processor = Get-CimInstance Win32_Processor | Select-Object -First 1
|
||||
$computer = Get-CimInstance Win32_ComputerSystem
|
||||
$containers = docker inspect mtusi2026lab2-fork-backend-1 mtusi2026lab2-fork-postgres-1 mtusi2026lab2-fork-redis-1 | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) { throw 'docker inspect failed' }
|
||||
$metadata = [ordered]@{
|
||||
recorded_at = [DateTime]::UtcNow.ToString('o')
|
||||
host_cpu = $processor.Name
|
||||
host_physical_cores = $processor.NumberOfCores
|
||||
host_logical_processors = $processor.NumberOfLogicalProcessors
|
||||
host_ram_bytes = $computer.TotalPhysicalMemory
|
||||
docker_version = $dockerInfo.ServerVersion
|
||||
docker_cpu_count = $dockerInfo.NCPU
|
||||
docker_ram_bytes = $dockerInfo.MemTotal
|
||||
docker_kernel = $dockerInfo.KernelVersion
|
||||
containers = @($containers | ForEach-Object {
|
||||
[ordered]@{ name = $_.Name; image = $_.Image; memory_limit_bytes = $_.HostConfig.Memory; nano_cpus_limit = $_.HostConfig.NanoCpus; cpu_quota = $_.HostConfig.CpuQuota }
|
||||
})
|
||||
before_image = 'sha256:075170aed76a42aa98b435cfb918d83879259b7e17177275d5274e328198e119'
|
||||
before_image_local_tag = 'lab2-cache-epochs:before'
|
||||
}
|
||||
$metadataPath = Join-Path (Get-Location).Path 'evidence/cache-epochs-environment.json'
|
||||
[System.IO.File]::WriteAllText($metadataPath, ($metadata | ConvertTo-Json -Depth 6) + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false))
|
||||
@@ -0,0 +1,91 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const reports = await Promise.all(
|
||||
['after', 'optimized'].map(async (name) =>
|
||||
JSON.parse(await readFile(`evidence/cache-epochs-${name}.json`, 'utf8')),
|
||||
),
|
||||
);
|
||||
assert.ok(reports.every((report) => report.completed));
|
||||
assert.deepEqual(reports[0].methodology, reports[1].methodology);
|
||||
const median = (values) =>
|
||||
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const labels = ['Отдельная транзакция', 'Один CTE'];
|
||||
const lines = [
|
||||
'# Оптимизация эпохи кэша в PostgreSQL',
|
||||
'',
|
||||
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
|
||||
'',
|
||||
'Сравнение сохранённого замера прежней реализации и нового замера после оптимизации. В обоих вариантах эпоха хранится в PostgreSQL; новый вариант объединяет проверку членства, UPDATE эпохи и INSERT сообщения одним запросом. При чтении истории убрано получение неиспользуемых метаданных чата.',
|
||||
'',
|
||||
'Медианы трёх повторов. Перцентили — медианы перцентилей повторов, без объединения выборок.',
|
||||
'',
|
||||
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | BEGIN/COMMIT на HTTP | CPU, % ядра | RSS, MiB | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const workload of new Set(
|
||||
reports[0].samples.map((sample) => sample.workload),
|
||||
)) {
|
||||
for (const [index, report] of reports.entries()) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.workload === workload,
|
||||
);
|
||||
assert.equal(samples.length, 3);
|
||||
const get = (select) => median(samples.map(select)).toFixed(3);
|
||||
const controls = (sample) =>
|
||||
Object.entries(sample.sql_calls)
|
||||
.filter(([key]) => /^(BEGIN|COMMIT):/.test(key))
|
||||
.reduce((total, [, count]) => total + count, 0) /
|
||||
(sample.successes + sample.errors);
|
||||
lines.push(
|
||||
`| ${workload} | ${labels[index]} | ${get((sample) => sample.successful_rps)} | ${get((sample) => sample.latency_ms.p50)} | ${get((sample) => sample.latency_ms.p95)} | ${get((sample) => sample.latency_ms.p99)} | ${get((sample) => sample.sql_per_request)} | ${get(controls)} | ${get((sample) => sample.process_cpu_percent_one_core)} | ${get((sample) => sample.peak_process_rss_mib)} | ${samples.reduce((total, sample) => total + sample.errors, 0)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'| Проверка | Отдельная транзакция | Один CTE |',
|
||||
'| --- | --- | --- |',
|
||||
);
|
||||
for (const [label, select] of [
|
||||
[
|
||||
'Чтение после записи при отказе Redis',
|
||||
(report) =>
|
||||
report.correctness.redis_reconnect.read_after_write_during_outage,
|
||||
],
|
||||
[
|
||||
'Чтение другой репликой после восстановления Redis',
|
||||
(report) =>
|
||||
report.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
],
|
||||
[
|
||||
'Старые ключи сохранились в Redis',
|
||||
(report) => report.correctness.redis_reconnect.old_keys_retained,
|
||||
],
|
||||
[
|
||||
'Восстановление с пустым кэшем',
|
||||
(report) => report.correctness.empty_cache_recovery,
|
||||
],
|
||||
[
|
||||
'Задержанная загрузка старой истории',
|
||||
(report) => report.correctness.delayed_old_fill,
|
||||
],
|
||||
]) {
|
||||
lines.push(
|
||||
`| ${label} | ${reports.map((report) => (select(report) ? 'PASS' : 'FAIL')).join(' | ')} |`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'Методика: тот же `test/cache-benchmark.mjs`, 10 000 сообщений, limit=50, конкурентность 16, 3 с прогрева и 10 с чтения, 512 записей в новый чат, три повтора. Реальный HTTP по loopback в отдельных benchmark-контейнерах; общий PostgreSQL/Redis. Токены и пользователи создаются напрямую, хеширование паролей не измеряется. BEGIN/COMMIT исключены из SQL/HTTP; запрос WITH учитывается как один вызов pg.Client.query.',
|
||||
'',
|
||||
'CPU/RSS включают API, генератор HTTP и инструментацию. Docker-host разделяется с другими работающими сервисами. Старый и новый замеры сделаны в разное время; изменение фоновой нагрузки и состояния host ограничивает сравнение RPS/latency. Замеры не доказывают максимальную ёмкость или SLA. UPDATE общей строки эпохи по-прежнему сериализует записи одного чата.',
|
||||
'',
|
||||
'Сырые результаты: [до оптимизации](cache-epochs-after.json), [после оптимизации](cache-epochs-optimized.json). Хеши измеренных модулей сохранены в JSON. Исторические замеры до введения эпох остаются в `cache-epochs-summary.md`.',
|
||||
);
|
||||
await writeFile(
|
||||
'evidence/cache-epochs-optimization-summary.md',
|
||||
lines.join('\n') + '\n',
|
||||
);
|
||||
console.log(lines.join('\n'));
|
||||
@@ -0,0 +1,200 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { CacheService } = require('../dist/cache/cache.service');
|
||||
const { ChatService } = require('../dist/Chat/chat.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
test(
|
||||
'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback',
|
||||
{ timeout: 90000 },
|
||||
async (t) => {
|
||||
assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL);
|
||||
const oldUrl = process.env.CACHE_URL;
|
||||
const oldPrefix = process.env.CACHE_PREFIX;
|
||||
const prefix = `test:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(oldUrl, 6379);
|
||||
const observer = new Redis(oldUrl);
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
const apps = [];
|
||||
let prisma, userId;
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
if (prisma) {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
}
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
process.env.CACHE_URL = oldUrl;
|
||||
if (oldPrefix === undefined) delete process.env.CACHE_PREFIX;
|
||||
else process.env.CACHE_PREFIX = oldPrefix;
|
||||
}
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
await app.getHttpAdapter().getInstance().ready();
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
}
|
||||
prisma = apps[0].get(PrismaService);
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic epochs',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
const call = async (replica, method, url, payload) => {
|
||||
const response = await apps[replica]
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(payload ? { payload } : {}),
|
||||
});
|
||||
return { status: response.statusCode, body: response.json() };
|
||||
};
|
||||
const createChat = async () => {
|
||||
const response = await call(0, 'POST', '/api/v1/chats', {
|
||||
title: 'Epoch test',
|
||||
member_ids: [userId],
|
||||
});
|
||||
assert.equal(response.status, 201);
|
||||
chatIds.push(response.body.id);
|
||||
return response.body.id;
|
||||
};
|
||||
assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const chatId = await createChat();
|
||||
proxy.setMode('online');
|
||||
const reconnect = async () => {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis.status === 'ready'))
|
||||
return;
|
||||
await pause(100);
|
||||
}
|
||||
assert.fail('Redis did not reconnect');
|
||||
};
|
||||
await reconnect();
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id),
|
||||
[chatId],
|
||||
);
|
||||
assert.equal(
|
||||
(await prisma.user.findUniqueOrThrow({ where: { id: userId } }))
|
||||
.chatsEpoch,
|
||||
1n,
|
||||
);
|
||||
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal((await call(1, 'GET', path)).body.items.length, 0);
|
||||
const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`);
|
||||
assert.ok(oldKeys.length);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const posted = await call(0, 'POST', path, {
|
||||
text: 'Write during Redis outage',
|
||||
});
|
||||
assert.equal(posted.status, 201);
|
||||
proxy.setMode('online');
|
||||
await reconnect();
|
||||
assert.equal(
|
||||
await observer.exists(...oldKeys),
|
||||
oldKeys.length,
|
||||
'old cache keys must still exist',
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', path)).body.items.map((x) => x.id),
|
||||
[posted.body.id],
|
||||
);
|
||||
|
||||
const service = apps[0].get(ChatService);
|
||||
const original = service.prisma;
|
||||
service.prisma = original.$extends({
|
||||
query: {
|
||||
$queryRaw({ args, query }) {
|
||||
const index = args.values.indexOf('Must roll back');
|
||||
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
assert.equal(
|
||||
(await call(0, 'POST', path, { text: 'Must roll back' })).status,
|
||||
500,
|
||||
);
|
||||
} finally {
|
||||
service.prisma = original;
|
||||
}
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 1n);
|
||||
|
||||
const writes = await Promise.all(
|
||||
Array.from({ length: 20 }, (_, i) =>
|
||||
call(i % 2, 'POST', path, { text: `Concurrent ${i}` }),
|
||||
),
|
||||
);
|
||||
assert.ok(writes.every((x) => x.status === 201));
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 21);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 21n);
|
||||
const history = (await call(1, 'GET', path)).body.items;
|
||||
assert.equal(new Set(history.map((x) => x.id)).size, 21);
|
||||
assert.ok(
|
||||
writes.every((x) => history.some((message) => message.id === x.body.id)),
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
const before = JSON.parse(
|
||||
await readFile('evidence/cache-epochs-before.json', 'utf8'),
|
||||
);
|
||||
const after = JSON.parse(
|
||||
await readFile('evidence/cache-epochs-after.json', 'utf8'),
|
||||
);
|
||||
assert.ok(before.completed && after.completed);
|
||||
assert.deepEqual(before.methodology, after.methodology);
|
||||
const median = (values) =>
|
||||
values.sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const lines = [
|
||||
'# Эпохи кэша — сравнение 2026-10-13',
|
||||
'',
|
||||
'Медианы трёх повторов. p50/p95/p99 — медианы перцентилей отдельных повторов, не перцентили объединённой выборки.',
|
||||
'',
|
||||
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | CPU процесса, % ядра | Пиковый RSS, MiB | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const workload of [...new Set(before.samples.map((x) => x.workload))]) {
|
||||
for (const report of [before, after]) {
|
||||
const samples = report.samples.filter((x) => x.workload === workload);
|
||||
assert.equal(samples.length, 3);
|
||||
const get = (fn) => median(samples.map(fn)).toFixed(3);
|
||||
lines.push(
|
||||
`| ${workload} | ${report.stage} | ${get((x) => x.successful_rps)} | ${get((x) => x.latency_ms.p50)} | ${get((x) => x.latency_ms.p95)} | ${get((x) => x.latency_ms.p99)} | ${get((x) => x.sql_per_request)} | ${get((x) => x.process_cpu_percent_one_core)} | ${get((x) => x.peak_process_rss_mib)} | ${samples.reduce((sum, x) => sum + x.errors, 0)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push('', '| Проверка | До | После |', '| --- | --- | --- |');
|
||||
for (const [label, select] of [
|
||||
[
|
||||
'Чтение после успешной записи при недоступном Redis',
|
||||
(x) => x.correctness.redis_reconnect.read_after_write_during_outage,
|
||||
],
|
||||
[
|
||||
'Чтение другой репликой после восстановления Redis со старыми ключами',
|
||||
(x) =>
|
||||
x.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
],
|
||||
[
|
||||
'Старые ключи действительно сохранились',
|
||||
(x) => x.correctness.redis_reconnect.old_keys_retained,
|
||||
],
|
||||
['Восстановление с пустым кэшем', (x) => x.correctness.empty_cache_recovery],
|
||||
[
|
||||
'Задержанная загрузка старой истории не отравляет следующий GET',
|
||||
(x) => x.correctness.delayed_old_fill,
|
||||
],
|
||||
])
|
||||
lines.push(
|
||||
`| ${label} | ${select(before) ? 'PASS' : 'FAIL'} | ${select(after) ? 'PASS' : 'FAIL'} |`,
|
||||
);
|
||||
lines.push(
|
||||
'',
|
||||
'Методика: `test/cache-benchmark.mjs`, сырой результат: `cache-epochs-before.json` и `cache-epochs-after.json`. Конфигурация: `cache-epochs-environment.json`.',
|
||||
'',
|
||||
'Реальный HTTP по loopback, 10 000 синтетических сообщений, первая страница limit=50. На повтор: холодная волна 32 запросов; прогрев 3 с; чтение с конкурентностью 16 в течение 10 с; прогрев 3 с и такой же контроль с отключённым кэшем истории; 512 записей в новый чат с конкурентностью 16. Три повтора. SQL считается на уровне pg.Client.query; BEGIN/COMMIT/ROLLBACK/SET вынесены в raw sql_calls и исключены из SQL/HTTP. Пароли не измеряются: пользователь и сессия создаются напрямую, токен остаётся в памяти.',
|
||||
'',
|
||||
'CPU/RSS относятся к процессу эксперимента, включающему API, генератор HTTP и инструментацию. Docker Desktop: 4 CPU, около 2.80 GiB RAM, отдельные лимиты сервисов не заданы. Замеры closed-loop, короткие и последовательные before → after; разброс и прогрев ограничивают выводы о latency/RPS. Это не тест максимальной ёмкости или SLA. Первый холодный повтор также включает прогрев runtime.',
|
||||
'',
|
||||
'Первичные короткие замеры (1 с прогрева, 5 с чтений, 128 записей) сохранены отдельно в cache-epochs-before-short.json и cache-epochs-after-short.json. Из-за большого разброса сделано одно повторное сравнение с увеличенным прогревом и нагрузкой; основные результаты получены в отдельных контейнерах benchmark из соответствующих образов. PostgreSQL/Redis и остальные контейнеры разделяют ресурсы Docker-host. Другие проекты на этом host не останавливались. Формулировки provenance в основных JSON уточнены после перехода от docker exec к отдельному контейнеру; численные измерения не менялись.',
|
||||
'',
|
||||
'До изменений использован фактически запущенный образ (локальный тег lab2-cache-epochs:before), после — пересобранный проект. Старый образ ещё использовал OR для курсорной пагинации; измеряемая первая страница не содержит курсора, поэтому этот путь в нагрузочном сравнении не выполняется. Хеши измеренных модулей записаны в JSON.',
|
||||
'',
|
||||
'Эпоха истории читается JOIN-ом вместе с проверкой членства; эпоха списка чатов — JOIN-ом с проверкой сессии. Для отдельного запроса дополнительного чтения эпохи нет. Однако raw JOIN проверки членства не использует прежний batching findUnique в Prisma: под конкуренцией суммарный SQL/HTTP может вырасти. Запись добавляет UPDATE чата и транзакционные команды; обновления одного чата сериализуются. Старые ключи недоступны новым читателям и удаляются по TTL.',
|
||||
'',
|
||||
'Корректность зависит от основного PostgreSQL и обновления эпох во всех путях изменения данных. Прямые SQL-изменения в обход приложения требуют увеличения соответствующей эпохи. После восстановления PostgreSQL из старого backup следует сменить CACHE_PREFIX или очистить кэш, чтобы исключить повторное использование эпох. Сохранность Docker-host и высокая доступность PostgreSQL не исследовались. Cache stampede не устраняется этим изменением.',
|
||||
);
|
||||
await writeFile('evidence/cache-epochs-summary.md', lines.join('\n') + '\n');
|
||||
console.log(lines.slice(0, 14).join('\n'));
|
||||
@@ -0,0 +1,117 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const resultPrefix = process.argv[2] ?? 'cache-strategy';
|
||||
assert.match(resultPrefix, /^[a-z0-9-]+$/);
|
||||
|
||||
const reports = await Promise.all(
|
||||
['full', 'no-redis-no-epochs'].map(async (name) =>
|
||||
JSON.parse(await readFile(`evidence/${resultPrefix}-${name}.json`, 'utf8')),
|
||||
),
|
||||
);
|
||||
const provenance = JSON.parse(
|
||||
await readFile(`evidence/${resultPrefix}-experiment.json`, 'utf8'),
|
||||
);
|
||||
assert.ok(reports.every((report) => report.completed));
|
||||
assert.deepEqual(reports[0].methodology, reports[1].methodology);
|
||||
assert.equal(provenance.exit_code, 0);
|
||||
assert.ok(provenance.sources_unchanged);
|
||||
assert.ok(reports[1].correctness.epochs_unchanged);
|
||||
assert.ok(reports[1].correctness.no_redis_client);
|
||||
const median = (values) =>
|
||||
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const labels = ['Redis и эпохи', 'Без Redis и эпох'];
|
||||
const aggregates = [];
|
||||
const settings = reports[0].methodology;
|
||||
const lines = [
|
||||
'# Полная стратегия кэша и вариант без Redis и эпох',
|
||||
'',
|
||||
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
|
||||
'',
|
||||
'Сначала выполнен свежий прогон текущего образа с Redis, PostgreSQL-эпохами и записью сообщения через CTE. Затем шесть модулей приложения заменены временными версиями без поддержания эпох: убраны изменения счётчиков и чтения эпох в проверках сессии/членства. Объединённые проверки сессии, публичного профиля и доступа к истории сохранены. Сообщение вставляется одним INSERT ... SELECT с проверкой членства, без обновления счётчиков. Второй API запущен без CACHE_URL: все кэши приложения отключены. Исходники рабочего проекта и запущенный backend не изменены.',
|
||||
'',
|
||||
`Параметры обоих прогонов совпадают: ${settings.repetitions} повтора, конкурентность ${settings.concurrency}, ${settings.seed_messages} сообщений, limit=50, прогрев ${settings.warmup_seconds} с и чтение ${settings.duration_seconds} с, ${settings.writes_per_repeat} записей в новый чат. RPS = общее число успешных запросов / общее время повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее отдельных p95, не перцентиль объединённой выборки. SQL/HTTP исключает команды транзакций.`,
|
||||
'',
|
||||
'| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | Медиана RPS | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const [workload, label] of [
|
||||
['cold-history-burst', 'Холодная волна истории'],
|
||||
['warm-history', 'Прогретая история'],
|
||||
['history-cache-bypassed', 'Контроль без кэша истории'],
|
||||
['writes-single-chat', 'Запись в один чат'],
|
||||
]) {
|
||||
if (
|
||||
reports.every(
|
||||
(report) =>
|
||||
!report.samples.some((sample) => sample.workload === workload),
|
||||
)
|
||||
)
|
||||
continue;
|
||||
for (const [index, report] of reports.entries()) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.workload === workload,
|
||||
);
|
||||
assert.equal(samples.length, settings.repetitions);
|
||||
const sum = (select) =>
|
||||
samples.reduce((total, sample) => total + select(sample), 0);
|
||||
const successes = sum((sample) => sample.successes);
|
||||
const errors = sum((sample) => sample.errors);
|
||||
const aggregate = {
|
||||
workload,
|
||||
variant: labels[index],
|
||||
successes,
|
||||
errors,
|
||||
mean_rps: successes / sum((sample) => sample.elapsed_seconds),
|
||||
mean_latency_ms:
|
||||
sum((sample) => sample.latency_ms.mean * sample.successes) / successes,
|
||||
mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length,
|
||||
sql_per_request:
|
||||
sum((sample) => sample.sql_queries) / (successes + errors),
|
||||
median_rps: median(samples.map((sample) => sample.successful_rps)),
|
||||
repeat_rps: samples.map((sample) => sample.successful_rps),
|
||||
};
|
||||
aggregates.push(aggregate);
|
||||
lines.push(
|
||||
`| ${label} | ${labels[index]} | ${aggregate.mean_rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.median_rps.toFixed(3)} | ${errors} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
`Учтено успешных HTTP-запросов: ${reports.map((report, index) => `${labels[index]} — ${report.samples.reduce((sum, sample) => sum + sample.successes, 0)}`).join('; ')}.`,
|
||||
'',
|
||||
...['warm-history', 'writes-single-chat'].map((workload) => {
|
||||
const pair = aggregates.filter(
|
||||
(aggregate) => aggregate.workload === workload,
|
||||
);
|
||||
return `${workload}: отношение среднего RPS полной стратегии к варианту без стратегии — ${(pair[0].mean_rps / pair[1].mean_rps).toFixed(3)}. RPS по повторам: ${pair.map((aggregate) => `${aggregate.variant}: ${aggregate.repeat_rps.join(' / ')}`).join('; ')}.`;
|
||||
}),
|
||||
'',
|
||||
settings.history_control === false
|
||||
? 'Дополнительный контроль с частично отключённым кэшем в этом прогоне пропущен. Сравниваются только полная стратегия и отсутствие Redis/поддержания эпох.'
|
||||
: 'В контрольном сценарии у первого API отключён только кэш истории; кэш профиля продолжает работать. У второго API отключены все кэши. Поэтому этот контроль не измеряет отдельно чистую стоимость эпох.',
|
||||
'',
|
||||
`Проверки полной стратегии: чтение после записи при отказе Redis, чтение другой репликой после восстановления со старыми ключами, восстановление с пустым кэшем, задержанная загрузка старой истории. Проверки варианта без стратегии: приложение не создало Redis-клиент, эпохи синтетических пользователей/чатов остались равны 0 после всех записей, ${settings.writes_per_repeat} сообщений записаны в каждом новом чате, история читается после записи без повторяющихся ID.`,
|
||||
'',
|
||||
`Оба прогона используют один базовый образ; изменённые модули временной версии скомпилированы TypeScript без полной пересборки контейнера и подключены отдельными bind mounts. Хеши исходников и JS сохранены в [метаданных эксперимента](${resultPrefix}-experiment.json). Колонки эпох остаются в схеме; это исключает миграцию и позволяет сохранить рабочую БД. SQL-проверки профиля и членства во втором варианте не используют эпохи. Временные файлы удалены, хеши рабочих исходников проверены.`,
|
||||
'',
|
||||
'Ограничения: прогоны последовательные, Docker-host разделяется с другими сервисами; большой разброс ограничивает выводы о скорости. CPU/RSS процесса включают API, HTTP-генератор и инструментацию; CPU PostgreSQL отдельно не измерен. Синтетические сессии обходят KDF. Холодная волна означает пустой кэш приложения, а не холодный PostgreSQL. Отдельные цены Redis и эпох этим сравнением не разделяются: меняется вся стратегия. Данные рабочей БД и volumes сохранены, удалены только фикстуры эксперимента.',
|
||||
'',
|
||||
'Повторение:',
|
||||
'',
|
||||
'```powershell',
|
||||
`docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/${resultPrefix}-full.json --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
|
||||
`$env:EXPERIMENT_PREFIX='${resultPrefix}'`,
|
||||
`node test/no-epochs-experiment.cjs --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
|
||||
`node test/cache-strategy-summary.mjs ${resultPrefix}`,
|
||||
'```',
|
||||
'',
|
||||
`Сырые результаты: [полная стратегия](${resultPrefix}-full.json), [без Redis и эпох](${resultPrefix}-no-redis-no-epochs.json).`,
|
||||
);
|
||||
await writeFile(`evidence/${resultPrefix}-summary.md`, lines.join('\n') + '\n');
|
||||
await writeFile(
|
||||
`evidence/${resultPrefix}-averages.json`,
|
||||
JSON.stringify(aggregates, null, 2) + '\n',
|
||||
);
|
||||
console.log(lines.slice(0, 18).join('\n'));
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { CacheService } = require('../dist/cache/cache.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
async function isolatedCache(t) {
|
||||
assert.ok(process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance');
|
||||
const previous = process.env.CACHE_PREFIX;
|
||||
const prefix = `test:cache:${randomUUID()}`;
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
const observer = new Redis(process.env.CACHE_URL, {
|
||||
maxRetriesPerRequest: 0,
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 1000,
|
||||
});
|
||||
observer.on('error', () => {});
|
||||
await observer.ping();
|
||||
t.after(async () => {
|
||||
try {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
observer.disconnect();
|
||||
if (previous === undefined) delete process.env.CACHE_PREFIX;
|
||||
else process.env.CACHE_PREFIX = previous;
|
||||
}
|
||||
});
|
||||
return { prefix, observer };
|
||||
}
|
||||
|
||||
test('Redis cache: hits, expiry, invalidation race and unavailable-server fallback', async (t) => {
|
||||
await isolatedCache(t);
|
||||
const cache = new CacheService();
|
||||
t.after(() => cache.onModuleDestroy());
|
||||
await cache.onModuleInit();
|
||||
let reads = 0;
|
||||
const load = async () => ({ value: ++reads });
|
||||
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
|
||||
value: 1,
|
||||
});
|
||||
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
|
||||
value: 1,
|
||||
});
|
||||
assert.equal(reads, 1);
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 2,
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 1100));
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 3,
|
||||
});
|
||||
await cache.invalidate('expiry');
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 4,
|
||||
});
|
||||
|
||||
let resolveOld;
|
||||
let began;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const pending = cache.remember('race', 'page', 30, () => {
|
||||
began();
|
||||
return new Promise((resolve) => {
|
||||
resolveOld = resolve;
|
||||
});
|
||||
});
|
||||
await started;
|
||||
await cache.invalidate('race');
|
||||
resolveOld({ value: 'old' });
|
||||
await pending;
|
||||
assert.deepEqual(
|
||||
await cache.remember('race', 'page', 30, async () => ({ value: 'new' })),
|
||||
{ value: 'new' },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await cache.remember('race', 'page', 30, async () => {
|
||||
throw new Error('cache miss');
|
||||
}),
|
||||
{ value: 'new' },
|
||||
);
|
||||
await assert.rejects(
|
||||
cache.remember('db-error', 'page', 30, async () => {
|
||||
throw new Error('database failure');
|
||||
}),
|
||||
/database failure/,
|
||||
);
|
||||
|
||||
const previous = process.env.CACHE_URL;
|
||||
process.env.CACHE_URL = 'redis://127.0.0.1:1/0';
|
||||
const offline = new CacheService();
|
||||
process.env.CACHE_URL = previous;
|
||||
t.after(() => offline.onModuleDestroy());
|
||||
await offline.onModuleInit();
|
||||
assert.deepEqual(
|
||||
await offline.remember('offline', 'page', 30, async () => ({
|
||||
value: 'database',
|
||||
})),
|
||||
{ value: 'database' },
|
||||
);
|
||||
await offline.invalidate('offline');
|
||||
});
|
||||
|
||||
test('API cache: fewer SQL reads, write invalidation, permissions, logout and Redis fallback', async (t) => {
|
||||
assert.ok(process.env.DATABASE_URL);
|
||||
const { prefix, observer } = await isolatedCache(t);
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
const prisma = app.get(PrismaService);
|
||||
const userIds = [];
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
});
|
||||
const actors = [];
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `cache_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Cache test',
|
||||
},
|
||||
});
|
||||
userIds.push(user.id);
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const session = await prisma.session.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
actors.push({ id: user.id, token, sessionId: session.id });
|
||||
}
|
||||
const [alice, outsider] = actors;
|
||||
const request = (method, url, payload, actor = alice) =>
|
||||
server.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
authorization: `Bearer ${actor.token}`,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
});
|
||||
const counts = new Map();
|
||||
const tracked = prisma.$extends({
|
||||
query: {
|
||||
$allModels: {
|
||||
$allOperations({ model, operation, args, query }) {
|
||||
const key = `${model}:${operation}`;
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
for (const name of [
|
||||
'User/user.service',
|
||||
'Chat/chat.service',
|
||||
'Auth/auth.service',
|
||||
]) {
|
||||
const exported = require(`../dist/${name}`);
|
||||
const Service = Object.values(exported)[0];
|
||||
app.get(Service).prisma = tracked;
|
||||
}
|
||||
const spy = (model, operation) => ({
|
||||
mock: { callCount: () => counts.get(`${model}:${operation}`) ?? 0 },
|
||||
});
|
||||
const profileReads = spy('User', 'findUnique');
|
||||
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
|
||||
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
|
||||
assert.equal(profileReads.mock.callCount(), 0);
|
||||
|
||||
const chatListReads = spy('Chat', 'findMany');
|
||||
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
|
||||
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
|
||||
assert.equal(chatListReads.mock.callCount(), 1);
|
||||
const created = await request('POST', '/api/v1/chats', {
|
||||
title: 'Cached chat',
|
||||
member_ids: [alice.id],
|
||||
});
|
||||
assert.equal(created.statusCode, 201);
|
||||
const chatId = created.json().id;
|
||||
chatIds.push(chatId);
|
||||
assert.deepEqual(
|
||||
(await request('GET', '/api/v1/chats')).json().items.map((x) => x.id),
|
||||
[chatId],
|
||||
);
|
||||
assert.equal(chatListReads.mock.callCount(), 2);
|
||||
|
||||
const metadataReads = spy('Chat', 'findUnique');
|
||||
const messageReads = spy('Message', 'findMany');
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal((await request('GET', path)).json().items.length, 0);
|
||||
assert.equal((await request('GET', path)).json().items.length, 0);
|
||||
assert.equal(metadataReads.mock.callCount(), 0);
|
||||
assert.equal(messageReads.mock.callCount(), 1);
|
||||
const message = await request('POST', path, {
|
||||
text: 'Invalidate cached empty history',
|
||||
});
|
||||
assert.equal(message.statusCode, 201);
|
||||
assert.deepEqual(
|
||||
(await request('GET', path)).json().items.map((x) => x.id),
|
||||
[message.json().id],
|
||||
);
|
||||
assert.equal(messageReads.mock.callCount(), 2);
|
||||
assert.equal(
|
||||
(await request('GET', path, undefined, outsider)).statusCode,
|
||||
404,
|
||||
);
|
||||
await prisma.chatMember.delete({
|
||||
where: { chatId_userId: { chatId, userId: alice.id } },
|
||||
});
|
||||
assert.equal((await request('GET', path)).statusCode, 404);
|
||||
await prisma.chatMember.create({
|
||||
data: { chatId, userId: alice.id, position: 0 },
|
||||
});
|
||||
await prisma.session.update({
|
||||
where: { id: outsider.sessionId },
|
||||
data: { revokedAt: new Date() },
|
||||
});
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me', undefined, outsider)).statusCode,
|
||||
401,
|
||||
);
|
||||
|
||||
const keys = [];
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, batch] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
keys.push(...batch);
|
||||
} while (cursor !== '0');
|
||||
assert.ok(keys.length >= 4, 'Redis must contain the cached responses');
|
||||
for (const key of keys) {
|
||||
const value = await observer.get(key);
|
||||
assert.ok(
|
||||
!value.includes('password_hash') && !value.includes('passwordHash'),
|
||||
);
|
||||
for (const actor of actors) assert.ok(!value.includes(actor.token));
|
||||
}
|
||||
app.get(CacheService).onModuleDestroy();
|
||||
assert.equal((await request('GET', path)).statusCode, 200);
|
||||
assert.equal(messageReads.mock.callCount(), 3);
|
||||
assert.equal(
|
||||
(await request('POST', path, { text: 'Write while Redis is unavailable' }))
|
||||
.statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal((await request('GET', path)).json().items.length, 2);
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the lab3 migration while a lab2 API uses an isolated temporary database."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
from urllib.parse import quote
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NETWORK = 'mtusi2026lab3-fork_default'
|
||||
OLD_API = 'mtusi2026lab2-fork-backend:latest'
|
||||
OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest'
|
||||
NEW_API = 'mtusi2026lab3-fork-api:latest'
|
||||
NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def request(base, method, path, body=None, token=None, basic=None, key=None):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
if basic:
|
||||
headers['Authorization'] = 'Basic ' + base64.b64encode(
|
||||
(basic[0] + ':' + basic[1]).encode()).decode()
|
||||
if key:
|
||||
headers['Idempotency-Key'] = key
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
headers['Content-Type'] = 'application/json'
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers,
|
||||
method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=3) as response:
|
||||
raw = response.read()
|
||||
return response.status, json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None
|
||||
|
||||
|
||||
def wait_http(base):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if request(base, 'GET', '/health/ready')[0] == 200:
|
||||
return
|
||||
except RuntimeError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError('temporary API did not become ready')
|
||||
|
||||
|
||||
def publish_port(name):
|
||||
value = docker('port', name, '8080/tcp')
|
||||
return 'http://127.0.0.1:' + value.rsplit(':', 1)[1]
|
||||
|
||||
|
||||
def make_user(base, label):
|
||||
username = 'compat_' + label + '_' + uuid.uuid4().hex[:8]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, user = request(base, 'POST', '/api/v1/users',
|
||||
{'username': username, 'display_name': label,
|
||||
'password': password})
|
||||
assert status == 201
|
||||
status, session = request(base, 'POST', '/api/v1/auth/sessions',
|
||||
basic=(username, password))
|
||||
assert status == 201
|
||||
return user, session['token']
|
||||
|
||||
|
||||
def message(base, token, chat_id, text, key=None):
|
||||
status, item = request(base, 'POST',
|
||||
f'/api/v1/chats/{chat_id}/messages',
|
||||
{'text': text}, token=token, key=key)
|
||||
assert status == 201
|
||||
return item
|
||||
|
||||
|
||||
def history(base, token, chat_id):
|
||||
items = []
|
||||
cursor = None
|
||||
for _ in range(100):
|
||||
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
|
||||
if cursor:
|
||||
path += '&cursor=' + quote(cursor, safe='')
|
||||
status, page = request(base, 'GET', path, token=token)
|
||||
assert status == 200
|
||||
items.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return items
|
||||
raise RuntimeError('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def run():
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
names = {label: 'lab3-compat-' + suffix + '-' + label
|
||||
for label in ('pg', 'redis', 'old', 'new')}
|
||||
password = secrets.token_urlsafe(18)
|
||||
db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat'
|
||||
cache_url = f'redis://{names["redis"]}:6379/0'
|
||||
created = []
|
||||
try:
|
||||
print('Starting isolated PostgreSQL and Redis...', flush=True)
|
||||
docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK,
|
||||
'--tmpfs', '/var/lib/postgresql/data:rw,mode=0700',
|
||||
'-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app',
|
||||
'-e', 'POSTGRES_PASSWORD=' + password,
|
||||
'postgres:17.6-bookworm')
|
||||
created.append(names['pg'])
|
||||
docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK,
|
||||
'redis:7.4.5-alpine', 'redis-server', '--save', '',
|
||||
'--appendonly', 'no')
|
||||
created.append(names['redis'])
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
docker('exec', names['pg'], 'pg_isready', '-U', 'app',
|
||||
'-d', 'compat', timeout=5)
|
||||
break
|
||||
except RuntimeError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('temporary PostgreSQL did not start')
|
||||
time.sleep(0.3)
|
||||
docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url,
|
||||
OLD_MIGRATE, timeout=90)
|
||||
print('Lab2 schema applied.', flush=True)
|
||||
before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert before_table == 'f', 'new table exists before migration'
|
||||
base_env = ['-e', 'DATABASE_URL=' + db_url,
|
||||
'-e', 'CACHE_URL=' + cache_url,
|
||||
'-e', 'CACHE_PREFIX=compat:' + suffix,
|
||||
'-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080',
|
||||
'-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres']
|
||||
docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env, OLD_API)
|
||||
created.append(names['old'])
|
||||
old_base = publish_port(names['old'])
|
||||
wait_http(old_base)
|
||||
print('Lab2 API ready; preparing synthetic chat.', flush=True)
|
||||
alice, token = make_user(old_base, 'alice')
|
||||
bob, _ = make_user(old_base, 'bob')
|
||||
status, chat = request(old_base, 'POST', '/api/v1/chats',
|
||||
{'title': 'Migration compatibility',
|
||||
'member_ids': [alice['id'], bob['id']]},
|
||||
token=token)
|
||||
assert status == 201
|
||||
pre = message(old_base, token, chat['id'], 'before migration')
|
||||
|
||||
stop = threading.Event()
|
||||
events = []
|
||||
|
||||
def old_writer():
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
at = time.monotonic()
|
||||
try:
|
||||
item = message(old_base, token, chat['id'],
|
||||
f'during migration {index}')
|
||||
events.append((at, True, item['id']))
|
||||
except (RuntimeError, AssertionError):
|
||||
events.append((at, False, None))
|
||||
index += 1
|
||||
stop.wait(0.3)
|
||||
|
||||
thread = threading.Thread(target=old_writer, daemon=True)
|
||||
thread.start()
|
||||
time.sleep(0.5)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
print('Applying lab3 migration while lab2 API writes...', flush=True)
|
||||
docker('run', '--rm', '--network', NETWORK,
|
||||
'-e', 'DATABASE_URL=' + db_url,
|
||||
NEW_MIGRATE, timeout=90)
|
||||
finally:
|
||||
migrated = time.monotonic()
|
||||
time.sleep(0.7)
|
||||
stop.set()
|
||||
thread.join(timeout=5)
|
||||
after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert after_table == 't', 'new table was not created'
|
||||
assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration'
|
||||
assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration'
|
||||
assert all(ok for _, ok, _ in events), 'old API write failed near migration'
|
||||
after = message(old_base, token, chat['id'], 'after migration')
|
||||
print('Lab2 API still reads and writes; starting lab3 API.', flush=True)
|
||||
|
||||
docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env,
|
||||
'-e', 'INSTANCE_ID=compat-new', NEW_API)
|
||||
created.append(names['new'])
|
||||
new_base = publish_port(names['new'])
|
||||
wait_http(new_base)
|
||||
print('Lab3 API ready; checking mixed-version reads and writes.', flush=True)
|
||||
status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token)
|
||||
assert status == 200 and profile['id'] == alice['id'], 'session not shared'
|
||||
key = uuid.uuid4().hex
|
||||
new_message = message(new_base, token, chat['id'], 'from new version', key)
|
||||
replay = message(new_base, token, chat['id'], 'from new version', key)
|
||||
assert replay == new_message, 'new API idempotent replay changed response'
|
||||
old_after_new = message(old_base, token, chat['id'], 'from old version')
|
||||
for version, base in (('old', old_base), ('new', new_base)):
|
||||
items = history(base, token, chat['id'])
|
||||
ids = [item['id'] for item in items]
|
||||
for label, item in (('pre', pre), ('after', after),
|
||||
('new', new_message), ('old-after-new', old_after_new)):
|
||||
assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message'
|
||||
result = {
|
||||
'old_image': OLD_API, 'new_image': NEW_API,
|
||||
'migration_image': NEW_MIGRATE,
|
||||
'migration_seconds': round(migrated - started, 3),
|
||||
'old_writes_near_migration': len(events),
|
||||
'old_write_errors': sum(not ok for _, ok, _ in events),
|
||||
'old_and_new_read_each_others_messages': True,
|
||||
'existing_session_accepted_by_new_api': True,
|
||||
'new_api_idempotent_replay': True,
|
||||
'temporary_database': True,
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
print('PASS: additive migration compatible with running lab2 API')
|
||||
finally:
|
||||
print('Stopping isolated test containers...', flush=True)
|
||||
for name in reversed(created):
|
||||
try:
|
||||
docker('stop', name, timeout=30)
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
print(f'WARNING: stop temporary container {name} manually',
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
run()
|
||||
except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROBE = r'''
|
||||
import json, sys, threading, time, urllib.error, urllib.request
|
||||
url = sys.argv[1]
|
||||
live_url = url.replace('/health/ready', '/health/live')
|
||||
phase = 'before'
|
||||
events = []
|
||||
def commands():
|
||||
global phase
|
||||
for line in sys.stdin:
|
||||
phase = line.strip()
|
||||
if phase == 'end': break
|
||||
def status(address):
|
||||
try:
|
||||
with urllib.request.urlopen(address, timeout=1) as response:
|
||||
return str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
return str(error.code)
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return 'connection_error'
|
||||
threading.Thread(target=commands, daemon=True).start()
|
||||
print('READY', flush=True)
|
||||
while phase != 'end':
|
||||
current = phase
|
||||
ready = status(url)
|
||||
live = status(live_url)
|
||||
events.append({'phase': current, 'ready': ready, 'live': live})
|
||||
time.sleep(0.05)
|
||||
print(json.dumps(events), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def command(*args, timeout=45):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def both_ready(base_url, expected):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response:
|
||||
seen.add(response.headers['X-Instance-Id'])
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if seen == expected:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError(f'both replicas did not recover: {seen}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
args = parser.parse_args()
|
||||
ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines()
|
||||
if len(ids) != 2:
|
||||
raise RuntimeError('shutdown probe requires two API replicas')
|
||||
expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids}
|
||||
target = ids[0]
|
||||
networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target))
|
||||
network, settings = next(iter(networks.items()))
|
||||
address = settings['IPAddress']
|
||||
nginx = command('docker', 'compose', 'ps', '-q', 'nginx')
|
||||
both_ready(args.base_url, expected)
|
||||
probe = subprocess.Popen(
|
||||
['docker', 'run', '--rm', '-i', '--network', network,
|
||||
'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE,
|
||||
f'http://{address}:8080/health/ready'],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
stop = None
|
||||
|
||||
def phase(value):
|
||||
probe.stdin.write(value + '\n')
|
||||
probe.stdin.flush()
|
||||
|
||||
try:
|
||||
if probe.stdout.readline().strip() != 'READY':
|
||||
raise RuntimeError('direct readiness probe did not start')
|
||||
time.sleep(1)
|
||||
phase('stopping')
|
||||
started = time.monotonic()
|
||||
stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
withdrawn_at = None
|
||||
withdrawn_while_running = False
|
||||
while stop.poll() is None and time.monotonic() - started < 35:
|
||||
upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10)
|
||||
if f'server {address}:8080 ' not in upstreams:
|
||||
withdrawn_at = time.monotonic() - started
|
||||
withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true'
|
||||
if withdrawn_while_running:
|
||||
phase('withdrawn')
|
||||
break
|
||||
time.sleep(0.1)
|
||||
stop.communicate(timeout=40)
|
||||
if stop.returncode:
|
||||
raise RuntimeError('docker stop failed')
|
||||
stop_seconds = time.monotonic() - started
|
||||
exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target))
|
||||
phase('stopped')
|
||||
time.sleep(1)
|
||||
phase('end')
|
||||
output, _ = probe.communicate(timeout=15)
|
||||
if probe.returncode:
|
||||
raise RuntimeError('direct probe failed')
|
||||
events = json.loads(output.strip())
|
||||
summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events)
|
||||
for value in sorted({e['ready'] for e in events if e['phase'] == name})}
|
||||
for name in ('before', 'stopping', 'withdrawn', 'stopped')}
|
||||
draining = [e for e in events if e['ready'] == '503']
|
||||
first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None)
|
||||
if summary['before'].get('200', 0) < 5:
|
||||
raise RuntimeError('readiness was not healthy before SIGTERM')
|
||||
if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]):
|
||||
raise RuntimeError('expected readiness 200 -> 503 with no late 200')
|
||||
# These are separate requests; transport failures can straddle HTTP
|
||||
# close or the client's deadline. Do not confuse them with unhealthy
|
||||
# HTTP responses. Record them separately rather than hiding them.
|
||||
if not draining or not any(e['live'] == '200' for e in draining):
|
||||
raise RuntimeError('liveness must remain 200 while readiness is 503')
|
||||
if any(e['live'] not in {'200', 'connection_error'} for e in draining):
|
||||
raise RuntimeError('liveness returned an unhealthy HTTP status during draining')
|
||||
if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events):
|
||||
raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive')
|
||||
if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0):
|
||||
raise RuntimeError('replica did not close HTTP after draining')
|
||||
if exit_code == 137:
|
||||
raise RuntimeError('container required SIGKILL instead of graceful shutdown')
|
||||
print(json.dumps({'stop_seconds': round(stop_seconds, 3),
|
||||
'lb_withdrawn_seconds': round(withdrawn_at, 3),
|
||||
'lb_withdrawn_before_http_close': True,
|
||||
'readiness_sequence': [200, 503, 'connection_error'],
|
||||
'readiness_503_samples': len(draining),
|
||||
'liveness_200_while_draining': sum(e['live'] == '200' for e in draining),
|
||||
'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining),
|
||||
'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2))
|
||||
print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close')
|
||||
finally:
|
||||
if probe.poll() is None:
|
||||
probe.kill()
|
||||
probe.communicate()
|
||||
if stop is not None and stop.poll() is None:
|
||||
stop.communicate(timeout=40)
|
||||
command('docker', 'start', target)
|
||||
both_ready(args.base_url, expected)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,12 @@
|
||||
services:
|
||||
benchmark:
|
||||
image: ${BENCHMARK_IMAGE:-mtusi2026lab2-fork-backend}
|
||||
working_dir: /app
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
volumes:
|
||||
- ./test:/app/test:ro
|
||||
- ./evidence:/results
|
||||
command: ['node', 'test/cache-benchmark.mjs', '${BENCHMARK_STAGE:-after}', '/results/cache-epochs-${BENCHMARK_STAGE:-after}.json']
|
||||
@@ -0,0 +1,5 @@
|
||||
services:
|
||||
api:
|
||||
command: ['sh', '-c', 'INSTANCE_ID=$$(hostname) exec node --require /app/diagnostics/pool.cjs dist/main.js']
|
||||
volumes:
|
||||
- ./scripts/diagnostics/pool.cjs:/app/diagnostics/pool.cjs:ro
|
||||
@@ -0,0 +1,35 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
import tempfile
|
||||
|
||||
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).resolve().parents[1] / "scripts/run-scaling-diagnostics.py")
|
||||
diagnostics = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diagnostics)
|
||||
|
||||
|
||||
class SummaryTest(unittest.TestCase):
|
||||
def test_cumulative_counters_become_interval_rates(self):
|
||||
summary = diagnostics.resource_summary([
|
||||
{"time_ms": 1000, "cpu_usage_usec": 100, "cpu_nr_periods": 10,
|
||||
"cpu_nr_throttled": 1, "cpu_throttled_usec": 100, "memory_bytes": 1024**2},
|
||||
{"time_ms": 3000, "cpu_usage_usec": 1000100, "cpu_nr_periods": 30,
|
||||
"cpu_nr_throttled": 6, "cpu_throttled_usec": 500100, "memory_bytes": 2*1024**2},
|
||||
])
|
||||
self.assertEqual(summary["mean_cpu_cores"], 0.5)
|
||||
self.assertEqual(summary["throttled_period_fraction"], 0.25)
|
||||
self.assertEqual(summary["throttled_seconds"], 0.5)
|
||||
self.assertEqual(summary["peak_memory_mib"], 2)
|
||||
|
||||
def test_short_sample_does_not_invent_rates(self):
|
||||
self.assertEqual(diagnostics.resource_summary([]), {"samples": 0})
|
||||
|
||||
def test_postgres_multiline_json_and_incomplete_tail(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "postgres.jsonl"
|
||||
path.write_text('{"kind":"postgres",\n"activity":[\n{"state":"active"}\n]}\n{"kind":"cgroup"}\n{"unfinished":', encoding="utf-8")
|
||||
self.assertEqual([r["kind"] for r in diagnostics.rows(path)], ["postgres", "cgroup"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,88 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { setImmediate, setTimeout } from 'node:timers/promises';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { Module } = require('@nestjs/common');
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppController } = require('../dist/app.controller');
|
||||
const { AppService } = require('../dist/app.service');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { createFastifyAdapter } = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
async function fixture(t, query = async () => []) {
|
||||
const state = { queries: 0, destroyed: false };
|
||||
const database = {
|
||||
async $queryRaw() {
|
||||
state.queries += 1;
|
||||
return query();
|
||||
},
|
||||
onModuleDestroy() {
|
||||
state.destroyed = true;
|
||||
},
|
||||
};
|
||||
class FixtureModule {}
|
||||
Module({
|
||||
controllers: [AppController],
|
||||
providers: [AppService, { provide: PrismaService, useValue: database }],
|
||||
})(FixtureModule);
|
||||
const adapter = createFastifyAdapter(30);
|
||||
const app = await NestFactory.create(FixtureModule, adapter, { logger: false });
|
||||
t.after(async () => {
|
||||
state.release?.();
|
||||
await app.close();
|
||||
});
|
||||
return { app, adapter, state, controller: app.get(AppController) };
|
||||
}
|
||||
|
||||
test('draining keeps liveness and dependencies alive until active HTTP work completes', { timeout: 30000 }, async (t) => {
|
||||
const { app, adapter, state, controller } = await fixture(t);
|
||||
const entered = Promise.withResolvers();
|
||||
const release = Promise.withResolvers();
|
||||
// Always release the request, including when an assertion fails.
|
||||
state.release = () => release.resolve();
|
||||
adapter.getInstance().get('/slow-work', async () => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
assert.equal(state.destroyed, false, 'database was destroyed during active work');
|
||||
return { completed: true };
|
||||
});
|
||||
await app.listen(0, '127.0.0.1');
|
||||
t.diagnostic('HTTP listener started');
|
||||
const { port } = app.getHttpServer().address();
|
||||
await controller.ready();
|
||||
const response = fetch(`http://127.0.0.1:${port}/slow-work`, { signal: AbortSignal.timeout(5000) });
|
||||
await entered.promise;
|
||||
t.diagnostic('Slow request started');
|
||||
let closed = false;
|
||||
const closing = app.close('SIGTERM').then(() => { closed = true; });
|
||||
await setImmediate();
|
||||
assert.equal(adapter.isDraining, true);
|
||||
assert.equal(adapter.beforeClose(), adapter.beforeClose(), 'shutdown must be idempotent');
|
||||
await assert.rejects(controller.ready(), (error) => error.getStatus() === 503);
|
||||
assert.equal(state.queries, 1, 'draining readiness must not acquire a DB connection');
|
||||
assert.deepEqual(controller.live(), { status: 'ok' });
|
||||
await setTimeout(80);
|
||||
assert.equal(closed, false, 'server closed before the active request completed');
|
||||
assert.equal(state.destroyed, false, 'dependencies closed before the active request');
|
||||
release.resolve();
|
||||
t.diagnostic('Slow request released');
|
||||
assert.deepEqual(await (await response).json(), { completed: true });
|
||||
await closing;
|
||||
t.diagnostic('Application shutdown completed');
|
||||
assert.equal(state.destroyed, true);
|
||||
});
|
||||
|
||||
test('readiness already querying the DB cannot return 200 after draining begins', { timeout: 10000 }, async (t) => {
|
||||
const query = Promise.withResolvers();
|
||||
const { app, adapter, controller } = await fixture(t, () => query.promise);
|
||||
await app.init();
|
||||
const pending = assert.rejects(controller.ready(), (error) => error.getStatus() === 503);
|
||||
const closing = app.close('SIGTERM');
|
||||
await setImmediate();
|
||||
assert.equal(adapter.isDraining, true);
|
||||
query.resolve([]);
|
||||
await pending;
|
||||
await closing;
|
||||
});
|
||||
@@ -0,0 +1,475 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import Module from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
|
||||
const require = createRequire('/app/package.json');
|
||||
const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8'));
|
||||
const pg = require('pg');
|
||||
let counts = {};
|
||||
const originalQuery = pg.Client.prototype.query;
|
||||
pg.Client.prototype.query = function (query, ...args) {
|
||||
const sql = typeof query === 'string' ? query : query.text;
|
||||
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
|
||||
counts[verb] = (counts[verb] ?? 0) + 1;
|
||||
return originalQuery.call(this, query, ...args);
|
||||
};
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('/app/dist/app.module');
|
||||
const { PrismaService } = require('/app/dist/prisma/prisma.service');
|
||||
const { CacheService } = require('/app/dist/cache/cache.service');
|
||||
const { ChatService } = require('/app/dist/Chat/chat.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('/app/dist/common/http/fastify-adapter');
|
||||
const exportedClasses = {
|
||||
'Auth/auth.service.ts': 'AuthService',
|
||||
'User/user.service.ts': 'UserService',
|
||||
'User/user.guard.ts': 'UserGuard',
|
||||
'Chat/chat.service.ts': 'ChatService',
|
||||
'Chat/chat.controller.ts': 'ChatController',
|
||||
};
|
||||
const originals = {};
|
||||
const variants = {};
|
||||
for (const [relative, name] of Object.entries(exportedClasses)) {
|
||||
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
|
||||
originals[name] = require(filename)[name];
|
||||
}
|
||||
for (const variant of manifest.variants) {
|
||||
variants[variant.name] = {};
|
||||
for (const [relative, name] of Object.entries(exportedClasses)) {
|
||||
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
|
||||
const source = await readFile(
|
||||
`/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`,
|
||||
'utf8',
|
||||
);
|
||||
const loaded = new Module(filename);
|
||||
loaded.filename = filename;
|
||||
loaded.paths = Module._nodeModulePaths('/app');
|
||||
loaded._compile(source, filename);
|
||||
variants[variant.name][name] = loaded.exports[name];
|
||||
}
|
||||
}
|
||||
function activate(name) {
|
||||
for (const [className, replacement] of Object.entries(variants[name])) {
|
||||
const target = originals[className].prototype;
|
||||
for (const method of Object.getOwnPropertyNames(replacement.prototype)) {
|
||||
if (method === 'constructor') continue;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(
|
||||
replacement.prototype,
|
||||
method,
|
||||
);
|
||||
const previous = target[method];
|
||||
if (
|
||||
typeof descriptor.value === 'function' &&
|
||||
typeof previous === 'function'
|
||||
) {
|
||||
for (const key of Reflect.getMetadataKeys(previous))
|
||||
Reflect.defineMetadata(
|
||||
key,
|
||||
Reflect.getMetadata(key, previous),
|
||||
descriptor.value,
|
||||
);
|
||||
}
|
||||
Object.defineProperty(target, method, descriptor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Redis = require('ioredis').default;
|
||||
const prefix = `experiment:epoch-variants:${randomUUID()}`;
|
||||
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
|
||||
const observer = new Redis(process.env.CACHE_URL);
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
const apps = {};
|
||||
const chatIds = [];
|
||||
const userIds = [];
|
||||
let prisma, token;
|
||||
const report = {
|
||||
recorded_at: new Date().toISOString(),
|
||||
methodology: {
|
||||
concurrency: 32,
|
||||
duration_seconds: 60,
|
||||
warmup_seconds: 10,
|
||||
repetitions: 3,
|
||||
seed_messages: 100000,
|
||||
writes_per_repeat: 4096,
|
||||
order: [
|
||||
['baseline', 'profile-join', 'access-join', 'striped'],
|
||||
['striped', 'access-join', 'profile-join', 'baseline'],
|
||||
['profile-join', 'striped', 'baseline', 'access-join'],
|
||||
],
|
||||
requests:
|
||||
'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes',
|
||||
resources:
|
||||
'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time',
|
||||
sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size',
|
||||
},
|
||||
variants: manifest.variants,
|
||||
samples: [],
|
||||
correctness: {},
|
||||
completed: false,
|
||||
};
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function request(name, method, url, body, bearer = token) {
|
||||
const response = await fetch(`${apps[name].url}${url}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${bearer}`,
|
||||
...(body ? { 'content-type': 'application/json' } : {}),
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
const json = await response.json();
|
||||
return { status: response.status, json };
|
||||
}
|
||||
async function call(name, method, url, body) {
|
||||
const result = await request(name, method, url, body);
|
||||
assert.equal(result.status, method === 'POST' ? 201 : 200);
|
||||
return result.json;
|
||||
}
|
||||
async function clearCache() {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
}
|
||||
async function newChat(userId) {
|
||||
const chat = await prisma.chat.create({
|
||||
data: {
|
||||
title: 'Variant benchmark',
|
||||
members: { create: { userId, position: 0 } },
|
||||
},
|
||||
});
|
||||
chatIds.push(chat.id);
|
||||
return chat.id;
|
||||
}
|
||||
async function epoch(name, chatId) {
|
||||
if (name !== 'striped')
|
||||
return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } }))
|
||||
.messagesEpoch;
|
||||
const [row] = await prisma.$queryRawUnsafe(
|
||||
`SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`,
|
||||
chatId,
|
||||
);
|
||||
return row.epoch;
|
||||
}
|
||||
async function measure(name, workload, repeat, operation, total) {
|
||||
counts = {};
|
||||
const latencies = [];
|
||||
let errors = 0,
|
||||
index = 0;
|
||||
const cpu = process.cpuUsage();
|
||||
const start = performance.now();
|
||||
await Promise.all(
|
||||
Array.from({ length: 32 }, async () => {
|
||||
while (
|
||||
total === undefined ? performance.now() - start < 60000 : index < total
|
||||
) {
|
||||
const item = index++;
|
||||
const began = performance.now();
|
||||
try {
|
||||
await operation(item);
|
||||
latencies.push(performance.now() - began);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
const seconds = (performance.now() - start) / 1000;
|
||||
const used = process.cpuUsage(cpu);
|
||||
latencies.sort((a, b) => a - b);
|
||||
const calls = { ...counts };
|
||||
const queries = Object.entries(calls)
|
||||
.filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb))
|
||||
.reduce((sum, [, count]) => sum + count, 0);
|
||||
const sample = {
|
||||
variant: name,
|
||||
workload,
|
||||
repeat,
|
||||
successes: latencies.length,
|
||||
errors,
|
||||
elapsed_seconds: +seconds.toFixed(3),
|
||||
rps: +(latencies.length / seconds).toFixed(3),
|
||||
mean_latency_ms: +(
|
||||
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
|
||||
).toFixed(3),
|
||||
p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3),
|
||||
sql_queries: queries,
|
||||
sql_per_request: +(queries / (latencies.length + errors)).toFixed(4),
|
||||
sql_calls: calls,
|
||||
cpu_percent_one_core: +(
|
||||
(used.user + used.system) /
|
||||
(seconds * 10000)
|
||||
).toFixed(2),
|
||||
rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2),
|
||||
};
|
||||
report.samples.push(sample);
|
||||
console.log(JSON.stringify(sample));
|
||||
assert.equal(errors, 0, 'HTTP workload errors');
|
||||
}
|
||||
async function reconnect() {
|
||||
proxy.setMode('online');
|
||||
for (let attempt = 0; attempt < 80; attempt++) {
|
||||
if (
|
||||
Object.values(apps).every(
|
||||
({ app }) => app.get(CacheService).redis.status === 'ready',
|
||||
)
|
||||
)
|
||||
return;
|
||||
await pause(100);
|
||||
}
|
||||
assert.fail('Redis reconnect timeout');
|
||||
}
|
||||
async function correctness(name, userId, otherToken) {
|
||||
activate(name);
|
||||
await clearCache();
|
||||
const replica = Object.keys(apps).find((candidate) => candidate !== name);
|
||||
const chatId = await newChat(userId);
|
||||
const url = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal(
|
||||
(
|
||||
await request(
|
||||
name,
|
||||
'GET',
|
||||
url,
|
||||
undefined,
|
||||
randomBytes(32).toString('base64url'),
|
||||
)
|
||||
).status,
|
||||
401,
|
||||
);
|
||||
assert.equal(
|
||||
(await request(name, 'GET', url, undefined, otherToken)).status,
|
||||
404,
|
||||
);
|
||||
assert.equal(
|
||||
(await request(name, 'POST', url, { text: 'Forbidden' }, otherToken))
|
||||
.status,
|
||||
404,
|
||||
);
|
||||
assert.equal(
|
||||
(await request(name, 'GET', '/api/v1/chats/invalid/messages')).status,
|
||||
400,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request(
|
||||
name,
|
||||
'GET',
|
||||
'/api/v1/chats/invalid/messages',
|
||||
undefined,
|
||||
randomBytes(32).toString('base64url'),
|
||||
)
|
||||
).status,
|
||||
401,
|
||||
);
|
||||
assert.equal((await call(replica, 'GET', url)).items.length, 0);
|
||||
const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`);
|
||||
assert.ok(keys.length);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const posted = await call(name, 'POST', url, { text: 'During outage' });
|
||||
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
|
||||
await reconnect();
|
||||
assert.equal(await observer.exists(...keys), keys.length);
|
||||
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
|
||||
const service = apps[name].app.get(ChatService);
|
||||
const actual = service.prisma;
|
||||
service.prisma = actual.$extends({
|
||||
query: {
|
||||
$queryRaw({ args, query }) {
|
||||
const index = args.values.indexOf('Must roll back');
|
||||
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
assert.equal(
|
||||
(await request(name, 'POST', url, { text: 'Must roll back' })).status,
|
||||
500,
|
||||
);
|
||||
} finally {
|
||||
service.prisma = actual;
|
||||
}
|
||||
assert.equal(await epoch(name, chatId), 1n);
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
||||
const raceChat = await newChat(userId);
|
||||
const raceUrl = `/api/v1/chats/${raceChat}/messages`;
|
||||
let began, release;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
service.prisma = actual.$extends({
|
||||
query: {
|
||||
message: {
|
||||
async findMany({ args, query }) {
|
||||
const rows = await query(args);
|
||||
if (args.where?.chatId === raceChat) {
|
||||
began();
|
||||
await held;
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const pending = call(name, 'GET', raceUrl);
|
||||
try {
|
||||
await Promise.race([
|
||||
started,
|
||||
pause(10000).then(() => {
|
||||
throw new Error('Race load timeout');
|
||||
}),
|
||||
]);
|
||||
const fresh = await call(replica, 'POST', raceUrl, {
|
||||
text: 'After old SELECT',
|
||||
});
|
||||
release();
|
||||
await pending;
|
||||
assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id);
|
||||
} finally {
|
||||
release();
|
||||
service.prisma = actual;
|
||||
}
|
||||
await clearCache();
|
||||
assert.equal((await call(name, 'GET', url)).items[0].id, posted.id);
|
||||
report.correctness[name] = {
|
||||
access_statuses: true,
|
||||
redis_outage_and_retained_keys: true,
|
||||
rollback: true,
|
||||
delayed_old_fill: true,
|
||||
empty_cache: true,
|
||||
};
|
||||
console.log(JSON.stringify({ correctness: name, result: 'PASS' }));
|
||||
}
|
||||
|
||||
try {
|
||||
for (const variant of manifest.variants) {
|
||||
activate(variant.name);
|
||||
process.env.CACHE_PREFIX = `${prefix}:${variant.name}`;
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps[variant.name] = { app };
|
||||
exposeRequestIdHeader(app);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
apps[variant.name].url = await app.getUrl();
|
||||
}
|
||||
prisma = apps.baseline.app.get(PrismaService);
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Variant user',
|
||||
},
|
||||
});
|
||||
userIds.push(user.id);
|
||||
token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 3600000),
|
||||
},
|
||||
});
|
||||
const other = await prisma.user.create({
|
||||
data: {
|
||||
username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Other user',
|
||||
},
|
||||
});
|
||||
userIds.push(other.id);
|
||||
const otherToken = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId: other.id,
|
||||
tokenHash: createHash('sha256').update(otherToken).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 3600000),
|
||||
},
|
||||
});
|
||||
for (const name of Object.keys(apps))
|
||||
await correctness(name, user.id, otherToken);
|
||||
const readChat = await newChat(user.id);
|
||||
for (let offset = 0; offset < 100000; offset += 1000) {
|
||||
await prisma.message.createMany({
|
||||
data: Array.from({ length: 1000 }, (_, i) => ({
|
||||
chatId: readChat,
|
||||
senderId: user.id,
|
||||
text: `Synthetic message ${offset + i}`,
|
||||
createdAt: new Date(1700000000000 + offset + i),
|
||||
})),
|
||||
});
|
||||
}
|
||||
await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } });
|
||||
await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat);
|
||||
for (const [index, order] of report.methodology.order.entries()) {
|
||||
for (const name of order) {
|
||||
activate(name);
|
||||
await clearCache();
|
||||
const url = `/api/v1/chats/${readChat}/messages?limit=50`;
|
||||
const warmUntil = performance.now() + 10000;
|
||||
while (performance.now() < warmUntil) await call(name, 'GET', url);
|
||||
await measure(name, 'warm-history', index + 1, () =>
|
||||
call(name, 'GET', url),
|
||||
);
|
||||
const writeChat = await newChat(user.id);
|
||||
const writeUrl = `/api/v1/chats/${writeChat}/messages`;
|
||||
assert.equal((await call(name, 'GET', writeUrl)).items.length, 0);
|
||||
await measure(
|
||||
name,
|
||||
'writes-single-chat',
|
||||
index + 1,
|
||||
(i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }),
|
||||
4096,
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
4096,
|
||||
);
|
||||
assert.equal(await epoch(name, writeChat), 4096n);
|
||||
assert.equal((await call(name, 'GET', writeUrl)).items.length, 50);
|
||||
}
|
||||
}
|
||||
report.completed = true;
|
||||
} finally {
|
||||
await reconnect();
|
||||
if (prisma) {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
|
||||
}
|
||||
await clearCache();
|
||||
for (const { app } of Object.values(apps)) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
await writeFile(
|
||||
'/results/epoch-variants.json',
|
||||
JSON.stringify(report, null, 2) + '\n',
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
completed: report.completed,
|
||||
correctness: report.correctness,
|
||||
}),
|
||||
);
|
||||
@@ -0,0 +1,261 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { randomUUID, createHash } = require('node:crypto');
|
||||
const { spawn } = require('node:child_process');
|
||||
const ts = require('typescript');
|
||||
require('dotenv').config({ quiet: true });
|
||||
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const statePath = path.join(root, '.epoch-variants-state.json');
|
||||
const files = [
|
||||
'Auth/auth.service.ts',
|
||||
'User/user.service.ts',
|
||||
'User/user.guard.ts',
|
||||
'Chat/chat.service.ts',
|
||||
'Chat/chat.controller.ts',
|
||||
];
|
||||
const digest = (value) => createHash('sha256').update(value).digest('hex');
|
||||
const command = process.argv[2];
|
||||
|
||||
async function main() {
|
||||
let state;
|
||||
if (command === 'init') {
|
||||
assert.ok(!fs.existsSync(statePath));
|
||||
assert.ok(
|
||||
/messagesEpoch\s+BigInt/.test(
|
||||
fs.readFileSync(path.join(root, 'prisma/schema.prisma'), 'utf8'),
|
||||
),
|
||||
'Historical comparison requires the pre-bucket schema. Original sources are saved in evidence/epoch-variants-sources.json',
|
||||
);
|
||||
state = {
|
||||
directory: fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-epoch-variants-')),
|
||||
schema: 'epoch_variants_' + randomUUID().replaceAll('-', ''),
|
||||
variants: [],
|
||||
};
|
||||
await database(async (client) => {
|
||||
await client.query(`CREATE SCHEMA "${state.schema}"`);
|
||||
await client.query(
|
||||
`CREATE TABLE "${state.schema}".chat_message_epochs (chat_id uuid NOT NULL REFERENCES public.chats(id) ON DELETE CASCADE, bucket smallint NOT NULL CHECK (bucket BETWEEN 0 AND 15), epoch bigint NOT NULL DEFAULT 0, PRIMARY KEY (chat_id, bucket))`,
|
||||
);
|
||||
});
|
||||
capture(state, 'baseline');
|
||||
} else {
|
||||
state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
}
|
||||
if (command === 'capture') capture(state, process.argv[3]);
|
||||
if (command === 'capture-striped') capture(state, 'striped', true);
|
||||
if (command === 'apply-striped') {
|
||||
const sources = {};
|
||||
for (const variant of state.variants) {
|
||||
sources[variant.name] = Object.fromEntries(
|
||||
files.map((relative) => [
|
||||
relative,
|
||||
fs.readFileSync(
|
||||
path.join(state.directory, variant.name, relative),
|
||||
'utf8',
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(root, 'evidence/epoch-variants-sources.json'),
|
||||
JSON.stringify(sources, null, 2) + '\n',
|
||||
);
|
||||
for (const relative of ['Auth/auth.service.ts', 'Chat/chat.service.ts']) {
|
||||
const source = sources.striped[relative].replaceAll(
|
||||
`"${state.schema}".chat_message_epochs`,
|
||||
'"public"."chat_message_epochs"',
|
||||
);
|
||||
fs.writeFileSync(path.join(root, 'src', relative), source);
|
||||
}
|
||||
console.log(
|
||||
'Applied striped epochs to application sources; saved all source snapshots',
|
||||
);
|
||||
}
|
||||
if (['init', 'capture', 'capture-striped'].includes(command)) {
|
||||
fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
|
||||
console.log(
|
||||
'Captured:',
|
||||
state.variants.map((variant) => variant.name).join(', '),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (command === 'run') {
|
||||
assert.equal(state.variants.length, 4);
|
||||
fs.writeFileSync(
|
||||
path.join(state.directory, 'manifest.json'),
|
||||
JSON.stringify(state, null, 2),
|
||||
);
|
||||
const override = path.join(state.directory, 'compose.json');
|
||||
fs.writeFileSync(
|
||||
override,
|
||||
JSON.stringify({
|
||||
services: {
|
||||
benchmark: {
|
||||
volumes: [
|
||||
{
|
||||
type: 'bind',
|
||||
source: state.directory,
|
||||
target: '/variants',
|
||||
read_only: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
const args = [
|
||||
'compose',
|
||||
'-f',
|
||||
'compose.yaml',
|
||||
'-f',
|
||||
'test/compose.benchmark.yaml',
|
||||
'-f',
|
||||
override,
|
||||
'run',
|
||||
'--rm',
|
||||
'--no-deps',
|
||||
'--pull',
|
||||
'never',
|
||||
'benchmark',
|
||||
'node',
|
||||
'test/epoch-variants-benchmark.mjs',
|
||||
];
|
||||
const code = await new Promise((resolve, reject) => {
|
||||
const child = spawn('docker', args, { cwd: root, stdio: 'inherit' });
|
||||
child.once('error', reject);
|
||||
child.once('exit', resolve);
|
||||
});
|
||||
fs.writeFileSync(
|
||||
path.join(root, 'evidence/epoch-variants-code.json'),
|
||||
JSON.stringify(
|
||||
{
|
||||
variants: state.variants,
|
||||
schema: state.schema,
|
||||
exit_code: code,
|
||||
implementation:
|
||||
'TypeScript source snapshots; variant methods activated sequentially before HTTP workloads. Separate cache prefixes and API instances, shared fixtures and Node process.',
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
process.exitCode = code ?? 1;
|
||||
}
|
||||
if (command === 'seed-read-fixture') {
|
||||
await database(async (client) => {
|
||||
const result = await client.query(
|
||||
`SELECT c.id FROM public.chats c JOIN public.messages m ON m.chat_id = c.id WHERE c.title = 'Variant benchmark' GROUP BY c.id HAVING COUNT(*) = 100000`,
|
||||
);
|
||||
assert.equal(result.rows.length, 1);
|
||||
const chatId = result.rows[0].id;
|
||||
await client.query(
|
||||
'UPDATE public.chats SET messages_epoch = 100000 WHERE id = $1::uuid',
|
||||
[chatId],
|
||||
);
|
||||
await client.query(
|
||||
`INSERT INTO "${state.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`,
|
||||
[chatId],
|
||||
);
|
||||
console.log('Seeded read fixture with 16 nonzero epoch buckets');
|
||||
});
|
||||
}
|
||||
if (command === 'cleanup') {
|
||||
await database((client) =>
|
||||
client.query(`DROP SCHEMA "${state.schema}" CASCADE`),
|
||||
);
|
||||
const resolved = path.resolve(state.directory);
|
||||
assert.equal(path.dirname(resolved), path.resolve(os.tmpdir()));
|
||||
assert.ok(path.basename(resolved).startsWith('lab2-epoch-variants-'));
|
||||
fs.rmSync(resolved, { recursive: true, force: true });
|
||||
fs.unlinkSync(statePath);
|
||||
console.log('Removed experiment schema and temporary source snapshots');
|
||||
}
|
||||
}
|
||||
|
||||
function capture(state, name, striped = false) {
|
||||
assert.match(name, /^[a-z-]+$/);
|
||||
assert.ok(!state.variants.some((variant) => variant.name === name));
|
||||
const modules = {};
|
||||
const table = `"${state.schema}".chat_message_epochs`;
|
||||
for (const relative of files) {
|
||||
let source = fs
|
||||
.readFileSync(path.join(root, 'src', relative), 'utf8')
|
||||
.replace(/\r\n/g, '\n');
|
||||
if (
|
||||
striped &&
|
||||
['Auth/auth.service.ts', 'Chat/chat.service.ts'].includes(relative)
|
||||
) {
|
||||
source = source.replaceAll(
|
||||
'c.messages_epoch',
|
||||
`COALESCE((SELECT SUM(e.epoch) FROM ${table} AS e WHERE e.chat_id = c.id), 0)::bigint`,
|
||||
);
|
||||
if (relative === 'Chat/chat.service.ts') {
|
||||
const start = source.indexOf(' const [message] =');
|
||||
const end = source.indexOf(' if (!message)', start);
|
||||
assert.ok(start !== -1 && end > start);
|
||||
source =
|
||||
source.slice(0, start) +
|
||||
' const messageId = randomUUID();\n' +
|
||||
' const bucket = parseInt(messageId.slice(0, 2), 16) % 16;\n' +
|
||||
' const [message] = await this.prisma.$queryRaw<StoredMessage[]>(Prisma.sql`\n' +
|
||||
` WITH updated AS (\n INSERT INTO ${table} AS e (chat_id, bucket, epoch)\n` +
|
||||
' SELECT cm.chat_id, ${bucket}::smallint, 1 FROM chat_members AS cm\n' +
|
||||
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
|
||||
' ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = e.epoch + 1\n' +
|
||||
' RETURNING chat_id\n )\n' +
|
||||
' INSERT INTO messages (id, chat_id, sender_id, text)\n' +
|
||||
' SELECT ${messageId}::uuid, updated.chat_id, ${userId}::uuid, ${text} FROM updated\n' +
|
||||
' RETURNING id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt"\n' +
|
||||
' `);\n' +
|
||||
source.slice(end);
|
||||
}
|
||||
}
|
||||
const compiled = ts.transpileModule(source, {
|
||||
compilerOptions: {
|
||||
target: ts.ScriptTarget.ES2023,
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
experimentalDecorators: true,
|
||||
emitDecoratorMetadata: true,
|
||||
},
|
||||
reportDiagnostics: true,
|
||||
});
|
||||
assert.ok(
|
||||
!compiled.diagnostics.some(
|
||||
(diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error,
|
||||
),
|
||||
);
|
||||
const destination = path.join(
|
||||
state.directory,
|
||||
name,
|
||||
relative.replace(/\.ts$/, '.js'),
|
||||
);
|
||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||
fs.writeFileSync(destination, compiled.outputText);
|
||||
fs.writeFileSync(destination.replace(/\.js$/, '.ts'), source);
|
||||
modules[relative] = {
|
||||
source_sha256: digest(source),
|
||||
js_sha256: digest(compiled.outputText),
|
||||
};
|
||||
}
|
||||
state.variants.push({ name, modules, striped });
|
||||
}
|
||||
|
||||
async function database(operation) {
|
||||
const client = new (require('pg').Client)({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
await client.connect();
|
||||
try {
|
||||
return await operation(client);
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const report = JSON.parse(
|
||||
await readFile('evidence/epoch-variants.json', 'utf8'),
|
||||
);
|
||||
assert.ok(report.completed);
|
||||
const labels = {
|
||||
baseline: 'Текущая схема',
|
||||
'profile-join': 'Профиль из JOIN',
|
||||
'access-join': 'Профиль + единая проверка доступа',
|
||||
striped: 'Единая проверка + 16 счётчиков',
|
||||
};
|
||||
const aggregates = [];
|
||||
const lines = [
|
||||
'# Сравнение трёх оптимизаций эпох и текущей схемы',
|
||||
'',
|
||||
`Дата UTC: ${report.recorded_at}.`,
|
||||
'',
|
||||
'Все режимы используют Redis. Варианты последовательно добавляют оптимизации: профиль из запроса сессии; объединённую проверку сессии, членства и эпохи; 16 счётчиков истории, увеличиваемых независимо, с суммой как версией кэша. Порядок вариантов меняется между тремя повторами. Чтения: 100 000 сообщений, limit=50, конкурентность 32, прогрев 10 с, измерение 60 с. Запись: 4096 сообщений в отдельный новый чат на вариант/повтор. Только один workload активен одновременно.',
|
||||
'',
|
||||
'RPS рассчитан по общему числу успешных запросов и времени повторов. Средняя задержка взвешена по числу запросов. p95 — среднее p95 отдельных повторов, а не перцентиль объединённой выборки. SQL/HTTP исключает BEGIN/COMMIT/ROLLBACK/SET.',
|
||||
'',
|
||||
'| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | RPS по повторам | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | --- | ---: |',
|
||||
];
|
||||
for (const workload of ['warm-history', 'writes-single-chat']) {
|
||||
for (const [name, label] of Object.entries(labels)) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.variant === name && sample.workload === workload,
|
||||
);
|
||||
assert.equal(samples.length, 3);
|
||||
const sum = (select) =>
|
||||
samples.reduce((total, sample) => total + select(sample), 0);
|
||||
const successes = sum((sample) => sample.successes);
|
||||
const errors = sum((sample) => sample.errors);
|
||||
const aggregate = {
|
||||
variant: name,
|
||||
workload,
|
||||
successes,
|
||||
errors,
|
||||
rps: successes / sum((sample) => sample.elapsed_seconds),
|
||||
mean_latency_ms:
|
||||
sum((sample) => sample.mean_latency_ms * sample.successes) / successes,
|
||||
mean_p95_ms: sum((sample) => sample.p95_ms) / samples.length,
|
||||
sql_per_request:
|
||||
sum((sample) => sample.sql_queries) / (successes + errors),
|
||||
repeat_rps: samples.map((sample) => sample.rps),
|
||||
};
|
||||
aggregates.push(aggregate);
|
||||
lines.push(
|
||||
`| ${workload} | ${label} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.repeat_rps.map((value) => value.toFixed(1)).join(' / ')} | ${errors} |`,
|
||||
);
|
||||
assert.ok(Object.values(report.correctness[name]).every(Boolean));
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'Все варианты прошли проверки 401/404/400, доступа постороннего пользователя, чтения после записи при недоступном Redis и после восстановления со старыми ключами, отката счётчика при ошибке INSERT, задержанного заполнения старого кэша и работы с пустым кэшем. После каждого нагрузочного блока проверены число сообщений и значение эпохи 4096.',
|
||||
'',
|
||||
'Четыре API и HTTP-генератор работают в одном Node-процессе, каждый API имеет отдельный Redis-prefix и пул Prisma прежнего размера. Методы переключаются между скомпилированными снимками исходников перед каждым workload; скрипт сохраняет метаданные маршрутов. Idle-экземпляры присутствуют во всех режимах. CPU/RSS включает генератор и все экземпляры. Другие сервисы Docker-host продолжают работать. Сессии создаются напрямую, KDF не измеряется. Порядок меняется, но три повтора и фоновые нагрузки всё ещё ограничивают выводы об устойчивых коэффициентах ускорения.',
|
||||
'',
|
||||
'Распределённые счётчики находятся в отдельной временной схеме. Рабочая схема приложения не переключается на них. В нагрузочном чате 16 ненулевых счётчиков; стоимость их суммирования входит в запрос доступа. Первое чтение baseline выполнено до заполнения счётчиков временной схемы; данные истории неизменны, счётчики подготовлены до первого измерения striped. Это изменение подготовки фикстуры добавлено в скрипт для следующих запусков.',
|
||||
'',
|
||||
'Сырые результаты: [epoch-variants.json](epoch-variants.json). Хеши исходников вариантов: [epoch-variants-code.json](epoch-variants-code.json).',
|
||||
);
|
||||
await writeFile('evidence/epoch-variants-summary.md', lines.join('\n') + '\n');
|
||||
await writeFile(
|
||||
'evidence/epoch-variants-averages.json',
|
||||
JSON.stringify(aggregates, null, 2) + '\n',
|
||||
);
|
||||
console.log(lines.slice(0, 19).join('\n'));
|
||||
@@ -0,0 +1,195 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { createRequire } from 'node:module';
|
||||
import { resolve } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { Client } = require('pg');
|
||||
assert.ok(process.env.DATABASE_URL, 'Set DATABASE_URL for the project PostgreSQL');
|
||||
const output = resolve('evidence');
|
||||
const schema = `explain_history_${randomUUID().replaceAll('-', '')}`;
|
||||
const table = `"${schema}"."messages"`;
|
||||
const client = new Client({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
connectionTimeoutMillis: 5000,
|
||||
query_timeout: 60000,
|
||||
options: '-c statement_timeout=60000 -c lock_timeout=2000',
|
||||
});
|
||||
const seed = `
|
||||
WITH base AS (
|
||||
SELECT n, CASE WHEN (n - 1) % 6 < 2 THEN 0 ELSE (n - 1) % 6 - 1 END AS chat_no
|
||||
FROM generate_series(1, 60000) AS n
|
||||
), ranked AS (
|
||||
SELECT n, chat_no, row_number() OVER (PARTITION BY chat_no ORDER BY n) AS chat_seq
|
||||
FROM base
|
||||
)
|
||||
INSERT INTO ${table} (id, chat_id, sender_id, text, created_at)
|
||||
SELECT md5('message:' || n)::uuid, md5('chat:' || chat_no)::uuid,
|
||||
md5('sender:' || (n % 20))::uuid,
|
||||
rpad('Synthetic message ' || n || ' ', 256, '.'),
|
||||
'2026-01-01T00:00:00Z'::timestamptz + ((chat_seq - 1) / 10) * interval '1 second'
|
||||
FROM ranked ORDER BY n`;
|
||||
const columns = 'id, chat_id, sender_id, text, created_at';
|
||||
const firstSql = `SELECT ${columns} FROM ${table}
|
||||
WHERE chat_id = $1::uuid ORDER BY created_at ASC, id ASC LIMIT $2`;
|
||||
const orSql = `SELECT ${columns} FROM ${table}
|
||||
WHERE chat_id = $1::uuid
|
||||
AND (created_at > $2::timestamptz OR (created_at = $2::timestamptz AND id > $3::uuid))
|
||||
ORDER BY created_at ASC, id ASC LIMIT $4`;
|
||||
const tupleSql = `SELECT ${columns} FROM ${table}
|
||||
WHERE chat_id = $1::uuid AND (created_at, id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY created_at ASC, id ASC LIMIT $4`;
|
||||
|
||||
function nodes(plan) {
|
||||
return [plan, ...(plan.Plans ?? []).flatMap(nodes)];
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.floor(sorted.length / 2)];
|
||||
}
|
||||
|
||||
function inspectDocker() {
|
||||
const template = '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"server_version":{{json .ServerVersion}},"os":{{json .OperatingSystem}},"arch":{{json .Architecture}}}';
|
||||
const result = spawnSync('docker', ['info', '--format', template], {
|
||||
encoding: 'utf8', timeout: 10000, windowsHide: true,
|
||||
});
|
||||
try {
|
||||
if (result.status !== 0) return null;
|
||||
const info = JSON.parse(result.stdout);
|
||||
const ids = spawnSync('docker', ['compose', 'ps', '--quiet', 'postgres'], {
|
||||
encoding: 'utf8', timeout: 10000, windowsHide: true,
|
||||
});
|
||||
if (ids.status === 0 && ids.stdout.trim()) {
|
||||
const limitsTemplate = '{"image":{{json .Config.Image}},"memory_limit_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}},"cpu_period":{{.HostConfig.CpuPeriod}},"cpuset_cpus":{{json .HostConfig.CpusetCpus}}}';
|
||||
const limits = spawnSync('docker', ['inspect', '--format', limitsTemplate, ids.stdout.trim()], {
|
||||
encoding: 'utf8', timeout: 10000, windowsHide: true,
|
||||
});
|
||||
if (limits.status === 0) info.postgres_container = JSON.parse(limits.stdout);
|
||||
}
|
||||
return info;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
await client.connect();
|
||||
const runs = [];
|
||||
const raw = [];
|
||||
const signatures = new Map();
|
||||
let metadata;
|
||||
let queries;
|
||||
let cleanup = false;
|
||||
let sourceIndexes;
|
||||
try {
|
||||
sourceIndexes = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows;
|
||||
await client.query('BEGIN');
|
||||
await client.query(`CREATE SCHEMA "${schema}"`);
|
||||
await client.query(`CREATE TABLE ${table} (LIKE public.messages INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
|
||||
await client.query(seed);
|
||||
await client.query(`ANALYZE ${table}`);
|
||||
const distribution = (await client.query(`SELECT chat_id, count(*)::integer AS messages FROM ${table} GROUP BY chat_id ORDER BY messages DESC, chat_id`)).rows;
|
||||
assert.equal(distribution.reduce((sum, row) => sum + row.messages, 0), 60000);
|
||||
const chatId = distribution.find((row) => row.messages === 20000).chat_id;
|
||||
const cursor = async (offset) => (await client.query(
|
||||
`SELECT id, created_at FROM ${table} WHERE chat_id = $1::uuid ORDER BY created_at, id OFFSET $2 LIMIT 1`,
|
||||
[chatId, offset],
|
||||
)).rows[0];
|
||||
const middle = await cursor(9999);
|
||||
const late = await cursor(17999);
|
||||
queries = [
|
||||
{ name: 'first_page', sql: firstSql, parameters: [chatId, 51] },
|
||||
{ name: 'middle_page_or', sql: orSql, parameters: [chatId, middle.created_at.toISOString(), middle.id, 51] },
|
||||
{ name: 'late_page_or', sql: orSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] },
|
||||
{ name: 'late_page_tuple', sql: tupleSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] },
|
||||
];
|
||||
const settings = (await client.query(
|
||||
'SELECT name, setting, unit FROM pg_settings WHERE name = ANY($1::text[]) ORDER BY name',
|
||||
[['server_version', 'shared_buffers', 'work_mem', 'random_page_cost', 'effective_cache_size', 'max_parallel_workers_per_gather', 'jit', 'block_size']],
|
||||
)).rows;
|
||||
metadata = {
|
||||
recorded_at: new Date().toISOString(),
|
||||
command: 'node test/explain-history.mjs',
|
||||
node_version: process.version,
|
||||
docker: inspectDocker(),
|
||||
settings,
|
||||
source_table_indexes_before: sourceIndexes,
|
||||
dataset: { total_messages: 60000, target_chat_messages: 20000, chats: 5, distribution, text_length: 256, timestamp_ties: 10, insertion_order: 'interleaved chats' },
|
||||
method: { warmup_runs_per_query_per_phase: 1, measured_runs: 5, page_size: 50, fetched_rows: 51, cursor_positions: [10000, 18000], cache: 'warm PostgreSQL buffers; Redis is bypassed', cleanup: 'ROLLBACK the experiment schema; live tables and indexes unchanged', planner: 'default scan settings; no forced index or disabled sequential scans', schema_difference: 'copied columns/checks/defaults; foreign keys omitted equally in all phases', tuple_variant: 'current cursor SQL in ChatService.listMessages; OR is the pre-fix baseline' },
|
||||
};
|
||||
const phases = [
|
||||
{ name: 'no_indexes', ddl: [] },
|
||||
{ name: 'without_history_index', ddl: [`CREATE UNIQUE INDEX messages_pkey ON ${table} (id)`, `CREATE INDEX messages_sender_id_idx ON ${table} (sender_id)`] },
|
||||
{ name: 'with_history_index', ddl: [`CREATE INDEX messages_chat_id_created_at_id_idx ON ${table} (chat_id, created_at, id)`] },
|
||||
];
|
||||
for (const phase of phases) {
|
||||
for (const ddl of phase.ddl) await client.query(ddl);
|
||||
await client.query(`ANALYZE ${table}`);
|
||||
const indexes = (await client.query('SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = $1 ORDER BY indexname', [schema])).rows;
|
||||
const phaseRuns = new Map(queries.map((query) => [query.name, []]));
|
||||
for (const query of queries) {
|
||||
const rows = (await client.query(query.sql, query.parameters)).rows;
|
||||
assert.equal(rows.length, 51);
|
||||
const signature = createHash('sha256').update(JSON.stringify(rows)).digest('hex');
|
||||
if (signatures.has(query.name)) assert.equal(signature, signatures.get(query.name));
|
||||
else signatures.set(query.name, signature);
|
||||
if (query.name === 'late_page_tuple') assert.equal(signature, signatures.get('late_page_or'));
|
||||
await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters);
|
||||
}
|
||||
for (let repetition = 1; repetition <= 5; repetition++) {
|
||||
for (const query of queries) {
|
||||
const plan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters)).rows[0]['QUERY PLAN'][0];
|
||||
phaseRuns.get(query.name).push(plan);
|
||||
}
|
||||
}
|
||||
for (const query of queries) {
|
||||
const plans = phaseRuns.get(query.name);
|
||||
const medianMs = median(plans.map((plan) => plan['Execution Time']));
|
||||
const representative = plans.find((plan) => plan['Execution Time'] === medianMs);
|
||||
const flat = nodes(representative.Plan);
|
||||
const scan = flat.find((node) => node['Node Type'].includes('Scan'));
|
||||
runs.push({
|
||||
phase: phase.name, query: query.name, indexes, sql: query.sql, parameters: query.parameters,
|
||||
result_sha256: signatures.get(query.name), execution_ms: plans.map((plan) => plan['Execution Time']),
|
||||
median_execution_ms: medianMs, representative_plan: representative,
|
||||
scan_type: scan?.['Node Type'], index_name: scan?.['Index Name'] ?? null,
|
||||
rows_removed_by_filter: scan?.['Rows Removed by Filter'] ?? 0,
|
||||
scanned_rows: ((scan?.['Actual Rows'] ?? 0) + (scan?.['Rows Removed by Filter'] ?? 0)) * (scan?.['Actual Loops'] ?? 1),
|
||||
sort: flat.some((node) => node['Node Type'] === 'Sort'),
|
||||
shared_buffer_hits: representative.Plan['Shared Hit Blocks'] ?? 0,
|
||||
shared_buffer_reads: representative.Plan['Shared Read Blocks'] ?? 0,
|
||||
raw_plans: plans,
|
||||
});
|
||||
const textPlan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS) ${query.sql}`, query.parameters)).rows.map((row) => row['QUERY PLAN']).join('\n');
|
||||
raw.push(`=== ${phase.name} / ${query.name} ===\nIndexes: ${indexes.map((index) => index.indexname).join(', ') || '(none)'}\nSQL:\n${query.sql}\nParameters: ${JSON.stringify(query.parameters)}\n\n${textPlan}\n`);
|
||||
console.log(`${phase.name} / ${query.name}: median ${medianMs.toFixed(3)} ms; ${scan?.['Node Type']}; scanned ${runs.at(-1).scanned_rows}; sort=${runs.at(-1).sort}`);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
if (metadata) {
|
||||
metadata.source_table_indexes_after = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows;
|
||||
assert.deepEqual(metadata.source_table_indexes_after, sourceIndexes);
|
||||
const leftovers = (await client.query('SELECT count(*)::integer AS count FROM pg_namespace WHERE nspname = $1', [schema])).rows[0].count;
|
||||
assert.equal(leftovers, 0);
|
||||
metadata.experiment_schema_absent = true;
|
||||
}
|
||||
cleanup = true;
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
assert.equal(cleanup, true);
|
||||
await mkdir(output, { recursive: true });
|
||||
await writeFile(resolve(output, 'history-index-explain.json'), JSON.stringify({ ...metadata, cleanup_completed: cleanup, runs }, null, 2) + '\n');
|
||||
await writeFile(resolve(output, 'history-index-plans.txt'), raw.join('\n'));
|
||||
const labels = { no_indexes: 'Без индексов', without_history_index: 'PK + sender_id, без индекса истории', with_history_index: 'С индексом истории' };
|
||||
const tableRows = runs.map((row) => `| ${labels[row.phase]} | ${row.query} | ${row.median_execution_ms.toFixed(3)} | ${row.scan_type} | ${row.scanned_rows} | ${row.sort ? 'да' : 'нет'} | ${row.shared_buffer_hits} / ${row.shared_buffer_reads} |`).join('\n');
|
||||
const speedup = (query) => runs.find((row) => row.query === query && row.phase === 'no_indexes').median_execution_ms / runs.find((row) => row.query === query && row.phase === 'with_history_index').median_execution_ms;
|
||||
const summary = `# EXPLAIN истории сообщений: с индексами и без\n\nДата UTC: ${metadata.recorded_at}. PostgreSQL ${metadata.settings.find((setting) => setting.name === 'server_version').setting}.\n\nКоманда: \`node test/explain-history.mjs\` (нужен \`DATABASE_URL\`; PostgreSQL проекта с применёнными миграциями).\n\n60 000 синтетических сообщений в пяти чатах; в целевом чате 20 000. Чаты перемежаются при вставке, текст 256 символов, по 10 сообщений имеют одинаковый created_at. Страница API: 50 сообщений + 1 для next_cursor. Курсоры после 10 000-го и 18 000-го сообщений. Запросы первой страницы и с OR соответствуют прежнему \`ChatService.listMessages\`; tuple — исправленное условие курсора, используемое в текущем API. Скрипт сравнивает прежний и исправленный запросы; сам код API не меняет.\n\n| Индексы | Запрос | Медиана, мс | Scan | Проверено строк | Sort | Buffers hit/read |\n| --- | --- | ---: | --- | ---: | --- | ---: |\n${tableRows}\n\nПервая страница с индексом быстрее варианта без индексов в ${speedup('first_page').toFixed(1)} раза на этом наборе и стенде. Само наличие PK и индекса sender_id не заменяет индекс истории. Составной B-tree \`(chat_id, created_at, id)\` соответствует равенству chat_id и порядку истории; наличие Sort и границы Index Cond видны в сохранённых планах. ORDER BY нужен и при наличии индекса.\n\nПоздняя страница с OR: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_or').scanned_rows} проверенных строк. Эквивалентное сравнение пары: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_tuple').scanned_rows} строк. Это показывает, удалось ли передать границу курсора в Index Cond или она осталась Filter. Совпадение результатов всех фаз и OR/tuple проверено SHA-256 от полных 51 строк.\n\nМетодика: отдельная таблица с колонками текущей public.messages в случайной схеме; одинаковые данные во всех фазах; ANALYZE после загрузки и изменений индексов; 1 прогрев + 5 измерений каждого запроса в каждой фазе, медиана PostgreSQL Execution Time. Планы не принуждались настройками enable_seqscan/enable_indexscan. Фаза с индексом сохраняет PK и sender_id. Все создание таблиц, данных и индексов выполнены внутри транзакции; ROLLBACK удалил экспериментальную схему. Рабочие таблицы и индексы не изменялись.\n\nОграничения: это прогретый SQL-тест без HTTP, Redis, аутентификации и конкуренции; не cold-cache benchmark, не измерение RPS и не доказательство latency всего API. FK не копировались, поскольку сравнивается чтение; это одинаково во всех фазах. Измерения зависят от ресурсов и нагрузки Docker-host. Лимиты и настройки PostgreSQL записаны в JSON; текстовые планы сняты отдельным запуском, поэтому их времена могут отличаться от медиан JSON.\n\nФайлы: [полные текстовые планы](history-index-plans.txt), [JSON, пять повторов, настройки и результаты](history-index-explain.json).\n`;
|
||||
await writeFile(resolve(output, 'history-index-summary.md'), summary);
|
||||
console.log(`Saved EXPLAIN evidence to ${output}; experiment schema rolled back.`);
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check failover of a named Compose API replica without changing public tests."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--target-index", type=int, choices=(1, 2), required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
ids = subprocess.run(
|
||||
["docker", "compose", "ps", "--all", "-q", "api"],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.splitlines()
|
||||
check(len(ids) == 2, "targeted failover requires two API replicas")
|
||||
target = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.Config.Hostname}}",
|
||||
ids[args.target_index - 1]],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.strip()
|
||||
expected = {
|
||||
subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.Config.Hostname}}", item],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.strip()
|
||||
for item in ids
|
||||
}
|
||||
|
||||
client = Client(args.base_url, timeout=10)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before = client.message(alice, chat, "Before targeted failover")["json"]
|
||||
observed = {
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
|
||||
"headers"
|
||||
]["X-Instance-Id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
check(observed == expected, "both replicas must answer before failure")
|
||||
client.timeout = 2
|
||||
|
||||
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
||||
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(
|
||||
[str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=10, check=False,
|
||||
)
|
||||
check(result.returncode == 0, "stop-one hook failed")
|
||||
|
||||
key = uuid.uuid4().hex
|
||||
text = "Targeted failover " + key
|
||||
failures = 0
|
||||
while True:
|
||||
try:
|
||||
written = client.message(alice, chat, text, key=key)
|
||||
break
|
||||
except Failure:
|
||||
failures += 1
|
||||
check(time.monotonic() - started <= 10, "write recovery exceeded 10 seconds")
|
||||
time.sleep(0.1)
|
||||
recovery = time.monotonic() - started
|
||||
check(recovery <= 10, "write recovery exceeded 10 seconds")
|
||||
check(written["headers"]["X-Instance-Id"] != target,
|
||||
"stopped replica answered the write")
|
||||
|
||||
reads = 0
|
||||
first_read = None
|
||||
while reads < 10:
|
||||
try:
|
||||
response = client.call("GET", "/api/v1/users/me", actor=alice,
|
||||
schema="User")
|
||||
except Failure:
|
||||
failures += 1
|
||||
check(time.monotonic() - started <= 10,
|
||||
"read recovery exceeded 10 seconds")
|
||||
continue
|
||||
if first_read is None:
|
||||
first_read = time.monotonic() - started
|
||||
check(first_read <= 10, "read recovery exceeded 10 seconds")
|
||||
check(response["headers"]["X-Instance-Id"] != target,
|
||||
"stopped replica still serves traffic")
|
||||
reads += 1
|
||||
messages = client.call(
|
||||
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100",
|
||||
actor=bob, schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
check(before in messages, "confirmed pre-failure message disappeared")
|
||||
check(sum(message["text"] == text for message in messages) == 1,
|
||||
"uncertain retry created a duplicate")
|
||||
print(f"PASS: replica {args.target_index} stopped; write_recovery={recovery:.3f}s; "
|
||||
f"read_recovery={first_read:.3f}s; transient_failures={failures}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run
|
||||
from resilience import history
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--action-script', required=True)
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
original = client.message(alice, chat, 'Сохранность после полного перезапуска')['json']
|
||||
check(history(client, bob, chat) == [original], 'pre-restart history mismatch')
|
||||
hook = Path(args.action_script).resolve()
|
||||
check(hook.is_file(), 'restart hook not found')
|
||||
print('Restarting backend, PostgreSQL and Redis; hook output suppressed.', flush=True)
|
||||
try:
|
||||
proc = subprocess.run([str(hook)], cwd=ROOT, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure('restart hook exceeded 90 seconds') from None
|
||||
check(proc.returncode == 0, 'restart hook failed')
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'API did not recover within 90 seconds')
|
||||
time.sleep(0.5)
|
||||
for actor in (alice, bob, eve):
|
||||
check(client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json']
|
||||
== actor['user'], 'user or existing session changed after restart')
|
||||
check(client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')['json']
|
||||
== chat, 'chat or membership changed after restart')
|
||||
check(history(client, bob, chat) == [original], 'confirmed message lost, changed or duplicated')
|
||||
client.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve, schema='Error')
|
||||
followup = client.message(bob, chat, 'Запись после восстановления')['json']
|
||||
check(history(client, alice, chat) == [original, followup], 'post-restart write/read mismatch')
|
||||
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
client.call('GET', '/api/v1/users/me', 401, actor=alice, schema='Error')
|
||||
print('PASS: users, chat, membership, confirmed message and all existing sessions survived; '
|
||||
'access control, new writes and logout work after restart.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 3 grade 4: measured replica failover or sequential replacement."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
def compose(*args, timeout=60):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args],
|
||||
cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def inspect(container_id, field):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", field, container_id],
|
||||
cwd=ROOT, capture_output=True, text=True, timeout=10, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker inspect failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, "docker inspect failed")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = compose("ps", "--all", "-q", service).splitlines()
|
||||
check(bool(ids), f"{service} container is missing")
|
||||
return ids
|
||||
|
||||
|
||||
def container(service):
|
||||
ids = containers(service)
|
||||
check(len(ids) == 1, f"expected one {service} container")
|
||||
return ids[0]
|
||||
|
||||
|
||||
def instance(container_id):
|
||||
return inspect(container_id, "{{.Config.Hostname}}")
|
||||
|
||||
|
||||
def healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if inspect(container_id, "{{.State.Health.Status}}") == "healthy":
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure(f"API container did not become healthy in {timeout} seconds")
|
||||
|
||||
|
||||
def both_observed(client, actor, expected, timeout=30):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call("GET", "/api/v1/users/me", actor=actor,
|
||||
schema="User")
|
||||
seen.add(response["headers"]["X-Instance-Id"])
|
||||
except Failure:
|
||||
pass
|
||||
if seen == expected:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise Failure(f"both replicas were not observed; saw {sorted(seen)}")
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base_url, actor, chat, concurrency):
|
||||
self.base_url = base_url
|
||||
self.actor = actor
|
||||
self.chat = chat
|
||||
self.concurrency = concurrency
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.confirmed = set()
|
||||
self.attempted = set()
|
||||
self.pool = ThreadPoolExecutor(max_workers=concurrency)
|
||||
|
||||
def start(self):
|
||||
self.futures = [self.pool.submit(self.worker, i)
|
||||
for i in range(self.concurrency)]
|
||||
|
||||
def worker(self, worker_id):
|
||||
client = Client(self.base_url, timeout=2)
|
||||
index = 0
|
||||
pending = None
|
||||
while not self.stop.is_set():
|
||||
write = worker_id % 2 == 0
|
||||
if write and pending is None:
|
||||
key = uuid.uuid4().hex
|
||||
pending = (key, f"grade4-{worker_id}-{index}-{key}")
|
||||
index += 1
|
||||
with self.lock:
|
||||
self.attempted.add(pending[1])
|
||||
try:
|
||||
if write:
|
||||
response = client.message(self.actor, self.chat,
|
||||
pending[1], key=pending[0])
|
||||
else:
|
||||
response = client.call("GET", "/api/v1/users/me",
|
||||
actor=self.actor, schema="User")
|
||||
event = (time.monotonic(), "write" if write else "read", True,
|
||||
response["headers"]["X-Instance-Id"])
|
||||
if write:
|
||||
with self.lock:
|
||||
self.confirmed.add(pending[1])
|
||||
pending = None
|
||||
except Failure:
|
||||
event = (time.monotonic(), "write" if write else "read", False, None)
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
self.stop.wait(0.05)
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
try:
|
||||
for future in self.futures:
|
||||
future.result(timeout=5)
|
||||
finally:
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
return list(self.events)
|
||||
|
||||
|
||||
def metrics(events, started, ended, target, action_done, scenario):
|
||||
window = [event for event in events if started <= event[0] <= ended]
|
||||
check(bool(window), "no traffic was recorded during the action")
|
||||
counts = {}
|
||||
for kind in ("read", "write"):
|
||||
records = [event for event in window if event[1] == kind]
|
||||
successes = [event for event in records if event[2]]
|
||||
check(bool(successes), f"no successful {kind}s during the action")
|
||||
survivor = next((event[0] for event in successes
|
||||
if event[0] >= started and event[3] != target), None)
|
||||
check(survivor is not None, f"no {kind} served by surviving replica")
|
||||
after_action = next((event[0] for event in successes
|
||||
if event[0] >= action_done), None)
|
||||
check(after_action is not None, f"no {kind} succeeded after the action")
|
||||
counts[kind] = {
|
||||
"successful": len(successes),
|
||||
"errors": len(records) - len(successes),
|
||||
"successful_rps": round(len(successes) / (ended - started), 3),
|
||||
"first_survivor_seconds": round(survivor - started, 3),
|
||||
"first_success_after_action_seconds": round(after_action - started, 3),
|
||||
}
|
||||
after = [event for event in window if event[0] >= action_done + 1 and event[2]]
|
||||
check(bool(after), "no successful traffic after the action")
|
||||
if scenario == "failover":
|
||||
check(all(event[3] != target for event in after),
|
||||
"stopped replica still served traffic after the action")
|
||||
target_times = [event[0] for event in window
|
||||
if event[2] and event[3] == target]
|
||||
counts["last_target_response_seconds"] = (
|
||||
round(max(target_times) - started, 3) if target_times else None
|
||||
)
|
||||
for kind in ("read", "write"):
|
||||
check(counts[kind]["first_success_after_action_seconds"] <= 10,
|
||||
f"{kind} did not recover within 10 seconds")
|
||||
return counts
|
||||
|
||||
|
||||
def verify_history(client, actor, chat, original, traffic):
|
||||
messages = history(client, actor, chat)
|
||||
counts = Counter(message["text"] for message in messages)
|
||||
check(any(message == original for message in messages),
|
||||
"confirmed pre-action message disappeared")
|
||||
for text in traffic.attempted:
|
||||
check(counts[text] <= 1, "an uncertain retry created a duplicate")
|
||||
for text in traffic.confirmed:
|
||||
check(counts[text] == 1, "confirmed write disappeared")
|
||||
|
||||
|
||||
def resource_limits():
|
||||
expected = {
|
||||
"nginx": (500_000_000, 128 * 1024**2),
|
||||
"migrate": (1_000_000_000, 512 * 1024**2),
|
||||
"postgres": (1_000_000_000, 512 * 1024**2),
|
||||
"redis": (500_000_000, 256 * 1024**2),
|
||||
}
|
||||
ids = containers("api")
|
||||
check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas")
|
||||
api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids))
|
||||
observed = {"api": []}
|
||||
for container_id in ids:
|
||||
actual = (
|
||||
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
|
||||
int(inspect(container_id, "{{.HostConfig.Memory}}")),
|
||||
)
|
||||
check(actual == api_limits, "API resource limits differ from fixed budget")
|
||||
observed["api"].append({"instance": instance(container_id),
|
||||
"cpus": actual[0] / 1e9,
|
||||
"memory_mib": actual[1] // 1024**2})
|
||||
for service, limits in expected.items():
|
||||
container_id = container(service)
|
||||
actual = (
|
||||
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
|
||||
int(inspect(container_id, "{{.HostConfig.Memory}}")),
|
||||
)
|
||||
check(actual == limits, f"{service} resource limits differ from compose.yaml")
|
||||
observed[service] = {"cpus": actual[0] / 1e9,
|
||||
"memory_mib": actual[1] // 1024**2}
|
||||
return observed
|
||||
|
||||
|
||||
def run(args):
|
||||
check(2 <= args.concurrency <= 8, "concurrency must be 2..8")
|
||||
check(3 <= args.observe_seconds <= 60,
|
||||
"observe-seconds must be 3..60")
|
||||
result = {"scenario": args.scenario, "resources": resource_limits(),
|
||||
"actions": []}
|
||||
if args.scenario == "resources":
|
||||
return result
|
||||
targets = containers("api")
|
||||
check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile")
|
||||
|
||||
client = Client(args.base_url, timeout=10)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
original = client.message(alice, chat, "Before grade 4 actions")["json"]
|
||||
both_observed(client, alice, {instance(item) for item in targets})
|
||||
migrate_id = container("migrate")
|
||||
traffic = Traffic(args.base_url, alice, chat, args.concurrency)
|
||||
traffic.start()
|
||||
try:
|
||||
for before_id in targets:
|
||||
target = instance(before_id)
|
||||
healthy(before_id)
|
||||
before_ids = set(containers("api"))
|
||||
both_observed(client, alice, {instance(item) for item in before_ids})
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
if args.scenario == "failover":
|
||||
docker("stop", "--time", "30", before_id, timeout=40)
|
||||
check(inspect(before_id, "{{.State.Status}}") == "exited",
|
||||
"target API container was not stopped")
|
||||
else:
|
||||
docker("stop", "--time", "30", before_id, timeout=40)
|
||||
docker("rm", before_id)
|
||||
compose("up", "-d", "--no-build", "--no-deps",
|
||||
"--scale", "api=2", "api", timeout=90)
|
||||
replacements = set(containers("api")) - before_ids
|
||||
check(len(replacements) == 1, "one API container was not replaced")
|
||||
healthy(next(iter(replacements)))
|
||||
action_done = time.monotonic()
|
||||
time.sleep(args.observe_seconds)
|
||||
ended = time.monotonic()
|
||||
item = {"target": target,
|
||||
"action_seconds": round(action_done - started, 3),
|
||||
"window_seconds": round(ended - started, 3),
|
||||
"traffic": metrics(traffic.snapshot(), started, ended,
|
||||
target, action_done, args.scenario)}
|
||||
if args.scenario == "failover":
|
||||
docker("start", before_id)
|
||||
healthy(before_id)
|
||||
both_observed(client, alice,
|
||||
{instance(item) for item in containers("api")})
|
||||
result["actions"].append(item)
|
||||
check(container("migrate") == migrate_id,
|
||||
"migration container changed during API replacement")
|
||||
finally:
|
||||
traffic.finish()
|
||||
# Restore both API replicas after a failed test; never remove volumes.
|
||||
try:
|
||||
compose("up", "-d", "--no-build", "--no-deps",
|
||||
"--scale", "api=2", "api")
|
||||
except Failure:
|
||||
print("WARNING: restore API replicas manually", file=sys.stderr)
|
||||
verify_history(client, bob, chat, original, traffic)
|
||||
result["confirmed_writes"] = len(traffic.confirmed)
|
||||
result["attempted_unique_writes"] = len(traffic.attempted)
|
||||
result["migration_container_unchanged"] = True
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--scenario", choices=("resources", "failover", "rolling"),
|
||||
required=True)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--observe-seconds", type=float, default=10)
|
||||
parser.add_argument("--output", help="optional JSON result path without tokens")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = run(args)
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(output + "\n", encoding="utf-8")
|
||||
print(output)
|
||||
print("PASS: grade 4 scenario")
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import net from 'node:net';
|
||||
|
||||
export async function faultProxy(connectionUrl, defaultPort) {
|
||||
const upstreamUrl = new URL(connectionUrl);
|
||||
const host = upstreamUrl.hostname.replace(/^\[|\]$/g, '');
|
||||
const port = Number(upstreamUrl.port || defaultPort);
|
||||
let mode = 'online';
|
||||
const sockets = new Set();
|
||||
const track = (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on('error', () => {});
|
||||
socket.on('close', () => sockets.delete(socket));
|
||||
return socket;
|
||||
};
|
||||
const server = net.createServer((client) => {
|
||||
track(client);
|
||||
if (mode === 'cut') {
|
||||
client.destroy();
|
||||
return;
|
||||
}
|
||||
const upstream = track(net.connect({ host, port }));
|
||||
client.on('data', (chunk) => {
|
||||
if (mode === 'online' && !upstream.destroyed) upstream.write(chunk);
|
||||
});
|
||||
upstream.on('data', (chunk) => {
|
||||
if (mode === 'online' && !client.destroyed) client.write(chunk);
|
||||
});
|
||||
client.on('close', () => upstream.destroy());
|
||||
upstream.on('close', () => client.destroy());
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
server.off('error', reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const relayUrl = new URL(connectionUrl);
|
||||
relayUrl.hostname = '127.0.0.1';
|
||||
relayUrl.port = String(server.address().port);
|
||||
return {
|
||||
url: relayUrl.toString(),
|
||||
setMode(nextMode) {
|
||||
assert.ok(['online', 'cut', 'hang'].includes(nextMode));
|
||||
mode = nextMode;
|
||||
if (mode === 'cut') {
|
||||
for (const socket of sockets) socket.destroy();
|
||||
}
|
||||
},
|
||||
async close() {
|
||||
for (const socket of sockets) socket.destroy();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export async function messageEpoch(prisma, chatId) {
|
||||
const [row] = await prisma.$queryRawUnsafe(
|
||||
'SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM public.chat_message_epochs WHERE chat_id = $1::uuid',
|
||||
chatId,
|
||||
);
|
||||
return row.epoch;
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
test(
|
||||
'message idempotency is shared by two API instances',
|
||||
{ timeout: 120000 },
|
||||
async (t) => {
|
||||
assert.ok(
|
||||
process.env.DATABASE_URL,
|
||||
'Set DATABASE_URL and apply migrations',
|
||||
);
|
||||
const apps = [];
|
||||
let db;
|
||||
let chatId;
|
||||
const userIds = [];
|
||||
t.after(async () => {
|
||||
if (db && chatId) {
|
||||
await db.$executeRawUnsafe(
|
||||
'DELETE FROM message_idempotency WHERE chat_id = $1::uuid',
|
||||
chatId,
|
||||
);
|
||||
await db.chat.deleteMany({ where: { id: chatId } });
|
||||
}
|
||||
if (db && userIds.length) {
|
||||
await db.user.deleteMany({ where: { id: { in: userIds } } });
|
||||
}
|
||||
for (const app of apps) await app.close();
|
||||
});
|
||||
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
await app.getHttpAdapter().getInstance().ready();
|
||||
apps.push(app);
|
||||
}
|
||||
db = apps[0].get(PrismaService);
|
||||
const servers = apps.map((app) => app.getHttpAdapter().getInstance());
|
||||
const call = (index, method, url, token, payload, key) =>
|
||||
servers[index].inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
...(token ? { authorization: `Bearer ${token}` } : {}),
|
||||
...(key === undefined ? {} : { 'idempotency-key': key }),
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
});
|
||||
const register = async (label) => {
|
||||
const username = `idem_${label}_${randomUUID().replaceAll('-', '').slice(0, 12)}`;
|
||||
const password = `T3st-${randomUUID()}`;
|
||||
const response = await call(0, 'POST', '/api/v1/users', undefined, {
|
||||
username,
|
||||
display_name: label,
|
||||
password,
|
||||
});
|
||||
assert.equal(response.statusCode, 201, response.body);
|
||||
const user = response.json();
|
||||
userIds.push(user.id);
|
||||
const login = await servers[0].inject({
|
||||
method: 'POST',
|
||||
url: '/api/v1/auth/sessions',
|
||||
headers: {
|
||||
authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`,
|
||||
},
|
||||
});
|
||||
assert.equal(login.statusCode, 201, login.body);
|
||||
return { user, token: login.json().token };
|
||||
};
|
||||
|
||||
const owner = await register('owner');
|
||||
const outsider = await register('outsider');
|
||||
const created = await call(0, 'POST', '/api/v1/chats', owner.token, {
|
||||
title: 'Idempotency test',
|
||||
member_ids: [owner.user.id],
|
||||
});
|
||||
assert.equal(created.statusCode, 201, created.body);
|
||||
chatId = created.json().id;
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
const key = randomUUID();
|
||||
const body = { text: 'One message' };
|
||||
|
||||
assert.equal(
|
||||
(await call(0, 'POST', path, owner.token, body)).statusCode,
|
||||
400,
|
||||
);
|
||||
assert.equal(
|
||||
(await call(0, 'POST', path, owner.token, body, 'bad key')).statusCode,
|
||||
400,
|
||||
);
|
||||
const first = await call(0, 'POST', path, owner.token, body, key);
|
||||
assert.equal(first.statusCode, 201, first.body);
|
||||
const replay = await call(1, 'POST', path, owner.token, body, key);
|
||||
assert.equal(replay.statusCode, 201, replay.body);
|
||||
assert.deepEqual(replay.json(), first.json());
|
||||
assert.equal(
|
||||
(await call(1, 'POST', path, owner.token, { text: 'Changed' }, key))
|
||||
.statusCode,
|
||||
409,
|
||||
);
|
||||
assert.equal(
|
||||
(await call(1, 'POST', path, outsider.token, body, key)).statusCode,
|
||||
404,
|
||||
);
|
||||
|
||||
const concurrentKey = randomUUID();
|
||||
const concurrent = await Promise.all(
|
||||
Array.from({ length: 12 }, (_, index) =>
|
||||
call(
|
||||
index % 2,
|
||||
'POST',
|
||||
path,
|
||||
owner.token,
|
||||
{ text: 'Concurrent' },
|
||||
concurrentKey,
|
||||
),
|
||||
),
|
||||
);
|
||||
for (const response of concurrent) {
|
||||
assert.equal(response.statusCode, 201, response.body);
|
||||
assert.deepEqual(response.json(), concurrent[0].json());
|
||||
}
|
||||
assert.equal(await db.message.count({ where: { chatId } }), 2);
|
||||
const [record] = await db.$queryRawUnsafe(
|
||||
'SELECT expires_at AS "expiresAt" FROM message_idempotency WHERE user_id = $1::uuid AND chat_id = $2::uuid AND "key" = $3',
|
||||
owner.user.id,
|
||||
chatId,
|
||||
concurrentKey,
|
||||
);
|
||||
assert.ok(record.expiresAt.getTime() - Date.now() >= 23 * 3600 * 1000);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,137 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
test('message keyset pagination: timestamp ties, chat isolation and deleted cursor row', async (t) => {
|
||||
assert.ok(process.env.DATABASE_URL);
|
||||
const previousCacheUrl = process.env.CACHE_URL;
|
||||
process.env.CACHE_URL = '';
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
const prisma = app.get(PrismaService);
|
||||
let userId;
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
} finally {
|
||||
await app.close();
|
||||
if (previousCacheUrl === undefined) delete process.env.CACHE_URL;
|
||||
else process.env.CACHE_URL = previousCacheUrl;
|
||||
}
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `page_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic pagination test',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const chat = await prisma.chat.create({
|
||||
data: {
|
||||
title: `Pagination ${i}`,
|
||||
members: { create: { userId, position: 0 } },
|
||||
},
|
||||
});
|
||||
chatIds.push(chat.id);
|
||||
}
|
||||
const [chatId, otherChatId] = chatIds;
|
||||
const rows = Array.from({ length: 140 }, (_, i) => ({
|
||||
id: randomUUID(),
|
||||
chatId,
|
||||
senderId: userId,
|
||||
text: `Message ${i}`,
|
||||
createdAt: new Date(Date.UTC(2026, 0, 1) + Math.floor(i / 10) * 1000),
|
||||
}));
|
||||
await prisma.message.createMany({
|
||||
data: [
|
||||
...rows,
|
||||
...rows.map((row) => ({
|
||||
...row,
|
||||
id: randomUUID(),
|
||||
chatId: otherChatId,
|
||||
text: 'Other chat',
|
||||
})),
|
||||
],
|
||||
});
|
||||
const expected = [...rows]
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.createdAt - b.createdAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0),
|
||||
)
|
||||
.map((row) => ({
|
||||
id: row.id,
|
||||
chat_id: chatId,
|
||||
sender_id: userId,
|
||||
text: row.text,
|
||||
created_at: row.createdAt.toISOString(),
|
||||
}));
|
||||
const request = (cursor, selectedChat = chatId) =>
|
||||
server.inject({
|
||||
method: 'GET',
|
||||
url: `/api/v1/chats/${selectedChat}/messages?limit=7${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`,
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
const seen = [];
|
||||
let cursor;
|
||||
let firstCursor;
|
||||
do {
|
||||
const response = await request(cursor);
|
||||
assert.equal(response.statusCode, 200);
|
||||
const page = response.json();
|
||||
assert.ok(page.items.length <= 7);
|
||||
seen.push(...page.items);
|
||||
if (!firstCursor) firstCursor = page.next_cursor;
|
||||
cursor = page.next_cursor;
|
||||
assert.ok(
|
||||
seen.length <= rows.length,
|
||||
'pagination must terminate without duplicates',
|
||||
);
|
||||
} while (cursor);
|
||||
assert.deepEqual(seen, expected);
|
||||
const anchor = JSON.parse(
|
||||
Buffer.from(firstCursor, 'base64url').toString('utf8'),
|
||||
);
|
||||
await prisma.message.delete({ where: { id: anchor.id } });
|
||||
const afterDeletion = await request(firstCursor);
|
||||
assert.equal(afterDeletion.statusCode, 200);
|
||||
assert.deepEqual(afterDeletion.json().items, expected.slice(7, 14));
|
||||
assert.equal((await request(firstCursor, otherChatId)).statusCode, 400);
|
||||
const pastEnd = Buffer.from(
|
||||
JSON.stringify({
|
||||
scope: `messages:${chatId}`,
|
||||
id: randomUUID(),
|
||||
created_at: '2027-01-01T00:00:00.000Z',
|
||||
}),
|
||||
).toString('base64url');
|
||||
const empty = await request(pastEnd);
|
||||
assert.equal(empty.statusCode, 200);
|
||||
assert.deepEqual(empty.json(), { items: [], next_cursor: null });
|
||||
});
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# A barrier makes sequential probes fail: all four requests must start together.
|
||||
set -eu
|
||||
sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh
|
||||
. /tmp/readiness-under-test.sh
|
||||
mkdir -p /tmp/readiness-barrier
|
||||
cycle=0
|
||||
getent() {
|
||||
for ip in 1 2 3 4; do printf '192.0.2.%s STREAM api\n' "$ip"; done
|
||||
}
|
||||
wget() {
|
||||
for arg do
|
||||
case "$arg" in http://*) address=${arg#http://}; address=${address%%:*} ;; esac
|
||||
done
|
||||
touch "/tmp/readiness-barrier/$address"
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 50 ]; do
|
||||
if [ "$(find /tmp/readiness-barrier -type f | wc -l)" -eq 4 ]; then return 0; fi
|
||||
attempt=$((attempt + 1))
|
||||
command sleep 0.01
|
||||
done
|
||||
return 1
|
||||
}
|
||||
nginx() { return 0; }
|
||||
sleep() {
|
||||
if [ "$cycle" -eq 2 ]; then
|
||||
count=$(grep -c 'server 192.0.2.' "$upstreams" || true)
|
||||
[ "$count" -eq 4 ] || { printf 'FAIL: readiness probes did not start in parallel\n'; exit 1; }
|
||||
printf 'PASS: four concurrent probes admitted after two successful rounds\n'
|
||||
exit 0
|
||||
fi
|
||||
rm -f /tmp/readiness-barrier/*
|
||||
cycle=$((cycle + 1))
|
||||
}
|
||||
check_ready
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
# Run in a disposable nginx container; never against a running load balancer.
|
||||
set -eu
|
||||
|
||||
sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh
|
||||
. /tmp/readiness-under-test.sh
|
||||
|
||||
cycle=0
|
||||
reloads=0
|
||||
getent() { printf '192.0.2.1 STREAM api\n'; }
|
||||
wget() {
|
||||
case "$cycle" in
|
||||
1|2|4|7|8) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
nginx() {
|
||||
if [ "$1" = '-s' ]; then reloads=$((reloads + 1)); fi
|
||||
}
|
||||
sleep() {
|
||||
# Check the previous cycle before advancing to the next probe.
|
||||
if [ "$cycle" -gt 0 ]; then
|
||||
case "$cycle" in
|
||||
2|3|4|5|8) expected=1 ;;
|
||||
*) expected=0 ;;
|
||||
esac
|
||||
actual=0
|
||||
if grep -q 'server 192.0.2.1:8080 ' "$upstreams"; then actual=1; fi
|
||||
if [ "$actual" -ne "$expected" ]; then
|
||||
printf 'FAIL: cycle %s, expected readiness %s, got %s\n' "$cycle" "$expected" "$actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
cycle=$((cycle + 1))
|
||||
if [ "$cycle" -eq 9 ]; then
|
||||
[ "$reloads" -eq 3 ] || exit 1
|
||||
printf 'PASS: admission, isolated failure, recovery, removal and re-admission; 3 reloads\n'
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
check_ready
|
||||
@@ -0,0 +1,268 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { spawn, execFileSync } = require('node:child_process');
|
||||
const ts = require('typescript');
|
||||
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const resultPrefix = process.env.EXPERIMENT_PREFIX ?? 'cache-strategy';
|
||||
assert.match(resultPrefix, /^[a-z0-9-]+$/);
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-no-epochs-'));
|
||||
const hashes = {};
|
||||
const volumes = [];
|
||||
const replace = (source, from, to) => {
|
||||
assert.ok(source.includes(from), `Missing replacement: ${from}`);
|
||||
assert.equal(
|
||||
source.indexOf(from),
|
||||
source.lastIndexOf(from),
|
||||
`Ambiguous replacement: ${from}`,
|
||||
);
|
||||
return source.replace(from, to);
|
||||
};
|
||||
const section = (source, from, to, replacement) => {
|
||||
const start = source.indexOf(from);
|
||||
const end = source.indexOf(to, start);
|
||||
assert.ok(start !== -1 && end > start, `Missing section: ${from}`);
|
||||
return source.slice(0, start) + replacement + source.slice(end);
|
||||
};
|
||||
const digest = (value) => createHash('sha256').update(value).digest('hex');
|
||||
|
||||
const transforms = {
|
||||
'Chat/chat.service.ts': (source) => {
|
||||
source = section(
|
||||
source,
|
||||
' await tx.$executeRaw(',
|
||||
' return created;',
|
||||
'',
|
||||
);
|
||||
source = replace(source, ' epoch: string,\n', '');
|
||||
source = replace(source, ' epoch,\n', '');
|
||||
source = section(
|
||||
source,
|
||||
' const [membership] =',
|
||||
' if (!membership)',
|
||||
' const [membership] = await this.prisma.$queryRaw<{ id: string }[]>(Prisma.sql`\n' +
|
||||
' SELECT cm.chat_id AS id FROM chat_members AS cm\n' +
|
||||
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
|
||||
' `);\n',
|
||||
);
|
||||
source = section(
|
||||
source,
|
||||
' const [message] =',
|
||||
' if (!message)',
|
||||
' const [message] = await this.prisma.$queryRaw<StoredMessage[]>(Prisma.sql`\n' +
|
||||
' INSERT INTO messages (id, chat_id, sender_id, text)\n' +
|
||||
' SELECT ${randomUUID()}::uuid, cm.chat_id, ${userId}::uuid, ${text}\n' +
|
||||
' FROM chat_members AS cm\n' +
|
||||
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
|
||||
' RETURNING id, chat_id AS "chatId", sender_id AS "senderId",\n' +
|
||||
' text, created_at AS "createdAt"\n' +
|
||||
' `);\n',
|
||||
);
|
||||
return replace(source, ' chat.messagesEpoch.toString(),\n', '');
|
||||
},
|
||||
'Auth/auth.service.ts': (source) => {
|
||||
source = source.replace(
|
||||
/COALESCE\(\(SELECT SUM\(e\.epoch\)[\s\S]*?0\)::bigint AS "messagesEpoch"/g,
|
||||
'0::bigint AS "messagesEpoch"',
|
||||
);
|
||||
return source
|
||||
.replaceAll('u.chats_epoch', '0::bigint')
|
||||
.replaceAll('c.messages_epoch', '0::bigint');
|
||||
},
|
||||
'User/user.guard.ts': (source) => {
|
||||
source = replace(source, ' currentChatsEpoch: string;\n', '');
|
||||
return replace(
|
||||
source,
|
||||
' request.currentChatsEpoch = session.chatsEpoch;\n',
|
||||
'',
|
||||
);
|
||||
},
|
||||
'Chat/chat.controller.ts': (source) =>
|
||||
replace(source, ' request.currentChatsEpoch,\n', ''),
|
||||
'cache/cache.service.ts': (source) => {
|
||||
source = replace(source, ' epoch?: string,\n', '');
|
||||
return section(
|
||||
source,
|
||||
' if (epoch !== undefined)',
|
||||
' const versionKey =',
|
||||
'',
|
||||
);
|
||||
},
|
||||
'User/user.service.ts': (source) => {
|
||||
source = replace(
|
||||
source,
|
||||
'const user = await this.prisma.user.findUnique({ where: { id } });',
|
||||
'const user = await this.prisma.user.findUnique({ where: { id }, select: { id: true, username: true, displayName: true, createdAt: true } });',
|
||||
);
|
||||
return source;
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
for (const [relative, transform] of Object.entries(transforms)) {
|
||||
const original = fs.readFileSync(path.join(root, 'src', relative), 'utf8');
|
||||
const source = transform(original.replace(/\r\n/g, '\n'));
|
||||
assert.ok(
|
||||
!/messages_epoch|chats_epoch|chat_message_epochs/.test(source),
|
||||
relative,
|
||||
);
|
||||
const compiled = ts.transpileModule(source, {
|
||||
compilerOptions: {
|
||||
target: ts.ScriptTarget.ES2023,
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
experimentalDecorators: true,
|
||||
emitDecoratorMetadata: true,
|
||||
},
|
||||
reportDiagnostics: true,
|
||||
});
|
||||
assert.equal(
|
||||
compiled.diagnostics.filter(
|
||||
(item) => item.category === ts.DiagnosticCategory.Error,
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
const sourcePath = path.join(directory, relative);
|
||||
const compiledPath = sourcePath.replace(/\.ts$/, '.js');
|
||||
fs.mkdirSync(path.dirname(sourcePath), { recursive: true });
|
||||
fs.writeFileSync(sourcePath, source);
|
||||
fs.writeFileSync(compiledPath, compiled.outputText);
|
||||
hashes[relative] = {
|
||||
original_source: digest(original),
|
||||
experiment_source: digest(source),
|
||||
experiment_js: digest(compiled.outputText),
|
||||
};
|
||||
volumes.push({
|
||||
type: 'bind',
|
||||
source: compiledPath,
|
||||
target: `/app/dist/${relative.replace(/\.ts$/, '.js')}`,
|
||||
read_only: true,
|
||||
});
|
||||
}
|
||||
let benchmark = fs
|
||||
.readFileSync(path.join(root, 'test/cache-benchmark.mjs'), 'utf8')
|
||||
.replace(/\r\n/g, '\n');
|
||||
benchmark = replace(
|
||||
benchmark,
|
||||
'process.env.CACHE_URL = proxy.url;',
|
||||
'delete process.env.CACHE_URL;',
|
||||
);
|
||||
benchmark = section(
|
||||
benchmark,
|
||||
' await prisma.chatMessageEpoch.createMany(',
|
||||
' const path =',
|
||||
'',
|
||||
);
|
||||
benchmark = section(
|
||||
benchmark,
|
||||
' const second = await start();',
|
||||
' report.completed = true;',
|
||||
' assert.equal(first.app.get(CacheService).redis, undefined);\n' +
|
||||
' for (const chatId of chatIds) {\n' +
|
||||
' assert.equal(await messageEpoch(prisma, chatId), 0n);\n' +
|
||||
' }\n' +
|
||||
' assert.equal((await prisma.user.findUniqueOrThrow({ where: { id: userId } })).chatsEpoch, 0n);\n' +
|
||||
' for (const chatId of chatIds.slice(1)) {\n' +
|
||||
' const page = await request("GET", `/api/v1/chats/${chatId}/messages`);\n' +
|
||||
' assert.equal(page.items.length, 50);\n' +
|
||||
' assert.equal(new Set(page.items.map(message => message.id)).size, 50);\n' +
|
||||
' }\n' +
|
||||
' report.correctness = { no_redis_client: true, epochs_unchanged: true, read_after_write: true };\n' +
|
||||
' }\n',
|
||||
);
|
||||
const benchmarkPath = path.join(directory, 'cache-benchmark.mjs');
|
||||
fs.writeFileSync(benchmarkPath, benchmark);
|
||||
volumes.push({
|
||||
type: 'bind',
|
||||
source: benchmarkPath,
|
||||
target: '/experiment/cache-benchmark.mjs',
|
||||
read_only: true,
|
||||
});
|
||||
volumes.push({
|
||||
type: 'bind',
|
||||
source: path.join(root, 'test/helpers'),
|
||||
target: '/experiment/helpers',
|
||||
read_only: true,
|
||||
});
|
||||
const override = path.join(directory, 'compose.no-epochs.yaml');
|
||||
fs.writeFileSync(
|
||||
override,
|
||||
JSON.stringify({ services: { benchmark: { volumes } } }, null, 2),
|
||||
);
|
||||
const args = [
|
||||
'compose',
|
||||
'-f',
|
||||
'compose.yaml',
|
||||
'-f',
|
||||
'test/compose.benchmark.yaml',
|
||||
'-f',
|
||||
override,
|
||||
'run',
|
||||
'--rm',
|
||||
'--no-deps',
|
||||
'--pull',
|
||||
'never',
|
||||
'benchmark',
|
||||
'node',
|
||||
'/experiment/cache-benchmark.mjs',
|
||||
'after',
|
||||
`/results/${resultPrefix}-no-redis-no-epochs.json`,
|
||||
...process.argv.slice(2),
|
||||
];
|
||||
const metadata = {
|
||||
recorded_at: new Date().toISOString(),
|
||||
image: execFileSync(
|
||||
'docker',
|
||||
['image', 'inspect', 'mtusi2026lab2-fork-backend', '--format', '{{.Id}}'],
|
||||
{ encoding: 'utf8' },
|
||||
).trim(),
|
||||
modified_sources: hashes,
|
||||
command: ['docker', ...args],
|
||||
implementation:
|
||||
'Separate temporary source copies compiled with TypeScript and mounted over six application modules. Workspace sources and running backend remain unchanged.',
|
||||
schema:
|
||||
'Epoch storage remains in the database; experiment API paths do not update counters or read authoritative epoch values.',
|
||||
redis:
|
||||
'Application starts without CACHE_URL. Observer/proxy are retained only for benchmark setup and cleanup.',
|
||||
full_result: `${resultPrefix}-full.json`,
|
||||
experiment_result: `${resultPrefix}-no-redis-no-epochs.json`,
|
||||
};
|
||||
const child = spawn('docker', args, {
|
||||
cwd: root,
|
||||
stdio: 'inherit',
|
||||
env: process.env,
|
||||
});
|
||||
child.once('error', finish);
|
||||
child.once('exit', (code, signal) => finish(null, code, signal));
|
||||
let finished = false;
|
||||
function finish(error, code, signal) {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
metadata.exit_code = error ? 1 : code;
|
||||
metadata.signal = signal;
|
||||
metadata.sources_unchanged = Object.entries(hashes).every(
|
||||
([relative, hash]) =>
|
||||
digest(fs.readFileSync(path.join(root, 'src', relative), 'utf8')) ===
|
||||
hash.original_source,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(root, `evidence/${resultPrefix}-experiment.json`),
|
||||
JSON.stringify(metadata, null, 2) + '\n',
|
||||
);
|
||||
cleanup();
|
||||
if (error) console.error(error.message);
|
||||
process.exitCode = error ? 1 : (code ?? 1);
|
||||
}
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
throw error;
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
const resolved = path.resolve(directory);
|
||||
assert.equal(path.dirname(resolved), path.resolve(os.tmpdir()));
|
||||
assert.ok(path.basename(resolved).startsWith('lab2-no-epochs-'));
|
||||
fs.rmSync(resolved, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { argon2 } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { PasswordService } = require('../dist/Auth/password.service');
|
||||
const password = '🔐'.repeat(40) + ':Синтетический пароль\0';
|
||||
|
||||
test('Argon2id only: PHC format, independent derivation, unique salts, Unicode and invalid hashes', async () => {
|
||||
const service = new PasswordService();
|
||||
const first = await service.hash(password);
|
||||
const second = await service.hash(password);
|
||||
assert.notEqual(
|
||||
first,
|
||||
second,
|
||||
'same password must have independently salted hashes',
|
||||
);
|
||||
assert.match(
|
||||
first,
|
||||
/^\$argon2id\$v=19\$m=19456,t=2,p=1\$[A-Za-z0-9+/]{22}\$[A-Za-z0-9+/]{43}$/,
|
||||
);
|
||||
assert.ok(!first.includes(password));
|
||||
const parts = first.split('$');
|
||||
const independent = await new Promise((resolve, reject) =>
|
||||
argon2(
|
||||
'argon2id',
|
||||
{
|
||||
message: password,
|
||||
nonce: Buffer.from(parts[4], 'base64'),
|
||||
memory: 19456,
|
||||
passes: 2,
|
||||
parallelism: 1,
|
||||
tagLength: 32,
|
||||
},
|
||||
(error, key) => (error ? reject(error) : resolve(key)),
|
||||
),
|
||||
);
|
||||
assert.deepEqual(Buffer.from(parts[5], 'base64'), independent);
|
||||
assert.equal(await service.verify(password, first), true);
|
||||
assert.equal(
|
||||
await service.verify(password.replace('🔐', '🔑'), first),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await service.verify(password.slice(0, -1), first),
|
||||
false,
|
||||
'NUL must not be silently truncated',
|
||||
);
|
||||
for (const invalid of [
|
||||
null,
|
||||
'',
|
||||
'scrypt$131072$8$1$unsupported$unsupported',
|
||||
'$argon2id$v=19$m=999999999,t=999999999,p=1$x$x',
|
||||
first.replace('v=19', 'v=16'),
|
||||
first.replace('argon2id', 'argon2i'),
|
||||
first + '$extra',
|
||||
first.replace(parts[5], 'a'),
|
||||
first.replace(parts[4], parts[4] + '='),
|
||||
]) {
|
||||
assert.equal(await service.verify(password, invalid), false);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,349 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
async function start() {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
try {
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
return { app, server, prisma: app.get(PrismaService) };
|
||||
} catch (error) {
|
||||
await app.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
test('PostgreSQL storage: constraints, pagination, access and application restart', async (t) => {
|
||||
assert.ok(
|
||||
process.env.DATABASE_URL,
|
||||
'Set DATABASE_URL and apply migrations before running this test',
|
||||
);
|
||||
let context = await start();
|
||||
const userIds = [];
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
try {
|
||||
await context.prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
await context.prisma.user.deleteMany({ where: { id: { in: userIds } } });
|
||||
} finally {
|
||||
await context.app.close();
|
||||
}
|
||||
});
|
||||
const suffix = randomUUID().replaceAll('-', '').slice(0, 16);
|
||||
const password = '🔐'.repeat(40) + ':пароль с пробелами ';
|
||||
const tokens = new Map();
|
||||
const request = (method, url, payload, userId) =>
|
||||
context.server.inject({
|
||||
method,
|
||||
url,
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
headers: {
|
||||
...(userId ? { authorization: `Bearer ${tokens.get(userId)}` } : {}),
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
|
||||
const registrations = await Promise.all(
|
||||
Array.from({ length: 4 }, () =>
|
||||
request('POST', '/api/v1/users', {
|
||||
username: `race_${suffix}`,
|
||||
display_name: 'Race',
|
||||
password,
|
||||
}),
|
||||
),
|
||||
);
|
||||
for (const response of registrations) {
|
||||
if (response.statusCode === 201) userIds.push(response.json().id);
|
||||
}
|
||||
assert.deepEqual(
|
||||
registrations.map((response) => response.statusCode).sort(),
|
||||
[201, 409, 409, 409],
|
||||
);
|
||||
const userId = userIds[0];
|
||||
const outsider = await request('POST', '/api/v1/users', {
|
||||
username: `other_${suffix}`,
|
||||
display_name: 'Other',
|
||||
password,
|
||||
});
|
||||
assert.equal(outsider.statusCode, 201);
|
||||
userIds.push(outsider.json().id);
|
||||
|
||||
const login = (username, secret = password, payload) =>
|
||||
context.server.inject({
|
||||
method: 'POST',
|
||||
url: '/api/v1/auth/sessions',
|
||||
headers: {
|
||||
authorization: `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`,
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
});
|
||||
for (const user of [
|
||||
registrations.find((r) => r.statusCode === 201).json(),
|
||||
outsider.json(),
|
||||
]) {
|
||||
assert.deepEqual(Object.keys(user).sort(), [
|
||||
'created_at',
|
||||
'display_name',
|
||||
'id',
|
||||
'username',
|
||||
]);
|
||||
const response = await login(user.username);
|
||||
assert.equal(response.statusCode, 201);
|
||||
assert.equal(response.json().token_type, 'Bearer');
|
||||
assert.match(response.json().token, /^[A-Za-z0-9_-]{43}$/);
|
||||
assert.deepEqual(response.json().user, user);
|
||||
tokens.set(user.id, response.json().token);
|
||||
const stored = await context.prisma.user.findUniqueOrThrow({
|
||||
where: { id: user.id },
|
||||
});
|
||||
assert.match(stored.passwordHash, /^\$argon2id\$v=19\$m=19456,t=2,p=1\$/);
|
||||
assert.ok(!stored.passwordHash.includes(password));
|
||||
}
|
||||
const sessions = await context.prisma.session.findMany({ where: { userId } });
|
||||
assert.equal(sessions.length, 1);
|
||||
assert.match(sessions[0].tokenHash, /^[a-f0-9]{64}$/);
|
||||
assert.notEqual(sessions[0].tokenHash, tokens.get(userId));
|
||||
const wrong = await login(`race_${suffix}`, 'wrong-password');
|
||||
const unknown = await login(`missing_${suffix}`, 'wrong-password');
|
||||
assert.equal(wrong.statusCode, 401);
|
||||
assert.equal(unknown.statusCode, 401);
|
||||
assert.equal(wrong.json().error.message, unknown.json().error.message);
|
||||
assert.equal(
|
||||
wrong.headers['www-authenticate'],
|
||||
'Basic realm="messenger", charset="UTF-8"',
|
||||
);
|
||||
assert.equal((await login(`race_${suffix}`, password, {})).statusCode, 400);
|
||||
for (const invalid of [
|
||||
undefined,
|
||||
'short',
|
||||
' '.repeat(12),
|
||||
'🔐'.repeat(129),
|
||||
]) {
|
||||
assert.equal(
|
||||
(
|
||||
await request('POST', '/api/v1/users', {
|
||||
username: `bad_${suffix}`,
|
||||
display_name: 'Bad',
|
||||
...(invalid === undefined ? {} : { password: invalid }),
|
||||
})
|
||||
).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
for (const authorization of [
|
||||
undefined,
|
||||
'Bearer unknown',
|
||||
`Basic ${Buffer.from(`race_${suffix}:${password}`).toString('base64')}`,
|
||||
]) {
|
||||
const response = await context.server.inject({
|
||||
method: 'GET',
|
||||
url: '/api/v1/users/me',
|
||||
headers: {
|
||||
'x-user-id': userId,
|
||||
...(authorization ? { authorization } : {}),
|
||||
},
|
||||
});
|
||||
assert.equal(response.statusCode, 401);
|
||||
assert.equal(
|
||||
response.headers['www-authenticate'],
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
}
|
||||
const identity = await context.server.inject({
|
||||
method: 'GET',
|
||||
url: '/api/v1/users/me',
|
||||
headers: {
|
||||
authorization: `Bearer ${tokens.get(userId)}`,
|
||||
'x-user-id': outsider.json().id,
|
||||
},
|
||||
});
|
||||
assert.equal(identity.json().id, userId);
|
||||
|
||||
const ready = await request('GET', '/health/ready');
|
||||
assert.equal(ready.statusCode, 200);
|
||||
|
||||
const invalidChat = await request(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
{
|
||||
title: `invalid_${suffix}`,
|
||||
member_ids: [userId, randomUUID()],
|
||||
},
|
||||
userId,
|
||||
);
|
||||
assert.equal(invalidChat.statusCode, 400);
|
||||
assert.equal(
|
||||
await context.prisma.chat.count({ where: { title: `invalid_${suffix}` } }),
|
||||
0,
|
||||
);
|
||||
|
||||
const rolledBackId = randomUUID();
|
||||
chatIds.push(rolledBackId);
|
||||
await assert.rejects(
|
||||
context.prisma.$transaction(async (tx) => {
|
||||
await tx.chat.create({ data: { id: rolledBackId, title: 'Rollback' } });
|
||||
await tx.chatMember.create({
|
||||
data: { chatId: rolledBackId, userId: randomUUID(), position: 0 },
|
||||
});
|
||||
}),
|
||||
{ code: 'P2003' },
|
||||
);
|
||||
assert.equal(
|
||||
await context.prisma.chat.count({ where: { id: rolledBackId } }),
|
||||
0,
|
||||
);
|
||||
|
||||
const createdChat = await request(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
{ title: 'Persisted chat', member_ids: [userId] },
|
||||
userId,
|
||||
);
|
||||
assert.equal(createdChat.statusCode, 201);
|
||||
const chatId = createdChat.json().id;
|
||||
chatIds.push(chatId);
|
||||
const forbidden = await request(
|
||||
'GET',
|
||||
`/api/v1/chats/${chatId}`,
|
||||
undefined,
|
||||
outsider.json().id,
|
||||
);
|
||||
assert.equal(forbidden.statusCode, 404);
|
||||
const unauthorized = await request('GET', `/api/v1/chats/${chatId}`);
|
||||
assert.equal(unauthorized.statusCode, 401);
|
||||
|
||||
const messageIds = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const message = await request(
|
||||
'POST',
|
||||
`/api/v1/chats/${chatId}/messages`,
|
||||
{ text: `Message ${i}` },
|
||||
userId,
|
||||
);
|
||||
assert.equal(message.statusCode, 201);
|
||||
messageIds.push(message.json().id);
|
||||
}
|
||||
await context.prisma.message.updateMany({
|
||||
where: { chatId },
|
||||
data: { createdAt: new Date('2026-01-01T00:00:00.000Z') },
|
||||
});
|
||||
const first = await request(
|
||||
'GET',
|
||||
`/api/v1/chats/${chatId}/messages?limit=2`,
|
||||
undefined,
|
||||
userId,
|
||||
);
|
||||
assert.equal(first.statusCode, 200);
|
||||
assert.equal(first.json().items.length, 2);
|
||||
assert.ok(first.json().next_cursor);
|
||||
|
||||
await context.app.close();
|
||||
context = await start();
|
||||
const me = await request('GET', '/api/v1/users/me', undefined, userId);
|
||||
assert.equal(me.statusCode, 200);
|
||||
const chat = await request(
|
||||
'GET',
|
||||
`/api/v1/chats/${chatId}`,
|
||||
undefined,
|
||||
userId,
|
||||
);
|
||||
assert.equal(chat.statusCode, 200);
|
||||
assert.deepEqual(chat.json().member_ids, [userId]);
|
||||
const listed = await request(
|
||||
'GET',
|
||||
'/api/v1/chats?limit=1',
|
||||
undefined,
|
||||
userId,
|
||||
);
|
||||
assert.deepEqual(
|
||||
listed.json().items.map((item) => item.id),
|
||||
[chatId],
|
||||
);
|
||||
|
||||
const seen = first.json().items.map((message) => message.id);
|
||||
let cursor = first.json().next_cursor;
|
||||
while (cursor) {
|
||||
const page = await request(
|
||||
'GET',
|
||||
`/api/v1/chats/${chatId}/messages?limit=2&cursor=${cursor}`,
|
||||
undefined,
|
||||
userId,
|
||||
);
|
||||
assert.equal(page.statusCode, 200);
|
||||
assert.ok(page.json().items.length <= 2);
|
||||
seen.push(...page.json().items.map((message) => message.id));
|
||||
cursor = page.json().next_cursor;
|
||||
}
|
||||
assert.deepEqual(seen, [...messageIds].sort());
|
||||
assert.equal(new Set(seen).size, 5);
|
||||
|
||||
const logouts = await Promise.all(
|
||||
Array.from({ length: 2 }, () =>
|
||||
request('DELETE', '/api/v1/auth/sessions/current', undefined, userId),
|
||||
),
|
||||
);
|
||||
assert.deepEqual(logouts.map((r) => r.statusCode).sort(), [204, 401]);
|
||||
assert.equal(logouts.find((r) => r.statusCode === 204).body, '');
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
|
||||
401,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request(
|
||||
'DELETE',
|
||||
'/api/v1/auth/sessions/current',
|
||||
undefined,
|
||||
userId,
|
||||
)
|
||||
).statusCode,
|
||||
401,
|
||||
);
|
||||
await context.app.close();
|
||||
const originalTtl = process.env.SESSION_TTL_SECONDS;
|
||||
process.env.SESSION_TTL_SECONDS = '2';
|
||||
try {
|
||||
context = await start();
|
||||
} finally {
|
||||
if (originalTtl === undefined) delete process.env.SESSION_TTL_SECONDS;
|
||||
else process.env.SESSION_TTL_SECONDS = originalTtl;
|
||||
}
|
||||
const short = await login(`race_${suffix}`);
|
||||
assert.equal(short.statusCode, 201);
|
||||
tokens.set(userId, short.json().token);
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
|
||||
200,
|
||||
);
|
||||
await context.app.close();
|
||||
context = await start();
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(
|
||||
resolve,
|
||||
Math.max(0, Date.parse(short.json().expires_at) - Date.now()) + 100,
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
|
||||
401,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import vm from 'node:vm';
|
||||
import test from 'node:test';
|
||||
|
||||
test('pool observer preserves promise/callback results, errors and release', async () => {
|
||||
let timer;
|
||||
let now = 0;
|
||||
const records = [];
|
||||
const released = () => {};
|
||||
const client = { release: released };
|
||||
const failure = new Error('synthetic');
|
||||
class FakePool {
|
||||
totalCount = 2;
|
||||
idleCount = 1;
|
||||
waitingCount = 3;
|
||||
fail = false;
|
||||
connect(callback) {
|
||||
now += 5;
|
||||
if (callback) {
|
||||
callback(this.fail ? failure : undefined, this.fail ? undefined : client, released);
|
||||
return undefined;
|
||||
}
|
||||
return this.fail ? Promise.reject(failure) : Promise.resolve(client);
|
||||
}
|
||||
}
|
||||
const modules = {
|
||||
pg: { Pool: FakePool },
|
||||
'node:fs': {
|
||||
writeFileSync() {},
|
||||
appendFileSync(path, value) { records.push(JSON.parse(value)); },
|
||||
readFileSync(path) { return path.endsWith('memory.current') ? '1024' : 'usage_usec 10\n'; },
|
||||
},
|
||||
'node:perf_hooks': {
|
||||
performance: { now: () => now },
|
||||
monitorEventLoopDelay: () => ({ enable() {}, reset() {}, mean: 1, percentile: () => 1 }),
|
||||
},
|
||||
};
|
||||
vm.runInNewContext(readFileSync('scripts/diagnostics/pool.cjs', 'utf8'), {
|
||||
require: (name) => modules[name],
|
||||
process: { pid: 1, env: { INSTANCE_ID: 'test' },
|
||||
memoryUsage: () => ({ rss: 2048 }), cpuUsage: () => ({ user: 1, system: 2 }) },
|
||||
setInterval(callback) { timer = callback; return { unref() {} }; },
|
||||
Date,
|
||||
});
|
||||
const pool = new FakePool();
|
||||
assert.equal(await pool.connect(), client);
|
||||
assert.equal(pool.connect((error, value, release) => {
|
||||
assert.equal(error, undefined);
|
||||
assert.equal(value, client);
|
||||
assert.equal(release, released);
|
||||
}), undefined);
|
||||
pool.fail = true;
|
||||
await assert.rejects(pool.connect(), (error) => error === failure);
|
||||
pool.connect((error) => assert.equal(error, failure));
|
||||
now = 2000;
|
||||
timer();
|
||||
assert.equal(records[0].pools[0].acquisitions, 4);
|
||||
assert.equal(records[0].pools[0].errors, 2);
|
||||
assert.equal(records[0].pools[0].acquire_ms, 20);
|
||||
assert.equal(records[0].pools[0].interval_max_waiting, 3);
|
||||
assert.equal(records[0].pools[0].interval_max_busy, 1);
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const report = JSON.parse(
|
||||
await readFile('evidence/redis-comparison.json', 'utf8'),
|
||||
);
|
||||
assert.ok(report.completed);
|
||||
assert.ok(Object.values(report.correctness.read_after_write).every(Boolean));
|
||||
const lines = [
|
||||
'# Redis и работа без кэша приложения',
|
||||
'',
|
||||
`Дата UTC: ${report.recorded_at}. Node.js ${report.node}.`,
|
||||
'',
|
||||
'Один прогон на одном образе: три повтора каждого режима с чередованием порядка. Второй экземпляр API запущен без CACHE_URL: отключены все кэши приложения, включая профиль пользователя. PostgreSQL-эпохи и запись одним CTE сохранены в обоих режимах. Контейнер Redis не останавливался; экземпляр без кэша не создаёт Redis-клиент.',
|
||||
'',
|
||||
'RPS = суммарное число успешных запросов / суммарное время трёх повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее трёх отдельных p95, не перцентиль объединённой выборки. SQL/HTTP = суммарные SQL-вызовы / все HTTP-запросы; команды транзакций исключены.',
|
||||
'',
|
||||
'| Сценарий | Режим | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | CPU, % ядра | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
const aggregates = [];
|
||||
for (const [workload, label] of [
|
||||
['cold-history-burst', 'Холодная волна истории'],
|
||||
['warm-history', 'Прогретая история'],
|
||||
['writes-single-chat', 'Запись в один чат'],
|
||||
]) {
|
||||
for (const mode of ['redis', 'no-redis']) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.workload === `${workload}:${mode}`,
|
||||
);
|
||||
assert.equal(samples.length, 3);
|
||||
const sum = (select) =>
|
||||
samples.reduce((total, sample) => total + select(sample), 0);
|
||||
const successes = sum((sample) => sample.successes);
|
||||
const errors = sum((sample) => sample.errors);
|
||||
const seconds = sum((sample) => sample.elapsed_seconds);
|
||||
const aggregate = {
|
||||
workload,
|
||||
mode,
|
||||
successes,
|
||||
errors,
|
||||
rps: successes / seconds,
|
||||
mean_latency_ms:
|
||||
sum((sample) => sample.latency_ms.mean * sample.successes) / successes,
|
||||
mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length,
|
||||
sql_per_request:
|
||||
sum((sample) => sample.sql_queries) / (successes + errors),
|
||||
cpu_percent_one_core:
|
||||
sum(
|
||||
(sample) =>
|
||||
sample.process_cpu_percent_one_core * sample.elapsed_seconds,
|
||||
) / seconds,
|
||||
};
|
||||
aggregates.push(aggregate);
|
||||
lines.push(
|
||||
`| ${label} | ${mode === 'redis' ? 'Redis' : 'Без Redis'} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.cpu_percent_one_core.toFixed(2)} | ${errors} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'В этом прогоне кэш полезен для прогретого чтения истории: выше пропускная способность, ниже средняя задержка и меньше SQL-вызовов. Для холодной волны и записи преимущество по времени не показано. Первый повтор с Redis значительно медленнее остальных; средние включают его, поэтому вывод о стабильной скорости записи ограничен. Это сравнение режимов кэша на текущей реализации, а не отдельное измерение цены PostgreSQL-эпох.',
|
||||
'',
|
||||
'Нагрузка: 10 000 сообщений, первая страница limit=50, конкурентность 16, 3 с прогрева и 10 с чтения на режим/повтор, холодная волна 32 запроса, 512 записей в новый чат на режим/повтор. Реальный HTTP по loopback; синтетические пользователи и сессии обходят хеширование паролей. Проверены 512 сообщений и эпоха 512 после каждого сценария записи, а также чтение истории после записи.',
|
||||
'',
|
||||
'Оба API и генератор HTTP работают в одном Node-процессе. CPU включает генератор, API и инструментацию; RSS общий и не показывает разницу памяти между режимами. PostgreSQL и Redis разделяют Docker-host с другими сервисами. Это короткая нагрузка, не предел производительности; холодные SQL-данные не имитируются. Уменьшение SQL/HTTP показывает снижение числа запросов, но не доказывает пропорциональное снижение CPU PostgreSQL. Цена самих эпох отдельно не измеряется.',
|
||||
'',
|
||||
'Команда повторения:',
|
||||
'',
|
||||
'```powershell',
|
||||
'docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/redis-comparison.json --compare-redis',
|
||||
'node test/redis-comparison-summary.mjs',
|
||||
'```',
|
||||
'',
|
||||
'Сырые результаты и хеши измеренных модулей: [redis-comparison.json](redis-comparison.json).',
|
||||
);
|
||||
await writeFile(
|
||||
'evidence/redis-comparison-summary.md',
|
||||
lines.join('\n') + '\n',
|
||||
);
|
||||
await writeFile(
|
||||
'evidence/redis-comparison-averages.json',
|
||||
JSON.stringify(aggregates, null, 2) + '\n',
|
||||
);
|
||||
console.log(lines.slice(0, 16).join('\n'));
|
||||
@@ -0,0 +1,444 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import test from 'node:test';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
const pause = (milliseconds) =>
|
||||
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
test(
|
||||
'Auth and dependency failures remain bounded and fail closed',
|
||||
{ timeout: 120000 },
|
||||
async (t) => {
|
||||
assert.ok(
|
||||
process.env.DATABASE_URL,
|
||||
'Set DATABASE_URL and apply migrations',
|
||||
);
|
||||
assert.ok(
|
||||
process.env.CACHE_URL,
|
||||
'Set CACHE_URL to a running Redis instance',
|
||||
);
|
||||
const original = {
|
||||
DATABASE_URL: process.env.DATABASE_URL,
|
||||
CACHE_URL: process.env.CACHE_URL,
|
||||
CACHE_PREFIX: process.env.CACHE_PREFIX,
|
||||
SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS,
|
||||
};
|
||||
const prefix = `test:resilience:${randomUUID()}`;
|
||||
const db = await faultProxy(original.DATABASE_URL, 5432);
|
||||
const redis = await faultProxy(original.CACHE_URL, 6379);
|
||||
Object.assign(process.env, {
|
||||
DATABASE_URL: db.url,
|
||||
CACHE_URL: redis.url,
|
||||
CACHE_PREFIX: prefix,
|
||||
SESSION_TTL_SECONDS: '2',
|
||||
});
|
||||
const observer = new Redis(original.CACHE_URL, {
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 1000,
|
||||
maxRetriesPerRequest: 0,
|
||||
});
|
||||
observer.on('error', () => {});
|
||||
let app;
|
||||
let prisma;
|
||||
let userId;
|
||||
let chatId;
|
||||
t.after(async () => {
|
||||
db.setMode('online');
|
||||
redis.setMode('online');
|
||||
try {
|
||||
if (prisma) {
|
||||
if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } });
|
||||
if (userId) await prisma.user.deleteMany({ where: { id: userId } });
|
||||
}
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
observer.disconnect();
|
||||
if (app) await app.close();
|
||||
await db.close();
|
||||
await redis.close();
|
||||
for (const [key, value] of Object.entries(original)) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
await observer.ping();
|
||||
app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
prisma = app.get(PrismaService);
|
||||
let token;
|
||||
const request = async (
|
||||
method,
|
||||
url,
|
||||
{
|
||||
payload,
|
||||
authorization = token ? `Bearer ${token}` : undefined,
|
||||
headers = {},
|
||||
budget = 8000,
|
||||
} = {},
|
||||
) => {
|
||||
const start = performance.now();
|
||||
let timer;
|
||||
try {
|
||||
const response = await Promise.race([
|
||||
server.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
...headers,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
...(authorization ? { authorization } : {}),
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
}),
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error(`${method} ${url} exceeded ${budget}ms`)),
|
||||
budget,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
assert.ok(
|
||||
performance.now() - start < budget,
|
||||
'request exceeded its deadline',
|
||||
);
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
const errorResponse = (response, status, code, challenge) => {
|
||||
assert.equal(
|
||||
response.statusCode,
|
||||
status,
|
||||
`${response.statusCode}: ${response.body}`,
|
||||
);
|
||||
const body = response.json();
|
||||
assert.deepEqual(Object.keys(body), ['error']);
|
||||
assert.deepEqual(Object.keys(body.error).sort(), [
|
||||
'code',
|
||||
'message',
|
||||
'request_id',
|
||||
]);
|
||||
assert.equal(body.error.code, code);
|
||||
assert.equal(body.error.request_id, response.headers['x-request-id']);
|
||||
assert.ok(body.error.request_id);
|
||||
assert.ok(body.error.message.length <= 200);
|
||||
for (const secret of [
|
||||
original.DATABASE_URL,
|
||||
original.CACHE_URL,
|
||||
token,
|
||||
].filter(Boolean))
|
||||
assert.ok(!response.body.includes(secret));
|
||||
assert.ok(
|
||||
!/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test(
|
||||
response.body,
|
||||
),
|
||||
'error leaks implementation details',
|
||||
);
|
||||
if (challenge)
|
||||
assert.equal(response.headers['www-authenticate'], challenge);
|
||||
};
|
||||
const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`;
|
||||
const password = 'Synthetic-test-password:🔐123';
|
||||
const basic = (secret) =>
|
||||
`Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
|
||||
const login = (secret = password) =>
|
||||
request('POST', '/api/v1/auth/sessions', {
|
||||
authorization: basic(secret),
|
||||
});
|
||||
const registered = await request('POST', '/api/v1/users', {
|
||||
payload: { username, display_name: 'Fault test', password },
|
||||
budget: 15000,
|
||||
});
|
||||
assert.equal(registered.statusCode, 201);
|
||||
userId = registered.json().id;
|
||||
|
||||
await t.test('incorrect password is 401 with Basic challenge', async () => {
|
||||
const wrong = await login('wrong-password');
|
||||
errorResponse(
|
||||
wrong,
|
||||
401,
|
||||
'unauthorized',
|
||||
'Basic realm="messenger", charset="UTF-8"',
|
||||
);
|
||||
assert.equal(await prisma.session.count({ where: { userId } }), 0);
|
||||
});
|
||||
await t.test(
|
||||
'SESSION_TTL_SECONDS=2 expires without renewing on reads',
|
||||
async () => {
|
||||
const response = await login();
|
||||
assert.equal(response.statusCode, 201);
|
||||
token = response.json().token;
|
||||
const stored = await prisma.session.findFirstOrThrow({
|
||||
where: { userId },
|
||||
});
|
||||
assert.ok(
|
||||
Math.abs(
|
||||
stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000,
|
||||
) < 500,
|
||||
);
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
await pause(
|
||||
Math.max(0, Date.parse(response.json().expires_at) - Date.now()) +
|
||||
100,
|
||||
);
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'logout revokes immediately, including with a cached profile',
|
||||
async () => {
|
||||
const response = await login();
|
||||
assert.equal(response.statusCode, 201);
|
||||
token = response.json().token;
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
const logout = await request('DELETE', '/api/v1/auth/sessions/current');
|
||||
assert.equal(logout.statusCode, 204);
|
||||
assert.equal(logout.body, '');
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
},
|
||||
);
|
||||
token = randomBytes(32).toString('base64url');
|
||||
const { createHash } = await import('node:crypto');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
const chat = await request('POST', '/api/v1/chats', {
|
||||
payload: { title: 'Fault isolation', member_ids: [userId] },
|
||||
});
|
||||
assert.equal(chat.statusCode, 201);
|
||||
chatId = chat.json().id;
|
||||
const history = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal(
|
||||
(await request('POST', history, { payload: { text: 'Before fault' } }))
|
||||
.statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal((await request('GET', history)).json().items.length, 1);
|
||||
|
||||
const anonymousChecks = async () => {
|
||||
for (const authorization of ['', 'Bearer invalid', basic(password)]) {
|
||||
const headers = { 'x-user-id': userId };
|
||||
errorResponse(
|
||||
await request('GET', history, { authorization, headers }),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('POST', history, {
|
||||
authorization,
|
||||
headers,
|
||||
payload: { text: 'Must not be stored' },
|
||||
}),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
}
|
||||
};
|
||||
await t.test(
|
||||
'Redis disconnect: SQL fallback and anonymous callers remain unauthorized',
|
||||
async () => {
|
||||
redis.setMode('cut');
|
||||
await anonymousChecks();
|
||||
assert.equal(
|
||||
(await request('GET', history, { budget: 3000 })).statusCode,
|
||||
200,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request('POST', history, {
|
||||
payload: { text: 'Redis offline' },
|
||||
budget: 3000,
|
||||
})
|
||||
).statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal(
|
||||
(await request('GET', history, { budget: 3000 })).json().items.length,
|
||||
2,
|
||||
);
|
||||
redis.setMode('online');
|
||||
await pause(1300);
|
||||
},
|
||||
);
|
||||
await t.test('Redis hangs: command timeout bypasses cache', async () => {
|
||||
redis.setMode('hang');
|
||||
try {
|
||||
assert.equal(
|
||||
(await request('GET', `${history}?limit=10`, { budget: 3000 }))
|
||||
.statusCode,
|
||||
200,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request('POST', history, {
|
||||
payload: { text: 'Redis blackhole' },
|
||||
budget: 3000,
|
||||
})
|
||||
).statusCode,
|
||||
201,
|
||||
);
|
||||
await anonymousChecks();
|
||||
} finally {
|
||||
redis.setMode('cut');
|
||||
redis.setMode('online');
|
||||
}
|
||||
await pause(1300);
|
||||
});
|
||||
await t.test(
|
||||
'PostgreSQL disconnect: bounded 503, no cached authorization bypass',
|
||||
async () => {
|
||||
db.setMode('cut');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('POST', history, { payload: { text: 'DB offline' } }),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(await login(), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('GET', '/health/ready'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
assert.equal((await request('GET', '/health/live')).statusCode, 200);
|
||||
} finally {
|
||||
db.setMode('online');
|
||||
}
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'PostgreSQL hangs on an established connection: client query timeout',
|
||||
async () => {
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
db.setMode('hang');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
} finally {
|
||||
db.setMode('cut');
|
||||
db.setMode('online');
|
||||
}
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'both services unavailable: fail closed and recover confirmed data',
|
||||
async () => {
|
||||
db.setMode('cut');
|
||||
redis.setMode('cut');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('GET', history, {
|
||||
authorization: `Bearer ${randomBytes(32).toString('base64url')}`,
|
||||
}),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
} finally {
|
||||
db.setMode('online');
|
||||
redis.setMode('online');
|
||||
}
|
||||
const recovered = await request('GET', `${history}?limit=100`);
|
||||
assert.equal(recovered.statusCode, 200);
|
||||
assert.deepEqual(
|
||||
recovered.json().items.map((message) => message.text),
|
||||
['Before fault', 'Redis offline', 'Redis blackhole'],
|
||||
);
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 3);
|
||||
assert.equal((await request('GET', '/health/ready')).statusCode, 200);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,29 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location('saturation', Path(__file__).resolve().parents[1]/'scripts/run-saturation.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
class ClassificationTest(unittest.TestCase):
|
||||
def classify(self, attempted=1000, dropped=0, errors=0, p95=500):
|
||||
return module.classify({'measurement': {'scheduled_slots': 1000,
|
||||
'client_dropped_slots': dropped, 'all': {'attempted': attempted, 'errors': errors}},
|
||||
'all_latency_ms': {'p95': p95}})['classification']
|
||||
|
||||
def test_delivered_healthy_workload_passes(self):
|
||||
self.assertEqual(self.classify(), 'pass')
|
||||
|
||||
def test_latency_or_errors_break_service_criterion(self):
|
||||
self.assertEqual(self.classify(p95=1500), 'slo_fail')
|
||||
self.assertEqual(self.classify(errors=20), 'slo_fail')
|
||||
|
||||
def test_underfed_workload_is_not_a_server_capacity_measurement(self):
|
||||
self.assertEqual(self.classify(attempted=800, dropped=200), 'generator_limited')
|
||||
self.assertEqual(self.classify(attempted=800, dropped=200, p95=3000), 'generator_limited')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user