From 6cccda14d2ec5fad2a86ac28b6f3fbd43dc52631 Mon Sep 17 00:00:00 2001 From: Zertonixa Date: Wed, 14 Oct 2026 06:29:28 +0300 Subject: [PATCH] test(api): cover persistence, cache consistency and dependency failures --- scripts/test-persistence.py | 101 ++++++++ test/README.md | 74 ++++++ test/cache-epochs.mjs | 195 +++++++++++++++ test/cache.mjs | 287 +++++++++++++++++++++ test/full-restart.py | 57 +++++ test/helpers/fault-proxy.mjs | 56 +++++ test/helpers/message-epoch.mjs | 7 + test/message-pagination.mjs | 137 ++++++++++ test/passwords.mjs | 63 +++++ test/persistence.mjs | 346 ++++++++++++++++++++++++++ test/resilience.mjs | 441 +++++++++++++++++++++++++++++++++ 11 files changed, 1764 insertions(+) create mode 100644 scripts/test-persistence.py create mode 100644 test/README.md create mode 100644 test/cache-epochs.mjs create mode 100644 test/cache.mjs create mode 100644 test/full-restart.py create mode 100644 test/helpers/fault-proxy.mjs create mode 100644 test/helpers/message-epoch.mjs create mode 100644 test/message-pagination.mjs create mode 100644 test/passwords.mjs create mode 100644 test/persistence.mjs create mode 100644 test/resilience.mjs diff --git a/scripts/test-persistence.py b/scripts/test-persistence.py new file mode 100644 index 00000000..e55872fd --- /dev/null +++ b/scripts/test-persistence.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = {'backend', 'postgres', 'redis'} + + +def snapshot(): + ids = subprocess.check_output( + ['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)], + cwd=ROOT, text=True, timeout=30, + ).split() + if len(ids) != len(SERVICES): + raise RuntimeError('Start backend, postgres and redis before this test') + template = ( + '{"service":{{json (index .Config.Labels "com.docker.compose.service")}},' + '"id":{{json .Id}},"started_at":{{json .State.StartedAt}},' + '"running":{{json .State.Running}},"mounts":{{json .Mounts}}}' + ) + output = subprocess.check_output( + ['docker', 'inspect', '--format', template, *ids], + cwd=ROOT, text=True, timeout=30, + ) + result = {} + for line in output.splitlines(): + entry = json.loads(line) + service = entry.pop('service') + entry['volumes'] = [ + {'name': mount['Name'], 'destination': mount['Destination']} + for mount in entry.pop('mounts') if mount['Type'] == 'volume' + ] + result[service] = entry + if set(result) != SERVICES or not all(v['running'] for v in result.values()): + raise RuntimeError('All three services must be running') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080')) + parser.add_argument('--ca-file') + parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public') + parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json') + args = parser.parse_args() + before = snapshot() + hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh') + if args.scenario == 'public': + command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence'] + else: + command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')] + command += ['--action-script', str(hook), '--base-url', args.base_url] + if args.ca_file: + command += ['--ca-file', args.ca_file] + started = time.monotonic() + proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True) + print(proc.stdout, end='') + print(proc.stderr, end='', file=sys.stderr) + if proc.returncode: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps({ + 'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'scenario': args.scenario, + 'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}', + 'hook': hook.relative_to(ROOT).as_posix(), + 'result': 'FAIL', 'exit_code': proc.returncode, + 'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(), + 'before': before, + }, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + return proc.returncode + after = snapshot() + for service in sorted(SERVICES): + old, new = before[service], after[service] + if old['id'] != new['id'] or old['started_at'] == new['started_at']: + raise RuntimeError(f'{service}: expected a restart of the same container') + if not old['volumes'] or old['volumes'] != new['volumes']: + raise RuntimeError(f'{service}: persistent volumes changed or are missing') + evidence = { + 'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'scenario': args.scenario, + 'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}', + 'hook': hook.relative_to(ROOT).as_posix(), + 'elapsed_seconds': round(time.monotonic() - started, 3), + 'result': 'PASS', 'test_output': proc.stdout.strip(), + 'before': before, 'after': after, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('PASS: backend, postgres and redis restarted; named volumes preserved.') + print(f'Evidence: {args.output}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..81c67f4c --- /dev/null +++ b/test/README.md @@ -0,0 +1,74 @@ +# Проверки приложения + +Текущий вариант использует 16 счётчиков эпохи истории, единый SQL проверки сессии и членства и профиль из JOIN. Сравнение трёх оптимизаций: [результаты](../evidence/epoch-variants-summary.md). Итоговый контроль против того же кода без кэша и обслуживания эпох: [методика и результаты](../evidence/cache-strategy-buckets-summary.md), включая точные команды повторения длинной нагрузки. Для актуальной схемы примените миграции и используйте проверки ниже. `epoch-variants-experiment.cjs` предназначен для исторической матрицы до миграции счётчиков; точные исходники вариантов сохранены в `evidence/epoch-variants-sources.json`. + +Нужны Node.js 24.20.0 (как в Dockerfile), установленные зависимости и PostgreSQL с применёнными миграциями. Переменные читаются из локального `.env`; Redis нужен проверкам кэша и отказов. + +```powershell +pnpm.cmd build +node --test test/passwords.mjs test/persistence.mjs test/resilience.mjs +node --test test/cache.mjs test/cache-epochs.mjs test/message-pagination.mjs +pnpm.cmd lint +``` + +`passwords.mjs` проверяет Argon2id в PHC-формате, независимое вычисление ключа, случайные соли, Unicode/NUL, неверные и повреждённые хеши. Поддерживается только Argon2id; прежние scrypt-хеши не принимаются. Пароли, хеши и токены не печатаются. + +Публичные проверки работают по HTTP с уже запущенным Compose: + +```powershell +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 scripts/test-persistence.py --scenario public --base-url http://127.0.0.1:8080 --output evidence/persistence-public.json +``` + +Последняя команда использует неизменённую публичную проверку с HTTP-дедлайном 2 секунды. Она создаёт синтетические данные, перезапускает API/PostgreSQL/Redis через `scripts/restart.cmd` (Windows) или `scripts/restart.sh` (Linux), сохраняет volumes и проверяет данные прежним токеном. Windows требует Git Bash. Для Linux сохраните executable bit скрипта (`chmod +x scripts/restart.sh`). + +На Windows можно запустить этот же нормативный сценарий через make: + +```powershell +make test-persistence ACTION_SCRIPT=scripts/restart.cmd PYTHON="python -X utf8" +``` + +Собственный сценарий с HTTP-дедлайном 30 секунд: + +```powershell +python -X utf8 scripts/test-persistence.py --scenario full-restart --base-url http://127.0.0.1:8080 --output evidence/argon2-full-restart.json +``` + +Измерения кэша описаны в `evidence/cache-epochs-summary.md`. Их основное сравнение выполнено до замены KDF, на синтетических сессиях без измерения регистрации/входа. Хеширование паролей не входит в эти RPS. + +Результаты оптимизации записи эпохи одним CTE: [сводка](../evidence/cache-epochs-optimization-summary.md). Повторение замера после сборки и запуска backend: + +```powershell +docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/cache-epochs-optimized.json +node test/cache-epochs-optimization-summary.mjs +``` + +Один сравнительный прогон с Redis и со всеми кэшами приложения, отключёнными через отсутствие CACHE_URL. Эпохи в PostgreSQL остаются включены в обоих режимах: + +```powershell +docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/redis-comparison.json --compare-redis +node test/redis-comparison-summary.mjs +``` + +[Средние результаты и ограничения](../evidence/redis-comparison-summary.md). Сборка не нужна, если текущий образ уже содержит проверяемую реализацию приложения. + +Сравнение полной стратегии с вариантом без Redis и поддержания эпох: + +```powershell +docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/cache-strategy-full.json +node test/no-epochs-experiment.cjs +node test/cache-strategy-summary.mjs +``` + +Второй шаг создаёт временные копии шести модулей, убирает обслуживание эпох, компилирует их TypeScript и подключает в отдельный benchmark-контейнер. Рабочие исходники и backend не меняются; служебные колонки БД сохраняются. По завершении временные файлы удаляются. [Результаты](../evidence/cache-strategy-summary.md). + +Увеличенная нагрузка: 100 000 сообщений, конкурентность 32, чтение по 60 секунд, прогрев 10 секунд, по 4096 записей, три повтора. Дополнительный контроль с частично отключённым кэшем пропущен: + +```powershell +docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/cache-strategy-long-full.json --concurrency 32 --duration-seconds 60 --warmup-seconds 10 --repetitions 3 --seed-messages 100000 --cold-burst 64 --writes-per-repeat 4096 --skip-history-control +$env:EXPERIMENT_PREFIX='cache-strategy-long' +node test/no-epochs-experiment.cjs --concurrency 32 --duration-seconds 60 --warmup-seconds 10 --repetitions 3 --seed-messages 100000 --cold-burst 64 --writes-per-repeat 4096 --skip-history-control +node test/cache-strategy-summary.mjs cache-strategy-long +``` + +[Результаты увеличенного прогона](../evidence/cache-strategy-long-summary.md). Старые короткие замеры сохраняются отдельно. diff --git a/test/cache-epochs.mjs b/test/cache-epochs.mjs new file mode 100644 index 00000000..9947cc62 --- /dev/null +++ b/test/cache-epochs.mjs @@ -0,0 +1,195 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; +import { faultProxy } from './helpers/fault-proxy.mjs'; +import { messageEpoch } from './helpers/message-epoch.mjs'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { CacheService } = require('../dist/cache/cache.service'); +const { ChatService } = require('../dist/Chat/chat.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +test( + 'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback', + { timeout: 90000 }, + async (t) => { + assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL); + const oldUrl = process.env.CACHE_URL; + const oldPrefix = process.env.CACHE_PREFIX; + const prefix = `test:epochs:${randomUUID()}`; + const proxy = await faultProxy(oldUrl, 6379); + const observer = new Redis(oldUrl); + process.env.CACHE_URL = proxy.url; + process.env.CACHE_PREFIX = prefix; + const apps = []; + let prisma, userId; + const chatIds = []; + t.after(async () => { + proxy.setMode('online'); + try { + if (prisma) { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + if (userId) await prisma.user.delete({ where: { id: userId } }); + } + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + for (const app of apps) await app.close(); + observer.disconnect(); + await proxy.close(); + process.env.CACHE_URL = oldUrl; + if (oldPrefix === undefined) delete process.env.CACHE_PREFIX; + else process.env.CACHE_PREFIX = oldPrefix; + } + }); + for (let i = 0; i < 2; i++) { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + apps.push(app); + exposeRequestIdHeader(app); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + require('nestjs-pino').PinoLogger.root.level = 'silent'; + } + prisma = apps[0].get(PrismaService); + const user = await prisma.user.create({ + data: { + username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Synthetic epochs', + }, + }); + userId = user.id; + const token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + const call = async (replica, method, url, payload) => { + const response = await apps[replica] + .getHttpAdapter() + .getInstance() + .inject({ + method, + url, + headers: { authorization: `Bearer ${token}` }, + ...(payload ? { payload } : {}), + }); + return { status: response.statusCode, body: response.json() }; + }; + const createChat = async () => { + const response = await call(0, 'POST', '/api/v1/chats', { + title: 'Epoch test', + member_ids: [userId], + }); + assert.equal(response.status, 201); + chatIds.push(response.body.id); + return response.body.id; + }; + assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0); + proxy.setMode('cut'); + await pause(100); + const chatId = await createChat(); + proxy.setMode('online'); + const reconnect = async () => { + for (let i = 0; i < 60; i++) { + if (apps.every((app) => app.get(CacheService).redis.status === 'ready')) + return; + await pause(100); + } + assert.fail('Redis did not reconnect'); + }; + await reconnect(); + assert.deepEqual( + (await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id), + [chatId], + ); + assert.equal( + (await prisma.user.findUniqueOrThrow({ where: { id: userId } })) + .chatsEpoch, + 1n, + ); + + const path = `/api/v1/chats/${chatId}/messages`; + assert.equal((await call(1, 'GET', path)).body.items.length, 0); + const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`); + assert.ok(oldKeys.length); + proxy.setMode('cut'); + await pause(100); + const posted = await call(0, 'POST', path, { + text: 'Write during Redis outage', + }); + assert.equal(posted.status, 201); + proxy.setMode('online'); + await reconnect(); + assert.equal( + await observer.exists(...oldKeys), + oldKeys.length, + 'old cache keys must still exist', + ); + assert.deepEqual( + (await call(1, 'GET', path)).body.items.map((x) => x.id), + [posted.body.id], + ); + + const service = apps[0].get(ChatService); + const original = service.prisma; + service.prisma = original.$extends({ + query: { + $queryRaw({ args, query }) { + const index = args.values.indexOf('Must roll back'); + if (index !== -1) args.values[index] = 'x'.repeat(4001); + return query(args); + }, + }, + }); + try { + assert.equal( + (await call(0, 'POST', path, { text: 'Must roll back' })).status, + 500, + ); + } finally { + service.prisma = original; + } + assert.equal(await prisma.message.count({ where: { chatId } }), 1); + assert.equal(await messageEpoch(prisma, chatId), 1n); + + const writes = await Promise.all( + Array.from({ length: 20 }, (_, i) => + call(i % 2, 'POST', path, { text: `Concurrent ${i}` }), + ), + ); + assert.ok(writes.every((x) => x.status === 201)); + assert.equal(await prisma.message.count({ where: { chatId } }), 21); + assert.equal(await messageEpoch(prisma, chatId), 21n); + const history = (await call(1, 'GET', path)).body.items; + assert.equal(new Set(history.map((x) => x.id)).size, 21); + assert.ok( + writes.every((x) => history.some((message) => message.id === x.body.id)), + ); + }, +); diff --git a/test/cache.mjs b/test/cache.mjs new file mode 100644 index 00000000..855cd58a --- /dev/null +++ b/test/cache.mjs @@ -0,0 +1,287 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { CacheService } = require('../dist/cache/cache.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +async function isolatedCache(t) { + assert.ok(process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance'); + const previous = process.env.CACHE_PREFIX; + const prefix = `test:cache:${randomUUID()}`; + process.env.CACHE_PREFIX = prefix; + const observer = new Redis(process.env.CACHE_URL, { + maxRetriesPerRequest: 0, + connectTimeout: 1000, + commandTimeout: 1000, + }); + observer.on('error', () => {}); + await observer.ping(); + t.after(async () => { + try { + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + observer.disconnect(); + if (previous === undefined) delete process.env.CACHE_PREFIX; + else process.env.CACHE_PREFIX = previous; + } + }); + return { prefix, observer }; +} + +test('Redis cache: hits, expiry, invalidation race and unavailable-server fallback', async (t) => { + await isolatedCache(t); + const cache = new CacheService(); + t.after(() => cache.onModuleDestroy()); + await cache.onModuleInit(); + let reads = 0; + const load = async () => ({ value: ++reads }); + assert.deepEqual(await cache.remember('hits', 'page', 30, load), { + value: 1, + }); + assert.deepEqual(await cache.remember('hits', 'page', 30, load), { + value: 1, + }); + assert.equal(reads, 1); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 2, + }); + await new Promise((resolve) => setTimeout(resolve, 1100)); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 3, + }); + await cache.invalidate('expiry'); + assert.deepEqual(await cache.remember('expiry', 'page', 1, load), { + value: 4, + }); + + let resolveOld; + let began; + const started = new Promise((resolve) => { + began = resolve; + }); + const pending = cache.remember('race', 'page', 30, () => { + began(); + return new Promise((resolve) => { + resolveOld = resolve; + }); + }); + await started; + await cache.invalidate('race'); + resolveOld({ value: 'old' }); + await pending; + assert.deepEqual( + await cache.remember('race', 'page', 30, async () => ({ value: 'new' })), + { value: 'new' }, + ); + assert.deepEqual( + await cache.remember('race', 'page', 30, async () => { + throw new Error('cache miss'); + }), + { value: 'new' }, + ); + await assert.rejects( + cache.remember('db-error', 'page', 30, async () => { + throw new Error('database failure'); + }), + /database failure/, + ); + + const previous = process.env.CACHE_URL; + process.env.CACHE_URL = 'redis://127.0.0.1:1/0'; + const offline = new CacheService(); + process.env.CACHE_URL = previous; + t.after(() => offline.onModuleDestroy()); + await offline.onModuleInit(); + assert.deepEqual( + await offline.remember('offline', 'page', 30, async () => ({ + value: 'database', + })), + { value: 'database' }, + ); + await offline.invalidate('offline'); +}); + +test('API cache: fewer SQL reads, write invalidation, permissions, logout and Redis fallback', async (t) => { + assert.ok(process.env.DATABASE_URL); + const { prefix, observer } = await isolatedCache(t); + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + const prisma = app.get(PrismaService); + const userIds = []; + const chatIds = []; + t.after(async () => { + try { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + await prisma.user.deleteMany({ where: { id: { in: userIds } } }); + } finally { + await app.close(); + } + }); + const actors = []; + for (let i = 0; i < 2; i++) { + const user = await prisma.user.create({ + data: { + username: `cache_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Cache test', + }, + }); + userIds.push(user.id); + const token = randomBytes(32).toString('base64url'); + const session = await prisma.session.create({ + data: { + userId: user.id, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + actors.push({ id: user.id, token, sessionId: session.id }); + } + const [alice, outsider] = actors; + const request = (method, url, payload, actor = alice) => + server.inject({ + method, + url, + headers: { authorization: `Bearer ${actor.token}` }, + ...(payload === undefined ? {} : { payload }), + }); + const counts = new Map(); + const tracked = prisma.$extends({ + query: { + $allModels: { + $allOperations({ model, operation, args, query }) { + const key = `${model}:${operation}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + return query(args); + }, + }, + }, + }); + for (const name of [ + 'User/user.service', + 'Chat/chat.service', + 'Auth/auth.service', + ]) { + const exported = require(`../dist/${name}`); + const Service = Object.values(exported)[0]; + app.get(Service).prisma = tracked; + } + const spy = (model, operation) => ({ + mock: { callCount: () => counts.get(`${model}:${operation}`) ?? 0 }, + }); + const profileReads = spy('User', 'findUnique'); + assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200); + assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200); + assert.equal(profileReads.mock.callCount(), 0); + + const chatListReads = spy('Chat', 'findMany'); + assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0); + assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0); + assert.equal(chatListReads.mock.callCount(), 1); + const created = await request('POST', '/api/v1/chats', { + title: 'Cached chat', + member_ids: [alice.id], + }); + assert.equal(created.statusCode, 201); + const chatId = created.json().id; + chatIds.push(chatId); + assert.deepEqual( + (await request('GET', '/api/v1/chats')).json().items.map((x) => x.id), + [chatId], + ); + assert.equal(chatListReads.mock.callCount(), 2); + + const metadataReads = spy('Chat', 'findUnique'); + const messageReads = spy('Message', 'findMany'); + const path = `/api/v1/chats/${chatId}/messages`; + assert.equal((await request('GET', path)).json().items.length, 0); + assert.equal((await request('GET', path)).json().items.length, 0); + assert.equal(metadataReads.mock.callCount(), 0); + assert.equal(messageReads.mock.callCount(), 1); + const message = await request('POST', path, { + text: 'Invalidate cached empty history', + }); + assert.equal(message.statusCode, 201); + assert.deepEqual( + (await request('GET', path)).json().items.map((x) => x.id), + [message.json().id], + ); + assert.equal(messageReads.mock.callCount(), 2); + assert.equal( + (await request('GET', path, undefined, outsider)).statusCode, + 404, + ); + await prisma.chatMember.delete({ + where: { chatId_userId: { chatId, userId: alice.id } }, + }); + assert.equal((await request('GET', path)).statusCode, 404); + await prisma.chatMember.create({ + data: { chatId, userId: alice.id, position: 0 }, + }); + await prisma.session.update({ + where: { id: outsider.sessionId }, + data: { revokedAt: new Date() }, + }); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, outsider)).statusCode, + 401, + ); + + const keys = []; + let cursor = '0'; + do { + const [next, batch] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + keys.push(...batch); + } while (cursor !== '0'); + assert.ok(keys.length >= 4, 'Redis must contain the cached responses'); + for (const key of keys) { + const value = await observer.get(key); + assert.ok( + !value.includes('password_hash') && !value.includes('passwordHash'), + ); + for (const actor of actors) assert.ok(!value.includes(actor.token)); + } + app.get(CacheService).onModuleDestroy(); + assert.equal((await request('GET', path)).statusCode, 200); + assert.equal(messageReads.mock.callCount(), 3); + assert.equal( + (await request('POST', path, { text: 'Write while Redis is unavailable' })) + .statusCode, + 201, + ); + assert.equal((await request('GET', path)).json().items.length, 2); +}); diff --git a/test/full-restart.py b/test/full-restart.py new file mode 100644 index 00000000..0c2d813e --- /dev/null +++ b/test/full-restart.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import argparse +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run +from resilience import history + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + parser.add_argument('--ca-file') + parser.add_argument('--action-script', required=True) + args = parser.parse_args() + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = client.bootstrap() + original = client.message(alice, chat, 'Сохранность после полного перезапуска')['json'] + check(history(client, bob, chat) == [original], 'pre-restart history mismatch') + hook = Path(args.action_script).resolve() + check(hook.is_file(), 'restart hook not found') + print('Restarting backend, PostgreSQL and Redis; hook output suppressed.', flush=True) + try: + proc = subprocess.run([str(hook)], cwd=ROOT, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=90) + except subprocess.TimeoutExpired: + raise Failure('restart hook exceeded 90 seconds') from None + check(proc.returncode == 0, 'restart hook failed') + deadline = time.monotonic() + 90 + while True: + try: + client.call('GET', '/health/ready', schema='Health') + break + except Failure: + check(time.monotonic() < deadline, 'API did not recover within 90 seconds') + time.sleep(0.5) + for actor in (alice, bob, eve): + check(client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'] + == actor['user'], 'user or existing session changed after restart') + check(client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')['json'] + == chat, 'chat or membership changed after restart') + check(history(client, bob, chat) == [original], 'confirmed message lost, changed or duplicated') + client.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve, schema='Error') + followup = client.message(bob, chat, 'Запись после восстановления')['json'] + check(history(client, alice, chat) == [original, followup], 'post-restart write/read mismatch') + client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + client.call('GET', '/api/v1/users/me', 401, actor=alice, schema='Error') + print('PASS: users, chat, membership, confirmed message and all existing sessions survived; ' + 'access control, new writes and logout work after restart.') + + +if __name__ == '__main__': + run(main) diff --git a/test/helpers/fault-proxy.mjs b/test/helpers/fault-proxy.mjs new file mode 100644 index 00000000..223dbf45 --- /dev/null +++ b/test/helpers/fault-proxy.mjs @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import net from 'node:net'; + +export async function faultProxy(connectionUrl, defaultPort) { + const upstreamUrl = new URL(connectionUrl); + const host = upstreamUrl.hostname.replace(/^\[|\]$/g, ''); + const port = Number(upstreamUrl.port || defaultPort); + let mode = 'online'; + const sockets = new Set(); + const track = (socket) => { + sockets.add(socket); + socket.on('error', () => {}); + socket.on('close', () => sockets.delete(socket)); + return socket; + }; + const server = net.createServer((client) => { + track(client); + if (mode === 'cut') { + client.destroy(); + return; + } + const upstream = track(net.connect({ host, port })); + client.on('data', (chunk) => { + if (mode === 'online' && !upstream.destroyed) upstream.write(chunk); + }); + upstream.on('data', (chunk) => { + if (mode === 'online' && !client.destroyed) client.write(chunk); + }); + client.on('close', () => upstream.destroy()); + upstream.on('close', () => client.destroy()); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const relayUrl = new URL(connectionUrl); + relayUrl.hostname = '127.0.0.1'; + relayUrl.port = String(server.address().port); + return { + url: relayUrl.toString(), + setMode(nextMode) { + assert.ok(['online', 'cut', 'hang'].includes(nextMode)); + mode = nextMode; + if (mode === 'cut') { + for (const socket of sockets) socket.destroy(); + } + }, + async close() { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + }, + }; +} diff --git a/test/helpers/message-epoch.mjs b/test/helpers/message-epoch.mjs new file mode 100644 index 00000000..1e2f49a4 --- /dev/null +++ b/test/helpers/message-epoch.mjs @@ -0,0 +1,7 @@ +export async function messageEpoch(prisma, chatId) { + const [row] = await prisma.$queryRawUnsafe( + 'SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM public.chat_message_epochs WHERE chat_id = $1::uuid', + chatId, + ); + return row.epoch; +} diff --git a/test/message-pagination.mjs b/test/message-pagination.mjs new file mode 100644 index 00000000..bc92b1e3 --- /dev/null +++ b/test/message-pagination.mjs @@ -0,0 +1,137 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +test('message keyset pagination: timestamp ties, chat isolation and deleted cursor row', async (t) => { + assert.ok(process.env.DATABASE_URL); + const previousCacheUrl = process.env.CACHE_URL; + process.env.CACHE_URL = ''; + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + const prisma = app.get(PrismaService); + let userId; + const chatIds = []; + t.after(async () => { + try { + await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + if (userId) await prisma.user.delete({ where: { id: userId } }); + } finally { + await app.close(); + if (previousCacheUrl === undefined) delete process.env.CACHE_URL; + else process.env.CACHE_URL = previousCacheUrl; + } + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + const user = await prisma.user.create({ + data: { + username: `page_${randomUUID().replaceAll('-', '').slice(0, 20)}`, + displayName: 'Synthetic pagination test', + }, + }); + userId = user.id; + const token = randomBytes(32).toString('base64url'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + for (let i = 0; i < 2; i++) { + const chat = await prisma.chat.create({ + data: { + title: `Pagination ${i}`, + members: { create: { userId, position: 0 } }, + }, + }); + chatIds.push(chat.id); + } + const [chatId, otherChatId] = chatIds; + const rows = Array.from({ length: 140 }, (_, i) => ({ + id: randomUUID(), + chatId, + senderId: userId, + text: `Message ${i}`, + createdAt: new Date(Date.UTC(2026, 0, 1) + Math.floor(i / 10) * 1000), + })); + await prisma.message.createMany({ + data: [ + ...rows, + ...rows.map((row) => ({ + ...row, + id: randomUUID(), + chatId: otherChatId, + text: 'Other chat', + })), + ], + }); + const expected = [...rows] + .sort( + (a, b) => + a.createdAt - b.createdAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ) + .map((row) => ({ + id: row.id, + chat_id: chatId, + sender_id: userId, + text: row.text, + created_at: row.createdAt.toISOString(), + })); + const request = (cursor, selectedChat = chatId) => + server.inject({ + method: 'GET', + url: `/api/v1/chats/${selectedChat}/messages?limit=7${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, + headers: { authorization: `Bearer ${token}` }, + }); + const seen = []; + let cursor; + let firstCursor; + do { + const response = await request(cursor); + assert.equal(response.statusCode, 200); + const page = response.json(); + assert.ok(page.items.length <= 7); + seen.push(...page.items); + if (!firstCursor) firstCursor = page.next_cursor; + cursor = page.next_cursor; + assert.ok( + seen.length <= rows.length, + 'pagination must terminate without duplicates', + ); + } while (cursor); + assert.deepEqual(seen, expected); + const anchor = JSON.parse( + Buffer.from(firstCursor, 'base64url').toString('utf8'), + ); + await prisma.message.delete({ where: { id: anchor.id } }); + const afterDeletion = await request(firstCursor); + assert.equal(afterDeletion.statusCode, 200); + assert.deepEqual(afterDeletion.json().items, expected.slice(7, 14)); + assert.equal((await request(firstCursor, otherChatId)).statusCode, 400); + const pastEnd = Buffer.from( + JSON.stringify({ + scope: `messages:${chatId}`, + id: randomUUID(), + created_at: '2027-01-01T00:00:00.000Z', + }), + ).toString('base64url'); + const empty = await request(pastEnd); + assert.equal(empty.statusCode, 200); + assert.deepEqual(empty.json(), { items: [], next_cursor: null }); +}); diff --git a/test/passwords.mjs b/test/passwords.mjs new file mode 100644 index 00000000..17c4eef0 --- /dev/null +++ b/test/passwords.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { argon2 } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { PasswordService } = require('../dist/Auth/password.service'); +const password = '🔐'.repeat(40) + ':Синтетический пароль\0'; + +test('Argon2id only: PHC format, independent derivation, unique salts, Unicode and invalid hashes', async () => { + const service = new PasswordService(); + const first = await service.hash(password); + const second = await service.hash(password); + assert.notEqual( + first, + second, + 'same password must have independently salted hashes', + ); + assert.match( + first, + /^\$argon2id\$v=19\$m=19456,t=2,p=1\$[A-Za-z0-9+/]{22}\$[A-Za-z0-9+/]{43}$/, + ); + assert.ok(!first.includes(password)); + const parts = first.split('$'); + const independent = await new Promise((resolve, reject) => + argon2( + 'argon2id', + { + message: password, + nonce: Buffer.from(parts[4], 'base64'), + memory: 19456, + passes: 2, + parallelism: 1, + tagLength: 32, + }, + (error, key) => (error ? reject(error) : resolve(key)), + ), + ); + assert.deepEqual(Buffer.from(parts[5], 'base64'), independent); + assert.equal(await service.verify(password, first), true); + assert.equal( + await service.verify(password.replace('🔐', '🔑'), first), + false, + ); + assert.equal( + await service.verify(password.slice(0, -1), first), + false, + 'NUL must not be silently truncated', + ); + for (const invalid of [ + null, + '', + 'scrypt$131072$8$1$unsupported$unsupported', + '$argon2id$v=19$m=999999999,t=999999999,p=1$x$x', + first.replace('v=19', 'v=16'), + first.replace('argon2id', 'argon2i'), + first + '$extra', + first.replace(parts[5], 'a'), + first.replace(parts[4], parts[4] + '='), + ]) { + assert.equal(await service.verify(password, invalid), false); + } +}); diff --git a/test/persistence.mjs b/test/persistence.mjs new file mode 100644 index 00000000..e2d86490 --- /dev/null +++ b/test/persistence.mjs @@ -0,0 +1,346 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); + +async function start() { + const app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + try { + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + return { app, server, prisma: app.get(PrismaService) }; + } catch (error) { + await app.close(); + throw error; + } +} + +test('PostgreSQL storage: constraints, pagination, access and application restart', async (t) => { + assert.ok( + process.env.DATABASE_URL, + 'Set DATABASE_URL and apply migrations before running this test', + ); + let context = await start(); + const userIds = []; + const chatIds = []; + t.after(async () => { + try { + await context.prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); + await context.prisma.user.deleteMany({ where: { id: { in: userIds } } }); + } finally { + await context.app.close(); + } + }); + const suffix = randomUUID().replaceAll('-', '').slice(0, 16); + const password = '🔐'.repeat(40) + ':пароль с пробелами '; + const tokens = new Map(); + const request = (method, url, payload, userId) => + context.server.inject({ + method, + url, + ...(payload === undefined ? {} : { payload }), + ...(userId + ? { headers: { authorization: `Bearer ${tokens.get(userId)}` } } + : {}), + }); + + const registrations = await Promise.all( + Array.from({ length: 4 }, () => + request('POST', '/api/v1/users', { + username: `race_${suffix}`, + display_name: 'Race', + password, + }), + ), + ); + for (const response of registrations) { + if (response.statusCode === 201) userIds.push(response.json().id); + } + assert.deepEqual( + registrations.map((response) => response.statusCode).sort(), + [201, 409, 409, 409], + ); + const userId = userIds[0]; + const outsider = await request('POST', '/api/v1/users', { + username: `other_${suffix}`, + display_name: 'Other', + password, + }); + assert.equal(outsider.statusCode, 201); + userIds.push(outsider.json().id); + + const login = (username, secret = password, payload) => + context.server.inject({ + method: 'POST', + url: '/api/v1/auth/sessions', + headers: { + authorization: `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`, + }, + ...(payload === undefined ? {} : { payload }), + }); + for (const user of [ + registrations.find((r) => r.statusCode === 201).json(), + outsider.json(), + ]) { + assert.deepEqual(Object.keys(user).sort(), [ + 'created_at', + 'display_name', + 'id', + 'username', + ]); + const response = await login(user.username); + assert.equal(response.statusCode, 201); + assert.equal(response.json().token_type, 'Bearer'); + assert.match(response.json().token, /^[A-Za-z0-9_-]{43}$/); + assert.deepEqual(response.json().user, user); + tokens.set(user.id, response.json().token); + const stored = await context.prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + }); + assert.match(stored.passwordHash, /^\$argon2id\$v=19\$m=19456,t=2,p=1\$/); + assert.ok(!stored.passwordHash.includes(password)); + } + const sessions = await context.prisma.session.findMany({ where: { userId } }); + assert.equal(sessions.length, 1); + assert.match(sessions[0].tokenHash, /^[a-f0-9]{64}$/); + assert.notEqual(sessions[0].tokenHash, tokens.get(userId)); + const wrong = await login(`race_${suffix}`, 'wrong-password'); + const unknown = await login(`missing_${suffix}`, 'wrong-password'); + assert.equal(wrong.statusCode, 401); + assert.equal(unknown.statusCode, 401); + assert.equal(wrong.json().error.message, unknown.json().error.message); + assert.equal( + wrong.headers['www-authenticate'], + 'Basic realm="messenger", charset="UTF-8"', + ); + assert.equal((await login(`race_${suffix}`, password, {})).statusCode, 400); + for (const invalid of [ + undefined, + 'short', + ' '.repeat(12), + '🔐'.repeat(129), + ]) { + assert.equal( + ( + await request('POST', '/api/v1/users', { + username: `bad_${suffix}`, + display_name: 'Bad', + ...(invalid === undefined ? {} : { password: invalid }), + }) + ).statusCode, + 400, + ); + } + for (const authorization of [ + undefined, + 'Bearer unknown', + `Basic ${Buffer.from(`race_${suffix}:${password}`).toString('base64')}`, + ]) { + const response = await context.server.inject({ + method: 'GET', + url: '/api/v1/users/me', + headers: { + 'x-user-id': userId, + ...(authorization ? { authorization } : {}), + }, + }); + assert.equal(response.statusCode, 401); + assert.equal( + response.headers['www-authenticate'], + 'Bearer realm="messenger"', + ); + } + const identity = await context.server.inject({ + method: 'GET', + url: '/api/v1/users/me', + headers: { + authorization: `Bearer ${tokens.get(userId)}`, + 'x-user-id': outsider.json().id, + }, + }); + assert.equal(identity.json().id, userId); + + const ready = await request('GET', '/health/ready'); + assert.equal(ready.statusCode, 200); + + const invalidChat = await request( + 'POST', + '/api/v1/chats', + { + title: `invalid_${suffix}`, + member_ids: [userId, randomUUID()], + }, + userId, + ); + assert.equal(invalidChat.statusCode, 400); + assert.equal( + await context.prisma.chat.count({ where: { title: `invalid_${suffix}` } }), + 0, + ); + + const rolledBackId = randomUUID(); + chatIds.push(rolledBackId); + await assert.rejects( + context.prisma.$transaction(async (tx) => { + await tx.chat.create({ data: { id: rolledBackId, title: 'Rollback' } }); + await tx.chatMember.create({ + data: { chatId: rolledBackId, userId: randomUUID(), position: 0 }, + }); + }), + { code: 'P2003' }, + ); + assert.equal( + await context.prisma.chat.count({ where: { id: rolledBackId } }), + 0, + ); + + const createdChat = await request( + 'POST', + '/api/v1/chats', + { title: 'Persisted chat', member_ids: [userId] }, + userId, + ); + assert.equal(createdChat.statusCode, 201); + const chatId = createdChat.json().id; + chatIds.push(chatId); + const forbidden = await request( + 'GET', + `/api/v1/chats/${chatId}`, + undefined, + outsider.json().id, + ); + assert.equal(forbidden.statusCode, 404); + const unauthorized = await request('GET', `/api/v1/chats/${chatId}`); + assert.equal(unauthorized.statusCode, 401); + + const messageIds = []; + for (let i = 0; i < 5; i++) { + const message = await request( + 'POST', + `/api/v1/chats/${chatId}/messages`, + { text: `Message ${i}` }, + userId, + ); + assert.equal(message.statusCode, 201); + messageIds.push(message.json().id); + } + await context.prisma.message.updateMany({ + where: { chatId }, + data: { createdAt: new Date('2026-01-01T00:00:00.000Z') }, + }); + const first = await request( + 'GET', + `/api/v1/chats/${chatId}/messages?limit=2`, + undefined, + userId, + ); + assert.equal(first.statusCode, 200); + assert.equal(first.json().items.length, 2); + assert.ok(first.json().next_cursor); + + await context.app.close(); + context = await start(); + const me = await request('GET', '/api/v1/users/me', undefined, userId); + assert.equal(me.statusCode, 200); + const chat = await request( + 'GET', + `/api/v1/chats/${chatId}`, + undefined, + userId, + ); + assert.equal(chat.statusCode, 200); + assert.deepEqual(chat.json().member_ids, [userId]); + const listed = await request( + 'GET', + '/api/v1/chats?limit=1', + undefined, + userId, + ); + assert.deepEqual( + listed.json().items.map((item) => item.id), + [chatId], + ); + + const seen = first.json().items.map((message) => message.id); + let cursor = first.json().next_cursor; + while (cursor) { + const page = await request( + 'GET', + `/api/v1/chats/${chatId}/messages?limit=2&cursor=${cursor}`, + undefined, + userId, + ); + assert.equal(page.statusCode, 200); + assert.ok(page.json().items.length <= 2); + seen.push(...page.json().items.map((message) => message.id)); + cursor = page.json().next_cursor; + } + assert.deepEqual(seen, [...messageIds].sort()); + assert.equal(new Set(seen).size, 5); + + const logouts = await Promise.all( + Array.from({ length: 2 }, () => + request('DELETE', '/api/v1/auth/sessions/current', undefined, userId), + ), + ); + assert.deepEqual(logouts.map((r) => r.statusCode).sort(), [204, 401]); + assert.equal(logouts.find((r) => r.statusCode === 204).body, ''); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 401, + ); + assert.equal( + ( + await request( + 'DELETE', + '/api/v1/auth/sessions/current', + undefined, + userId, + ) + ).statusCode, + 401, + ); + await context.app.close(); + const originalTtl = process.env.SESSION_TTL_SECONDS; + process.env.SESSION_TTL_SECONDS = '2'; + try { + context = await start(); + } finally { + if (originalTtl === undefined) delete process.env.SESSION_TTL_SECONDS; + else process.env.SESSION_TTL_SECONDS = originalTtl; + } + const short = await login(`race_${suffix}`); + assert.equal(short.statusCode, 201); + tokens.set(userId, short.json().token); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 200, + ); + await context.app.close(); + context = await start(); + await new Promise((resolve) => + setTimeout( + resolve, + Math.max(0, Date.parse(short.json().expires_at) - Date.now()) + 100, + ), + ); + assert.equal( + (await request('GET', '/api/v1/users/me', undefined, userId)).statusCode, + 401, + ); +}); diff --git a/test/resilience.mjs b/test/resilience.mjs new file mode 100644 index 00000000..0937ef7e --- /dev/null +++ b/test/resilience.mjs @@ -0,0 +1,441 @@ +import 'dotenv/config'; +import assert from 'node:assert/strict'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { performance } from 'node:perf_hooks'; +import test from 'node:test'; +import { faultProxy } from './helpers/fault-proxy.mjs'; + +const require = createRequire(import.meta.url); +const Redis = require('ioredis').default; +const { NestFactory } = require('@nestjs/core'); +const { AppModule } = require('../dist/app.module'); +const { PrismaService } = require('../dist/prisma/prisma.service'); +const { + createFastifyAdapter, + exposeRequestIdHeader, +} = require('../dist/common/http/fastify-adapter'); +const pause = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +test( + 'Auth and dependency failures remain bounded and fail closed', + { timeout: 120000 }, + async (t) => { + assert.ok( + process.env.DATABASE_URL, + 'Set DATABASE_URL and apply migrations', + ); + assert.ok( + process.env.CACHE_URL, + 'Set CACHE_URL to a running Redis instance', + ); + const original = { + DATABASE_URL: process.env.DATABASE_URL, + CACHE_URL: process.env.CACHE_URL, + CACHE_PREFIX: process.env.CACHE_PREFIX, + SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS, + }; + const prefix = `test:resilience:${randomUUID()}`; + const db = await faultProxy(original.DATABASE_URL, 5432); + const redis = await faultProxy(original.CACHE_URL, 6379); + Object.assign(process.env, { + DATABASE_URL: db.url, + CACHE_URL: redis.url, + CACHE_PREFIX: prefix, + SESSION_TTL_SECONDS: '2', + }); + const observer = new Redis(original.CACHE_URL, { + connectTimeout: 1000, + commandTimeout: 1000, + maxRetriesPerRequest: 0, + }); + observer.on('error', () => {}); + let app; + let prisma; + let userId; + let chatId; + t.after(async () => { + db.setMode('online'); + redis.setMode('online'); + try { + if (prisma) { + if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } }); + if (userId) await prisma.user.deleteMany({ where: { id: userId } }); + } + let cursor = '0'; + do { + const [next, keys] = await observer.scan( + cursor, + 'MATCH', + `${prefix}:*`, + 'COUNT', + 100, + ); + cursor = next; + if (keys.length) await observer.del(...keys); + } while (cursor !== '0'); + } finally { + observer.disconnect(); + if (app) await app.close(); + await db.close(); + await redis.close(); + for (const [key, value] of Object.entries(original)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); + await observer.ping(); + app = await NestFactory.create(AppModule, createFastifyAdapter(), { + logger: false, + abortOnError: false, + }); + exposeRequestIdHeader(app); + await app.init(); + const server = app.getHttpAdapter().getInstance(); + await server.ready(); + prisma = app.get(PrismaService); + let token; + const request = async ( + method, + url, + { + payload, + authorization = token ? `Bearer ${token}` : undefined, + headers = {}, + budget = 8000, + } = {}, + ) => { + const start = performance.now(); + let timer; + try { + const response = await Promise.race([ + server.inject({ + method, + url, + headers: { + ...headers, + ...(authorization ? { authorization } : {}), + }, + ...(payload === undefined ? {} : { payload }), + }), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${method} ${url} exceeded ${budget}ms`)), + budget, + ); + }), + ]); + assert.ok( + performance.now() - start < budget, + 'request exceeded its deadline', + ); + return response; + } finally { + clearTimeout(timer); + } + }; + const errorResponse = (response, status, code, challenge) => { + assert.equal( + response.statusCode, + status, + `${response.statusCode}: ${response.body}`, + ); + const body = response.json(); + assert.deepEqual(Object.keys(body), ['error']); + assert.deepEqual(Object.keys(body.error).sort(), [ + 'code', + 'message', + 'request_id', + ]); + assert.equal(body.error.code, code); + assert.equal(body.error.request_id, response.headers['x-request-id']); + assert.ok(body.error.request_id); + assert.ok(body.error.message.length <= 200); + for (const secret of [ + original.DATABASE_URL, + original.CACHE_URL, + token, + ].filter(Boolean)) + assert.ok(!response.body.includes(secret)); + assert.ok( + !/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test( + response.body, + ), + 'error leaks implementation details', + ); + if (challenge) + assert.equal(response.headers['www-authenticate'], challenge); + }; + const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`; + const password = 'Synthetic-test-password:🔐123'; + const basic = (secret) => + `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`; + const login = (secret = password) => + request('POST', '/api/v1/auth/sessions', { + authorization: basic(secret), + }); + const registered = await request('POST', '/api/v1/users', { + payload: { username, display_name: 'Fault test', password }, + budget: 15000, + }); + assert.equal(registered.statusCode, 201); + userId = registered.json().id; + + await t.test('incorrect password is 401 with Basic challenge', async () => { + const wrong = await login('wrong-password'); + errorResponse( + wrong, + 401, + 'unauthorized', + 'Basic realm="messenger", charset="UTF-8"', + ); + assert.equal(await prisma.session.count({ where: { userId } }), 0); + }); + await t.test( + 'SESSION_TTL_SECONDS=2 expires without renewing on reads', + async () => { + const response = await login(); + assert.equal(response.statusCode, 201); + token = response.json().token; + const stored = await prisma.session.findFirstOrThrow({ + where: { userId }, + }); + assert.ok( + Math.abs( + stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000, + ) < 500, + ); + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + await pause( + Math.max(0, Date.parse(response.json().expires_at) - Date.now()) + + 100, + ); + errorResponse( + await request('GET', '/api/v1/users/me'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + }, + ); + await t.test( + 'logout revokes immediately, including with a cached profile', + async () => { + const response = await login(); + assert.equal(response.statusCode, 201); + token = response.json().token; + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + const logout = await request('DELETE', '/api/v1/auth/sessions/current'); + assert.equal(logout.statusCode, 204); + assert.equal(logout.body, ''); + errorResponse( + await request('GET', '/api/v1/users/me'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + }, + ); + token = randomBytes(32).toString('base64url'); + const { createHash } = await import('node:crypto'); + await prisma.session.create({ + data: { + userId, + tokenHash: createHash('sha256').update(token).digest('hex'), + expiresAt: new Date(Date.now() + 600000), + }, + }); + const chat = await request('POST', '/api/v1/chats', { + payload: { title: 'Fault isolation', member_ids: [userId] }, + }); + assert.equal(chat.statusCode, 201); + chatId = chat.json().id; + const history = `/api/v1/chats/${chatId}/messages`; + assert.equal( + (await request('POST', history, { payload: { text: 'Before fault' } })) + .statusCode, + 201, + ); + assert.equal((await request('GET', history)).json().items.length, 1); + + const anonymousChecks = async () => { + for (const authorization of ['', 'Bearer invalid', basic(password)]) { + const headers = { 'x-user-id': userId }; + errorResponse( + await request('GET', history, { authorization, headers }), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + errorResponse( + await request('POST', history, { + authorization, + headers, + payload: { text: 'Must not be stored' }, + }), + 401, + 'unauthorized', + 'Bearer realm="messenger"', + ); + } + }; + await t.test( + 'Redis disconnect: SQL fallback and anonymous callers remain unauthorized', + async () => { + redis.setMode('cut'); + await anonymousChecks(); + assert.equal( + (await request('GET', history, { budget: 3000 })).statusCode, + 200, + ); + assert.equal( + ( + await request('POST', history, { + payload: { text: 'Redis offline' }, + budget: 3000, + }) + ).statusCode, + 201, + ); + assert.equal( + (await request('GET', history, { budget: 3000 })).json().items.length, + 2, + ); + redis.setMode('online'); + await pause(1300); + }, + ); + await t.test('Redis hangs: command timeout bypasses cache', async () => { + redis.setMode('hang'); + try { + assert.equal( + (await request('GET', `${history}?limit=10`, { budget: 3000 })) + .statusCode, + 200, + ); + assert.equal( + ( + await request('POST', history, { + payload: { text: 'Redis blackhole' }, + budget: 3000, + }) + ).statusCode, + 201, + ); + await anonymousChecks(); + } finally { + redis.setMode('cut'); + redis.setMode('online'); + } + await pause(1300); + }); + await t.test( + 'PostgreSQL disconnect: bounded 503, no cached authorization bypass', + async () => { + db.setMode('cut'); + try { + await anonymousChecks(); + errorResponse( + await request('GET', '/api/v1/users/me'), + 503, + 'unavailable', + ); + errorResponse(await request('GET', history), 503, 'unavailable'); + errorResponse( + await request('POST', history, { payload: { text: 'DB offline' } }), + 503, + 'unavailable', + ); + errorResponse( + await request('DELETE', '/api/v1/auth/sessions/current'), + 503, + 'unavailable', + ); + errorResponse(await login(), 503, 'unavailable'); + errorResponse( + await request('GET', '/health/ready'), + 503, + 'unavailable', + ); + assert.equal((await request('GET', '/health/live')).statusCode, 200); + } finally { + db.setMode('online'); + } + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + }, + ); + await t.test( + 'PostgreSQL hangs on an established connection: client query timeout', + async () => { + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + db.setMode('hang'); + try { + await anonymousChecks(); + errorResponse(await request('GET', history), 503, 'unavailable'); + } finally { + db.setMode('cut'); + db.setMode('online'); + } + assert.equal( + (await request('GET', '/api/v1/users/me')).statusCode, + 200, + ); + }, + ); + await t.test( + 'both services unavailable: fail closed and recover confirmed data', + async () => { + db.setMode('cut'); + redis.setMode('cut'); + try { + await anonymousChecks(); + errorResponse(await request('GET', history), 503, 'unavailable'); + errorResponse( + await request('GET', history, { + authorization: `Bearer ${randomBytes(32).toString('base64url')}`, + }), + 503, + 'unavailable', + ); + } finally { + db.setMode('online'); + redis.setMode('online'); + } + const recovered = await request('GET', `${history}?limit=100`); + assert.equal(recovered.statusCode, 200); + assert.deepEqual( + recovered.json().items.map((message) => message.text), + ['Before fault', 'Redis offline', 'Redis blackhole'], + ); + assert.equal(await prisma.message.count({ where: { chatId } }), 3); + assert.equal((await request('GET', '/health/ready')).statusCode, 200); + }, + ); + }, +);