Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c3c4c54ec4 | ||
|
|
c1fec73ccc | ||
|
|
5c8bcf0f80 | ||
|
|
676b38c18b | ||
|
|
24f32c3fbe | ||
|
|
298204dbe9 |
+6
-8
@@ -1,13 +1,11 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Variables used by compose.yaml for this local lab stack.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
POSTGRES_DB=chat
|
||||
POSTGRES_USER=app
|
||||
POSTGRES_PASSWORD=change_for_local_lab
|
||||
DB_POOL_MAX=15
|
||||
CACHE_PREFIX=messenger:cache:v1
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
test/*.sh text eol=lf
|
||||
test/*.cmd text eol=crlf
|
||||
scripts/*.sh text eol=lf
|
||||
scripts/*.cmd text eol=crlf
|
||||
infra/*.sh text eol=lf
|
||||
scripts/diagnostics/*.sh text eol=lf
|
||||
@@ -19,11 +19,15 @@ __pycache__/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
dist/
|
||||
src/generated/prisma/
|
||||
*.tsbuildinfo
|
||||
.pnpm-store/
|
||||
build/
|
||||
target/
|
||||
bin/
|
||||
obj/
|
||||
coverage/
|
||||
.artifacts/
|
||||
*.log
|
||||
.DS_Store
|
||||
.idea/
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"$schema": "./node_modules/oxlint/configuration_schema.json",
|
||||
"ignorePatterns": ["src/generated/**"],
|
||||
"rules": {
|
||||
"typescript/no-explicit-any": "off",
|
||||
"typescript/no-floating-promises": "error"
|
||||
},
|
||||
"env": {
|
||||
"node": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all"
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
FROM node:24.20.0-bookworm-slim AS base
|
||||
|
||||
WORKDIR /app
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends openssl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN npm install --global pnpm@10.14.0
|
||||
|
||||
FROM base AS dependencies
|
||||
|
||||
COPY package.json pnpm-lock.yaml ./
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
FROM dependencies AS migrate
|
||||
|
||||
COPY prisma ./prisma
|
||||
COPY prisma.config.ts ./
|
||||
CMD ["pnpm", "db:deploy"]
|
||||
|
||||
FROM migrate AS build
|
||||
|
||||
COPY nest-cli.json tsconfig.json tsconfig.build.json ./
|
||||
COPY src ./src
|
||||
RUN pnpm build
|
||||
|
||||
FROM base AS production-dependencies
|
||||
|
||||
COPY package.json pnpm-lock.yaml ./
|
||||
RUN pnpm install --prod --frozen-lockfile
|
||||
|
||||
FROM node:24.20.0-bookworm-slim AS runtime
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV NODE_ENV=production \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
|
||||
RUN mkdir -p /app/data && chown 10001:10001 /app/data
|
||||
|
||||
COPY --from=production-dependencies --chown=10001:10001 /app/node_modules ./node_modules
|
||||
COPY --from=build --chown=10001:10001 /app/dist ./dist
|
||||
COPY --from=build --chown=10001:10001 /app/package.json ./package.json
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=15s --retries=3 \
|
||||
CMD curl --fail --silent --show-error --max-time 2 --output /dev/null "http://127.0.0.1:${APP_PORT:-8080}/health/ready"
|
||||
|
||||
CMD ["node", "dist/main.js"]
|
||||
@@ -1,8 +0,0 @@
|
||||
# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile.
|
||||
# 1. Choose a runtime/toolchain version and pin the base image tag/digest.
|
||||
# 2. Copy dependency manifests first; install from the language lockfile.
|
||||
# 3. Copy source and build. For compiled languages use a separate runtime stage.
|
||||
# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080.
|
||||
# 5. Add non-root, signals and healthcheck according to the current rubric.
|
||||
# 6. Pass runtime config/secrets at startup, never COPY .env or secret files.
|
||||
# This file deliberately has no FROM/CMD: selecting/implementing them is the task.
|
||||
@@ -1,37 +1,170 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: **Пинаев Олег Константинович, БПИ2302**.
|
||||
- Целевая оценка: **5, трек 1 — цена масштабирования**.
|
||||
- Версия: API — коммит `24f32c3`, инфраструктура — `676b38c`, проверки — `5c8bcf0`; основа — skeleton `0ab3a34`. Хеши и образы измеренной версии сохранены в [measurements.json](evidence/measurements.json), актуальной проверенной версии — в [checks.json](evidence/checks.json).
|
||||
- Источники: `README.md`, `COURSE.md`, `contracts/`, документация PostgreSQL, Redis, Node.js и Docker. ИИ использовался при разработке, подготовке проверок, замеров и отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose, Python **3.9+**, Node.js **24.20.0**, pnpm **10.14.0**. Образы: `node:24.20.0-bookworm-slim`, `postgres:17.6-bookworm`, `redis:7.4.5-alpine`, `nginx:1.30.6-alpine`. Для графиков используется закреплённый в команде диапазон Matplotlib, для нагрузки — `grafana/k6:1.3.0`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
pnpm install --frozen-lockfile
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
python -X utf8 scripts/collect-report-checks.py --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
Учебный пароль PostgreSQL заменить в локальном `.env`, сам файл не публиковать. `APP_PORT` задаёт только публичный порт Nginx; в контейнерах API слушает 8080. `POSTGRES_*` — настройки общей БД, `DB_POOL_MAX` — максимум соединений **одной** API, `SESSION_TTL_SECONDS` — срок сессии, `IDEMPOTENCY_TTL_SECONDS` — срок идемпотентности не менее 86400 с, `CACHE_PREFIX` — пространство ключей Redis. Адреса БД и Redis внутри Compose устанавливаются именами сервисов. API не публикуют порты наружу.
|
||||
|
||||
Миграции выполняет отдельный сервис `migrate`; API запускаются после успешного завершения миграции и готовности хранилищ, Nginx — после готовности API. Перед HTTP-проверками должны быть healthy обе API и доступен `/health/ready` через Nginx. `/swagger` и `/docs` доступны через тот же LB. На Windows используется `.cmd`-обёртка с Git Bash; при нестандартной установке Git задаётся `GIT_BASH`. В выполненных проверках локальный `.env` задавал порт **18080**, поэтому их команды используют этот порт. Запуск из полностью чистого clone отдельно не проверялся.
|
||||
|
||||
Безопасная остановка — `docker compose down`: named volumes остаются. **Намеренное удаление учебных данных** — `docker compose down --volumes`; для обычного перезапуска эту команду не использовать.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → **Nginx → NestJS/Fastify × 2 → PostgreSQL**, Redis используется как общий кэш. Из lab2 сохранены пользователи, чаты, членство, Argon2id, Bearer-сессии, cursor-пагинация и согласованность кэша через эпохи. PostgreSQL остаётся источником истины; сессии хранятся в общей таблице, в БД записывается SHA-256 токена. Переключение API не требует повторного входа. Потеря Redis приводит к чтению из PostgreSQL, недоступность БД — к 503.
|
||||
|
||||
Новые элементы lab3 — заголовок `X-Instance-Id`, несколько реплик одного Compose-service, общая идемпотентность POST сообщения, балансировка и проверки отказа. `INSTANCE_ID` получает hostname контейнера. `container_name` и sticky sessions не используются.
|
||||
|
||||
Идемпотентность хранится в PostgreSQL с уникальным ключом `(user_id, chat_id, key)`. В одной транзакции API резервирует ключ через `INSERT ... ON CONFLICT`, создаёт сообщение, изменяет эпоху истории и связывает ключ с сообщением. Конкурирующий повтор получает то же сообщение; другое содержимое с тем же ключом возвращает 409. После истечения TTL запись удаляется при следующем обращении с этим ключом; фонового удаления всех истёкших записей нет. Счётчик истории разделён на 16 строк; запись и увеличение одного счётчика выполняются одним CTE.
|
||||
|
||||
Nginx применяет `least_conn`. Каждый обход получает адреса API из Docker DNS и проверяет `/health/ready` **параллельно**, ограничивая каждый HTTP-запрос одной секундой; после обхода выдерживается пауза 2 с. Backend добавляется после двух успехов и исключается после двух неудач; исчезнувший из DNS адрес удаляется сразу. Сетевые таймауты реплик не суммируются, но DNS, планирование CPU и reload добавляют задержку: пауза 2 с не означает обнаружение за 2 с. Пассивные проверки используют `max_fails=2 fail_timeout=2s`, connect timeout — 2 с, read timeout — 30 с.
|
||||
|
||||
Readiness выполняет `SELECT 1`, liveness не обращается к БД. При SIGTERM/SIGINT ранний хук HTTP-адаптера сначала включает **draining**: `/health/ready` возвращает 503 без запроса к БД, `/health/live` продолжает возвращать 200. API ещё 7 с обслуживает обычные запросы, пока Nginx исключает её после двух неудачных проверок. Задержка согласована с двумя параллельными обходами по `(1 с таймаута + 2 с паузы)` и запасом 1 с на обновление upstream; это расчёт для текущих настроек, не гарантия при произвольной нагрузке хоста.
|
||||
|
||||
После окна draining Fastify прекращает приём новых соединений и ждёт завершения активных HTTP-запросов. Их ответы на этапе закрытия получают `Connection: close`, чтобы keep-alive не задерживал остановку. Только затем NestJS закрывает Prisma/Redis; повторное начало shutdown не запускает второй таймер и повторное закрытие HTTP. Compose, restart-hook и stop-one-hook дают процессу до 30 с до принудительного завершения. Проверка подключения/SQL ограничена 5 с, транзакции — 10 с. Образ разделён на build/migrate/runtime, API работает от UID/GID 10001.
|
||||
|
||||
Обычный профиль — две API по **1 CPU / 512 МиБ**, PostgreSQL — **1 CPU / 512 МиБ**, Nginx — **0,5 CPU / 128 МиБ**, Redis — **0,5 CPU / 256 МиБ**. Пул по умолчанию 15 на API. В исследовании общий бюджет соединений задавался отдельно: по 32/16/8 для 1/2/4 API, до 32 суммарно.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Повторные проверки выполнены 29.10.2026; команды, длительности, метрики и состояния контейнеров собраны в одном [checks.json](evidence/checks.json).
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Проверка | Команда/сценарий | Результат |
|
||||
| --- | --- | --- |
|
||||
| Структура, сборка, линтер | `python scripts/check_contract.py`, `pnpm lint`, `pnpm build` | PASS |
|
||||
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:18080` | PASS, включая `X-Instance-Id` и POST с `Idempotency-Key` |
|
||||
| Общая идемпотентность | `test/idempotency.mjs` через `docker compose run` | PASS: два экземпляра приложения, 12 конкурентных запросов с одним ключом, одинаковое сообщение; другой текст — 409, чужой участник — 404 |
|
||||
| Полный перезапуск | Публичный `tests/resilience.py --mode persistence` с `scripts/restart.cmd` | PASS, 59,507 с; API/Nginx/PG/Redis действительно перезапущены, контейнеры и volumes сохранены, прежние данные и сессия доступны |
|
||||
| Публичный отказ | `tests/resilience.py --mode failover` с `scripts/stop-one.cmd` | PASS: до отказа две реплики, 73 успешные записи, 1 неудачная попытка, первая запись после завершения hook за 7,900 с от его запуска |
|
||||
| Ресурсы и доступ | `test/grade4.py --scenario resources` | PASS: реальные CPU/RAM совпадают с профилем, в снимке контейнеров публичный порт есть только у Nginx, `migrate` завершён с кодом 0 |
|
||||
| Отказ каждой API | `test/grade4.py --scenario failover --concurrency 4 --observe-seconds 10` | PASS: поочерёдно остановлены обе реплики; прежняя сессия, подтверждённые записи и отсутствие дублей проверены |
|
||||
| Поочерёдная замена | `test/grade4.py --scenario rolling --concurrency 4 --observe-seconds 10` | PASS: ID обеих API заменены по очереди, трафик продолжался, контейнер миграции не менялся |
|
||||
| Завершение активного запроса | `node --test --test-reporter=tap test/draining.mjs` | PASS, 2 теста: запрос завершён до закрытия БД, liveness доступен при draining, повторный shutdown идемпотентен, readiness не возвращает запоздалый 200 |
|
||||
| Пороги readiness Nginx | `test/nginx-readiness.sh`, `test/nginx-readiness-parallel.sh` в изолированных контейнерах | PASS: пороги двух успехов/неудач сохранены, четыре пробы запускаются параллельно |
|
||||
| SIGTERM и draining | `python -X utf8 test/check_shutdown.py --base-url http://127.0.0.1:18080` | PASS: readiness `200 → 503 → закрыто`; 114 ответов 503 при liveness=200; LB исключил API через 3,282 с при живом HTTP, остановка 9,963 с, exit code 143 без SIGKILL |
|
||||
| Совместимость миграции | `python -X utf8 test/check_migration_compat.py` | PASS в отдельной временной БД: миграция 28,386 с, 77 записей старого API без ошибок; общая сессия и взаимное чтение старой/новой версии |
|
||||
| Трек 1 | Матрица 1/2/4 API, рост CPU, квота PG и размер пула | 54 сохранённых прогона, [CSV](evidence/scaling.csv), [агрегаты и диагностика](evidence/measurements.json), рисунки ниже |
|
||||
|
||||
Все 16 проверок актуальной версии имеют итоговый PASS. Первый вариант прямого shutdown-теста завершился ошибкой из-за того, что считал клиентские ошибки соединения в парных пробах признаком нездорового HTTP-ответа liveness. В повторе дедлайн проб увеличен с 0,3 до 1 с; HTTP-статусы и транспортные ошибки считаются отдельно. Получены 114 ответов liveness=200 при readiness=503 и ноль транспортных ошибок в этих парах. Неудачная попытка сохранена в `previous_attempts`, хеши прежнего теста — в `source_before_recheck_sha256`. Предыдущие 13 проверок версии без draining, включая историю повтора сохранности, оставлены в компактном `previous_validation`.
|
||||
|
||||
Hook полного перезапуска сохраняет число API и volumes, останавливает Nginx/API/PG/Redis, затем поднимает хранилища, API и LB. Hook отказа сопоставляет полученный `TARGET_INSTANCE_ID` с hostname и оставляет именно эту API остановленной. Сборщик доказательств после проверки восстанавливает две реплики.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Отказ и замена API
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Четыре параллельных клиента через LB читают профиль и пишут сообщения; неподтверждённая запись повторяется с исходным ключом. Метрики ниже относятся к окну действия и 10 с наблюдения после него.
|
||||
|
||||
| Воздействие | Успешные RPS чтения / записи | Ошибки чтения / записи | Наблюдаемый результат |
|
||||
| --- | ---: | ---: | --- |
|
||||
| Остановка первой API | 21,199 / 14,348 | 0 / 0 | После окончания остановки чтение/запись успешны на 8,581 / 8,570 с от начала действия |
|
||||
| Остановка второй API | 19,580 / 13,140 | 0 / 0 | После окончания остановки чтение/запись успешны на 9,259 / 9,304 с от начала действия |
|
||||
| Замена первой API | 19,516 / 12,519 | 0 / 2 | Новый контейнер готов через 31,297 с, сохранившаяся реплика отвечала уже через 0,051 с |
|
||||
| Замена второй API | 18,670 / 12,175 | 0 / 0 | Новый контейнер готов через 24,332 с, сохранившаяся реплика отвечала уже через 0,056 с |
|
||||
|
||||
В сценарии отказа проверена 1001 подтверждённая запись, в сценарии замены — 1082; потеря подтверждённых сообщений и дубли не обнаружены. Последний ответ останавливаемой API наблюдался через 4,718 и 5,443 с от начала действия: в окне draining она ещё обслуживала запросы до исключения из LB. Это наблюдение конкретных прогонов, а не теоретическая верхняя граница исключения backend. Время создания нового контейнера не равно времени недоступности сервиса: другая API продолжала обслуживать запросы. Нулевые ошибки двух отказов не означают их отсутствие при любых нагрузках: публичный сценарий и первая замена зафиксировали переходные неудачные попытки.
|
||||
|
||||
### Трек 1: цена масштабирования
|
||||
|
||||
Гипотеза: дополнительные API при неизменном суммарном бюджете не дают линейного роста, поскольку делят CPU, создают отдельные пулы и используют одну БД. Увеличение нужного ресурса или ограничение невыгодного параллелизма должно улучшать качество.
|
||||
|
||||
Генератор и наблюдатели делят ресурсы с приложением. Набор синтетический: новый чат с 30 сообщениями на прогон, первая страница до 50 сообщений, 80% чтений и 20% отправок. Время регистрации/входа не входит в измеряемое окно. База между прогонами не очищалась.
|
||||
|
||||
В основной матрице k6 одна итерация равна одному запросу, задан постоянный arrival rate, 128 VU на сценарий, таймаут 5 с, POST без автоматических повторов. Измерение 30 с после 10 с смешанного прогрева. 36 прогонов: 1/2/4 API × общий CPU 1/2 × 40/80 RPS × три повтора. Общая RAM API — 1 ГиБ, общий максимум соединений — 32, PG — 1 CPU / 512 МиБ во всех вариантах. Порядок попарный: `one-small → one → two-small → two → four-small → four`; внутри каждого профиля 40/80 чередуются.
|
||||
|
||||
Экспериментальный критерий качества — ошибки ≤1% и **p95 всех запросов ≤1000 мс**. Достаточная подача — 98–102% планового числа начатых запросов и не более 2% пропусков.
|
||||
|
||||
| API | CPU суммарно | CPU / RAM одной API | Медиана p95 на 40 RPS, мс | Проход / нарушение / недоподача |
|
||||
| ---: | ---: | --- | ---: | --- |
|
||||
| 1 | 1 | 1 / 1024 МиБ | 1363 | 0 / 3 / 0 |
|
||||
| 2 | 1 | 0,5 / 512 МиБ | 1338 | 0 / 3 / 0 |
|
||||
| 4 | 1 | 0,25 / 256 МиБ | 808 | 2 / 0 / 1 |
|
||||
| 1 | 2 | 2 / 1024 МиБ | 751 | 2 / 1 / 0 |
|
||||
| 2 | 2 | 1 / 512 МиБ | **452** | **3 / 0 / 0** |
|
||||
| 4 | 2 | 0,5 / 256 МиБ | 1044 | 1 / 2 / 0 |
|
||||
|
||||
При фиксированных 2 CPU две API прошли все повторы на 40 RPS, четыре — один из трёх. Рост числа процессов не улучшил качество. Четыре API при 1 CPU дали два прохода.
|
||||
|
||||
При увеличении CPU **двух API с 1 до 2** p95 трёх повторов изменился с **1119 / 1757 / 1338** на **777 / 452 / 404 мс**. Все шесть повторов получили достаточную нагрузку, ошибок нет. Медиана p95 снизилась примерно **в 2,96 раза**, медиана максимальной среди API доли throttled-периодов — **18,83 → 1,30%**. Это повторяемое улучшение задержки при прежних RAM, пуле и БД.
|
||||
|
||||
На 80 RPS у двух API с общим 1 CPU было 658 / 473 / 499 ошибок, при 2 CPU — 0 / 31 / 0. У четырёх API — 219 / 1 / 101 против 0 / 3 / 0.
|
||||
|
||||

|
||||
|
||||
### Ограничение PostgreSQL и увеличение его квоты
|
||||
|
||||
Раннее сравнение: четыре API с суммарными 2 CPU / 1 ГиБ, пул по 15, то есть до 60 соединений. Linux Node-генератор через Nginx, 60 заданных RPS, по три повтора по 120 с после 20 с прогрева чтениями. PostgreSQL получил 1 CPU вместо 0,5, RAM осталась 512 МиБ. Облегчённый наблюдатель v3 держал одно постоянное psql-соединение.
|
||||
|
||||
| Метрика | PG 0,5 CPU | PG 1 CPU |
|
||||
| --- | ---: | ---: |
|
||||
| Средняя загрузка PG, CPU | 0,32–0,39 | 0,32–0,39 |
|
||||
| Загрузка относительно выделенной квоты | 64–78% | 32–39% |
|
||||
| Периоды с CPU throttling PG | 33–55% | 3–8% |
|
||||
| Медиана p95 успешных запросов, мс | 826 | **296** |
|
||||
| Ошибки за три повтора | 121 | **1** |
|
||||
| Средние успешные RPS | 52,81 | 53,00 |
|
||||
| Медиана среднего получения соединения, мс | 23,61 | **1,17** |
|
||||
|
||||
При квоте 0,5 CPU среднее потребление ниже лимита, но во время всплесков квота регулярно исчерпывалась. В проблемной области наблюдались ошибки получения соединений и ожидания блокировок записи; четыре API не добавляли ресурсов общей БД. Увеличение CPU базы снизило p95 примерно в 2,8 раза и ошибки при почти одинаковом числе отправленных запросов. Это подтверждает влияние CPU-квоты PG на задержки и надёжность.
|
||||
|
||||
Последующий контроль **PG 1 → 2 CPU** выполнен отдельно: две API с общими 2 CPU / 1 ГиБ, пул 32, k6, 80 RPS, одинаковые 30 с смешанного прогрева и 30 с измерения. При 1 CPU получены два прохода и одно нарушение задержки: p95 **793 / 2026 / 744 мс**. При 2 CPU — два недоподанных повтора и один проход: **2122 / 1555 / 713 мс**. Throttling PG исчез, но устойчивого выигрыша не получено. Следовательно, усиление БД после снятия жёсткой квоты не гарантирует дальнейшего роста.
|
||||
|
||||
### Ограничение общего числа соединений
|
||||
|
||||
Четыре API, PG 1 CPU / 512 МиБ, остальные ресурсы прежние. Сравниваются пулы по 15 и по 8 на API: общий максимум **60 → 32**. Node-генератор, 100 заданных RPS, по три повтора по 60 с после 15 с прогрева чтением.
|
||||
|
||||
| Метрика | 60 соединений | 32 соединения |
|
||||
| --- | ---: | ---: |
|
||||
| Успешных запросов за 180 с | 13 447 | **14 543** |
|
||||
| Средние успешные RPS | 74,71 | **80,79** |
|
||||
| Ошибки | 488 | **4** |
|
||||
| Ошибки получения соединений | 418 | **0** |
|
||||
| Медиана успешных RPS повторов | 83,93 | 82,05 |
|
||||
| Медиана p95 успешных запросов, мс | 289 | 389 |
|
||||
|
||||
Средний полезный поток вырос **на 8,15%**, ошибок стало меньше. Однако все 488 ошибок исходного варианта пришлись на один нестабильный повтор, а фактически отправленный поток вырос с 77,42 до 80,82 RPS. Это наблюдаемый выигрыш надёжности и среднего полезного потока, а не доказательство ускорения каждого запроса или увеличения предельного RPS. Ограничение пула согласуется с гипотезой защиты общей БД от невыгодного параллелизма; само увеличение числа соединений производительность не гарантирует.
|
||||
|
||||

|
||||
|
||||
### Исходные данные и повторение
|
||||
|
||||
В `evidence/` оставлены **пять файлов**: `checks.json`, `measurements.json`, `scaling.csv`, `scaling.png`, `improvements.png`. JSON содержит 54 прогона из семи исходных серий, их HTTP-агрегаты, p50/p95/p99, CPU/RAM, очереди, SQL-состояния, профили, manifest и хеши исходных результатов. CSV — плоская таблица тех же прогонов. Запросы по отдельности, stdout/stderr, временные ряды, экспериментальные README и дубли рисунков удалены. Ненужные предварительные и отменённые серии в отчёт не включены.
|
||||
|
||||
Повтор основной матрицы в отдельный временный каталог:
|
||||
|
||||
```sh
|
||||
python -X utf8 scripts/run-saturation.py --profiles one-small one two-small two four-small four --confirm-rates 40 80 --confirm-seconds 30 --no-expand --output .artifacts/scaling/repeat
|
||||
python -X utf8 scripts/summarize-scaling-diagnostics.py .artifacts/scaling/repeat
|
||||
python -X utf8 scripts/audit-saturation.py .artifacts/scaling/repeat
|
||||
```
|
||||
|
||||
После серии runner восстанавливает обычные две API. Варианты исторических сравнений требуют соответствующих квот PG и размеров пулов, записанных в `measurements.json`; их не следует повторять с текущим пулом 8 и выдавать за старый вариант с пулом 15. Инструкции измерителей — в `scripts/diagnostics/`.
|
||||
|
||||
Пересчитать CSV и два рисунка из сохранённых данных без новой нагрузки:
|
||||
|
||||
```sh
|
||||
uv run --no-project --with 'matplotlib>=3.9,<4' python scripts/report-evidence.py --plot
|
||||
```
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Один Docker-host, одна БД, один Redis и один LB. Проверен отказ и замена API, но не выживание при потере узла PostgreSQL, Nginx или всего ноутбука. Резервирование хранилищ и распределённый кластер не реализованы. Rolling-проверка использовала тот же образ; совместимость схемы lab2/lab3 проверена отдельно, но старая API из lab2 не умеет `Idempotency-Key`, поэтому во время смешанного развёртывания нельзя обещать идемпотентность POST, попавшего на старую версию.
|
||||
|
||||
В штатном режиме readiness зависит от ответа PG: краткие задержки базы способны исключить живую API и усилить ошибки. Draining использует 7-секундное окно под текущие параллельные проверки Nginx; при изменении таймаутов, паузы или порогов LB это окно нужно пересогласовать. Активные запросы завершаются в пределах общего 30-секундного бюджета Docker; бесконечно долгие запросы этот бюджет не продлевают. Чистый bootstrap на другой машине и под другим Docker-окружением не проверен. Нагрузочные сравнения не повторялись после добавления draining: их данные относятся к прежнему образу, проверки остановки и отказов — к актуальному.
|
||||
|
||||
Итог исследования: две API — устойчивый рабочий вариант при текущем бюджете; увеличение их CPU улучшило задержки, усиление PG с 0,5 до 1 CPU улучшило работу зависимого пути, ограничение соединений сопровождалось снижением ошибок. Дальнейший рост реплик, пула или квоты PG сам по себе выигрыша не гарантирует. Следующий шаг для уточнения границы — независимый генератор или больший запас хоста и чередование вариантов с достаточной фактической подачей.
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Topology worksheet, deliberately NOT a runnable stack (no application supplied).
|
||||
# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment.
|
||||
# Service names below are logical roles, not imposed container names.
|
||||
x-lab-number: 3
|
||||
x-required-roles:
|
||||
- api
|
||||
- postgres
|
||||
- load_balancer
|
||||
# TODO: implement real images/builds, network wiring, volumes, healthchecks,
|
||||
# credentials and bootstrap. Details: contracts/infrastructure.json and README.md.
|
||||
services: {}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
services:
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
target: runtime
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
cpus: '1.0'
|
||||
mem_limit: 512m
|
||||
deploy:
|
||||
replicas: 2
|
||||
command: ['sh', '-c', 'INSTANCE_ID=$$(hostname) exec node dist/main.js']
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-15}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
CACHE_PREFIX: ${CACHE_PREFIX:-messenger:cache:v1}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- curl
|
||||
- --fail
|
||||
- --silent
|
||||
- --show-error
|
||||
- --max-time
|
||||
- '2'
|
||||
- --output
|
||||
- /dev/null
|
||||
- http://127.0.0.1:8080/health/ready
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
start_period: 20s
|
||||
retries: 5
|
||||
stop_grace_period: 30s
|
||||
|
||||
nginx:
|
||||
image: nginx:1.30.6-alpine
|
||||
restart: unless-stopped
|
||||
cpus: '0.5'
|
||||
mem_limit: 128m
|
||||
ports:
|
||||
- '127.0.0.1:${APP_PORT:-8080}:80'
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/start-nginx.sh:/etc/nginx/start-nginx.sh:ro
|
||||
entrypoint: ['/bin/sh', '/etc/nginx/start-nginx.sh']
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
target: migrate
|
||||
cpus: '1.0'
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
postgres:
|
||||
image: postgres:17.6-bookworm
|
||||
restart: unless-stopped
|
||||
cpus: '1.0'
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-chat}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-app}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-local_dev_password}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
start_period: 10s
|
||||
retries: 10
|
||||
stop_grace_period: 30s
|
||||
|
||||
redis:
|
||||
image: redis:7.4.5-alpine
|
||||
restart: unless-stopped
|
||||
cpus: '0.5'
|
||||
mem_limit: 256m
|
||||
command:
|
||||
['redis-server', '--appendonly', 'yes', '--appendfsync', 'everysec']
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
stop_grace_period: 30s
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,8 @@
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
DB_POOL_MAX: '8'
|
||||
deploy:
|
||||
replicas: 4
|
||||
cpus: '1.0'
|
||||
mem_limit: 512m
|
||||
@@ -0,0 +1,8 @@
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
DB_POOL_MAX: '8'
|
||||
deploy:
|
||||
replicas: 4
|
||||
cpus: '0.25'
|
||||
mem_limit: 256m
|
||||
@@ -0,0 +1,8 @@
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
DB_POOL_MAX: '8'
|
||||
deploy:
|
||||
replicas: 4
|
||||
cpus: '0.5'
|
||||
mem_limit: 256m
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
api:
|
||||
deploy:
|
||||
replicas: 1
|
||||
cpus: '1.0'
|
||||
mem_limit: 1024m
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
api:
|
||||
deploy:
|
||||
replicas: 1
|
||||
cpus: '2.0'
|
||||
mem_limit: 1024m
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
api:
|
||||
deploy:
|
||||
replicas: 2
|
||||
cpus: '0.5'
|
||||
mem_limit: 512m
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
api:
|
||||
deploy:
|
||||
replicas: 2
|
||||
cpus: '1.0'
|
||||
mem_limit: 512m
|
||||
@@ -1,5 +0,0 @@
|
||||
# Доказательства
|
||||
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 141 KiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
dataset,variant,label,replicas,api_cpu,api_ram_mib,pool_total,pg_cpu,pg_ram_mib,offered_rps,duration_seconds,warmup_seconds,attempted,successful,errors,successful_rps,attempted_rps,delivery_fraction,delivery_valid,p50_ms,p95_ms,p99_ms,all_request_p95_ms,api_cpu_cores,api_peak_memory_mib_sum,api_max_throttle_percent,pg_cpu_cores,pg_peak_memory_mib,pg_throttle_percent,host_cpu_pressure_percent,acquisition_mean_ms,acquisition_errors,pool_max_waiting,nginx_min_ready,diagnostics_complete
|
||||
api-cpu,one-small,one-small-r40-confirmation-4,1,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,26.847786,1363.2556869999999,1703.2769420000002,1363.2556869999999,0.4689939283545841,180.99609375,6.64451827242525,0.33446107407407405,246.28125,4.10958904109589,40.80378148148148,0.5064286857716331,0,0,1,True
|
||||
api-cpu,one-small,one-small-r80-confirmation-5,1,1.0,1024.0,32,1.0,512.0,80,30,10,2221,1145,1076,38.166666666666664,74.03333333333333,0.9254166666666667,False,675.249624,3612.0851274,4642.559255439987,3913.775195,0.5187701269052575,220.72265625,9.00900900900901,0.3064097142857143,254.94921875,2.9661016949152543,48.179525,35.72717908921662,0,29,0,True
|
||||
api-cpu,one-small,one-small-r40-confirmation-6,1,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,86.459411,1682.559437,2313.335861,1682.559437,0.5015700869077656,242.703125,6.188925081433225,0.46909210344827584,253.98828125,14.057507987220447,51.16733103448276,5.8440807538536665,0,8,1,True
|
||||
api-cpu,one-small,one-small-r80-confirmation-7,1,1.0,1024.0,32,1.0,512.0,80,30,10,2213,1736,477,57.86666666666667,73.76666666666667,0.9220833333333334,False,146.5581615,2659.44941475,4480.5786870500015,2288.1632106000006,0.46460503597122305,243.12109375,4.487179487179487,0.36433525925925925,262.03125,3.4615384615384617,45.44102222222222,3.753992680559904,0,4,0,True
|
||||
api-cpu,one-small,one-small-r40-confirmation-8,1,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,70.035875,1322.7580870000002,2087.550949,1322.7580870000002,0.5208842712940144,242.90234375,3.7900874635568513,0.41083307142857145,264.1015625,7.023411371237458,49.05793571428572,0.7800225128654482,0,0,1,True
|
||||
api-cpu,one-small,one-small-r80-confirmation-9,1,1.0,1024.0,32,1.0,512.0,80,30,10,2401,2259,142,75.3,80.03333333333333,1.0004166666666667,True,23.386415,623.2823438999998,816.6689335600003,623.018676,0.5334591003791113,244.5078125,2.229299363057325,0.39796175862068967,285.01171875,2.2435897435897436,38.453079310344826,0.3408927703400665,0,0,1,True
|
||||
api-cpu,one,one-r40-confirmation-10,1,2.0,1024.0,32,1.0,512.0,40,30,10,1201,1158,43,38.6,40.03333333333333,1.0008333333333332,True,28.589108000000003,2413.214785799998,3846.0589049900004,2319.3248719999997,0.4332059223037218,200.88671875,0.0,0.3532345,283.921875,12.927756653992395,39.199192857142855,16.47106694684052,0,8,0,True
|
||||
api-cpu,one,one-r80-confirmation-11,1,2.0,1024.0,32,1.0,512.0,80,30,10,2270,1554,716,51.8,75.66666666666667,0.9458333333333333,False,150.2324305,1506.410031149999,2727.858566280001,1849.6533385999996,0.5490057858528841,222.453125,0.33112582781456956,0.370641,289.72265625,5.535055350553505,50.593651724137935,14.885148709623188,0,18,0,True
|
||||
api-cpu,one,one-r40-confirmation-12,1,2.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,18.939717,418.903777,957.1179070000001,418.903777,0.3838068209416091,244.421875,0.0,0.2989388571428571,291.7734375,3.040540540540541,40.00857857142857,0.48221057456625477,0,0,1,True
|
||||
api-cpu,one,one-r80-confirmation-13,1,2.0,1024.0,32,1.0,512.0,80,30,10,2400,2400,0,80.0,80.0,1.0,True,205.252095,1378.379113749999,2144.6287917899995,1378.379113749999,0.6651004694501743,244.953125,0.0,0.49829189285714287,301.0703125,6.0606060606060606,46.72890714285714,3.464600104437483,0,10,1,True
|
||||
api-cpu,one,one-r40-confirmation-14,1,2.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,17.859334999999998,751.296029,1386.120503,751.296029,0.35999272408798466,245.04296875,0.0,0.34728475862068964,299.8671875,7.540983606557377,34.73657931034483,0.27413348812545674,0,0,1,True
|
||||
api-cpu,one,one-r80-confirmation-15,1,2.0,1024.0,32,1.0,512.0,80,30,10,2369,2369,0,78.96666666666667,78.96666666666667,0.9870833333333333,True,38.92324300000001,966.6013613999995,1567.104490280001,966.6013613999995,0.577001595871108,245.30078125,0.2932551319648094,0.4352526666666667,302.50390625,3.4482758620689653,43.26645185185185,0.42102274316591715,0,0,1,True
|
||||
api-cpu,two-small,two-small-r40-confirmation-16,2,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,48.445418,1118.8671709999999,1544.904627,1118.8671709999999,0.5745100966367777,248.95703125,21.172638436482085,0.30872242857142856,302.90234375,2.5179856115107913,54.86230357142857,0.6318267755826504,0,1,1,True
|
||||
api-cpu,two-small,two-small-r80-confirmation-17,2,1.0,1024.0,32,1.0,512.0,80,30,10,2371,1713,658,57.1,79.03333333333333,0.9879166666666667,True,25.045635,691.4786511999989,1143.6126581199976,588.881168,0.6049542357784888,330.078125,21.5625,0.33328606896551727,293.62109375,3.4013605442176873,54.181648275862074,41.866071340397994,0,33,0,True
|
||||
api-cpu,two-small,two-small-r40-confirmation-18,2,1.0,1024.0,32,1.0,512.0,40,30,10,1206,1206,0,40.2,40.2,1.005,True,93.989056,1757.4142032500001,2544.2509913000004,1757.4142032500001,0.5981754698234121,373.01171875,18.38006230529595,0.4452461111111111,276.8125,7.357859531772576,62.34872962962963,2.1698786494695783,0,10,2,True
|
||||
api-cpu,two-small,two-small-r80-confirmation-19,2,1.0,1024.0,32,1.0,512.0,80,30,10,1647,1174,473,39.13333333333333,54.9,0.68625,False,1051.8127515,4573.172054199999,4829.25868523,5009.7729897,0.6464339585556628,377.83984375,51.006711409395976,0.399076,274.375,3.3112582781456954,60.90348571428571,156.04911624711062,0,42,0,True
|
||||
api-cpu,two-small,two-small-r40-confirmation-20,2,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,62.321341000000004,1338.075269,1721.736195,1338.075269,0.5502207297293924,398.22265625,18.83116883116883,0.38870475862068965,274.83203125,4.777070063694268,56.438,1.3796656221576238,0,2,2,True
|
||||
api-cpu,two-small,two-small-r80-confirmation-21,2,1.0,1024.0,32,1.0,512.0,80,30,10,1941,1442,499,48.06666666666667,64.7,0.80875,False,1037.0655474999999,3700.5747892000004,4145.45928118,5000.846076000001,0.6176817281440554,421.109375,38.67924528301887,0.3516609642857143,273.04296875,2.364864864864865,60.43923571428571,90.55197401443517,0,42,0,True
|
||||
api-cpu,two,two-r40-confirmation-22,2,2.0,1024.0,32,1.0,512.0,40,30,10,1200,1200,0,40.0,40.0,1.0,True,25.912608499999997,776.7075832999998,1450.893997529999,776.7075832999998,0.4979710890309068,265.3203125,0.6451612903225806,0.32912351724137934,281.47265625,3.061224489795918,42.24538275862069,0.7639616891295447,0,2,2,True
|
||||
api-cpu,two,two-r80-confirmation-23,2,2.0,1024.0,32,1.0,512.0,80,30,10,2328,2328,0,77.6,77.6,0.97,False,69.108862,1732.0478621500022,3892.17638994,1732.0478621500022,0.9329810136367049,346.1171875,4.229607250755287,0.5218088518518519,277.859375,9.491525423728813,52.210596296296295,53.69368175558641,0,44,1,True
|
||||
api-cpu,two,two-r40-confirmation-24,2,2.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,13.742251,451.573919,844.77165,451.573919,0.4716363654058743,407.20703125,1.3029315960912053,0.270069,272.84375,3.169014084507042,27.00871111111111,1.9952917991740617,0,0,2,True
|
||||
api-cpu,two,two-r80-confirmation-25,2,2.0,1024.0,32,1.0,512.0,80,30,10,2010,1979,31,65.96666666666667,67.0,0.8375,False,171.42565199999999,3153.9972928999987,4344.177250020001,3644.38635635,0.8843039698112047,474.21484375,5.379746835443038,0.5575650769230769,258.26171875,5.944055944055944,57.75113076923078,163.35389117991014,0,53,1,True
|
||||
api-cpu,two,two-r40-confirmation-26,2,2.0,1024.0,32,1.0,512.0,40,30,10,1200,1200,0,40.0,40.0,1.0,True,20.1536085,404.04996024999986,675.41993862,404.04996024999986,0.4924617244558488,498.21484375,1.7857142857142856,0.300782962962963,256.08984375,4.545454545454546,39.64784814814814,0.270736000000329,0,0,2,True
|
||||
api-cpu,two,two-r80-confirmation-27,2,2.0,1024.0,32,1.0,512.0,80,30,10,2400,2400,0,80.0,80.0,1.0,True,18.8891145,372.4251258499997,658.5146975299939,372.4251258499997,0.6646231235220466,502.234375,1.2944983818770228,0.3887145,260.2265625,3.678929765886288,42.85142142857143,0.2999987548356928,0,0,2,True
|
||||
api-cpu,four-small,four-small-r40-confirmation-28,4,1.0,1024.0,32,1.0,512.0,40,30,10,1143,930,213,31.0,38.1,0.9525,False,563.57299,4513.76266625,4855.86492899,5002.5158862,0.7358752597636645,409.2109375,58.11688311688312,0.32318432142857145,257.17578125,1.3745704467353952,65.94500357142857,79.96626631756442,9,15,0,True
|
||||
api-cpu,four-small,four-small-r80-confirmation-29,4,1.0,1024.0,32,1.0,512.0,80,30,10,1505,1286,219,42.86666666666667,50.166666666666664,0.6270833333333333,False,1625.0898905,4334.39354775,4838.7090065,5011.016073,0.8051559679951759,506.86328125,65.0326797385621,0.4044294482758621,244.546875,4.113924050632911,83.87952413793104,290.3396483314464,0,34,1,True
|
||||
api-cpu,four-small,four-small-r40-confirmation-30,4,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,16.783684,664.626691,1316.7376120000001,664.626691,0.5542242782175116,573.66796875,24.600638977635782,0.2841889310344828,235.04296875,2.903225806451613,54.391493103448276,4.554068587618393,0,0,4,True
|
||||
api-cpu,four-small,four-small-r80-confirmation-31,4,1.0,1024.0,32,1.0,512.0,80,30,10,2209,2208,1,73.6,73.63333333333334,0.9204166666666667,False,645.10074,2796.4947147000003,3875.9293856899994,2819.614260199997,0.9155072796500234,664.3203125,76.43312101910828,0.4689557777777778,229.32421875,3.7037037037037033,81.98151481481482,170.3762436461243,0,32,3,True
|
||||
api-cpu,four-small,four-small-r40-confirmation-32,4,1.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,32.074924,808.456865,1587.9421240000001,808.456865,0.6178738390069644,688.2890625,32.05128205128205,0.31956855172413795,230.54296875,2.9702970297029703,61.64351034482759,1.8690922410455422,0,6,4,True
|
||||
api-cpu,four-small,four-small-r80-confirmation-33,4,1.0,1024.0,32,1.0,512.0,80,30,10,1542,1441,101,48.03333333333333,51.4,0.6425,False,1824.245409,4418.3310790000005,4710.956379199998,5000.4370198,0.7125214524412071,699.04296875,72.88135593220339,0.3575675172413793,230.703125,2.2082018927444795,80.39199655172415,422.90973522269843,0,55,1,True
|
||||
api-cpu,four,four-r40-confirmation-34,4,2.0,1024.0,32,1.0,512.0,40,30,10,1202,1202,0,40.06666666666667,40.06666666666667,1.0016666666666667,True,27.289679,1443.624807400003,3055.7631298200004,1443.624807400003,0.7505123280542068,431.33984375,8.653846153846153,0.32891293103448277,238.578125,3.1847133757961785,53.46226896551725,18.29400296340266,0,11,4,True
|
||||
api-cpu,four,four-r80-confirmation-35,4,2.0,1024.0,32,1.0,512.0,80,30,10,2263,2263,0,75.43333333333334,75.43333333333334,0.9429166666666666,False,39.859282,2765.0429346000005,3836.9857128000017,2765.0429346000005,1.0209338416080294,528.44140625,10.946745562130179,0.41786425,245.6484375,4.09556313993174,54.01412142857143,131.48510588421536,0,30,3,True
|
||||
api-cpu,four,four-r40-confirmation-36,4,2.0,1024.0,32,1.0,512.0,40,30,10,1200,1200,0,40.0,40.0,1.0,True,18.764722999999996,678.1664768499995,1272.8332105600002,678.1664768499995,0.6441531318902833,622.16015625,9.523809523809524,0.3159538571428571,237.0390625,3.492063492063492,46.82114285714285,1.2023917273639246,0,1,4,True
|
||||
api-cpu,four,four-r80-confirmation-37,4,2.0,1024.0,32,1.0,512.0,80,30,10,2107,2104,3,70.13333333333334,70.23333333333333,0.8779166666666667,False,213.502144,2947.47390755,3767.175285259999,2950.8826460999994,1.0004379639593417,631.91796875,14.61038961038961,0.47014373076923077,239.87890625,3.1914893617021276,69.7715076923077,196.41256616530842,0,33,3,True
|
||||
api-cpu,four,four-r40-confirmation-38,4,2.0,1024.0,32,1.0,512.0,40,30,10,1201,1201,0,40.03333333333333,40.03333333333333,1.0008333333333332,True,28.576715,1044.026936,1622.646796,1044.026936,0.579652601289575,699.5078125,4.870129870129871,0.30819364285714285,241.28515625,0.9803921568627451,53.72631785714286,1.4715062709821798,0,2,4,True
|
||||
api-cpu,four,four-r80-confirmation-39,4,2.0,1024.0,32,1.0,512.0,80,30,10,2306,2306,0,76.86666666666666,76.86666666666666,0.9608333333333333,False,178.069107,2106.622289,2973.419937949996,2106.622289,0.9669768640103195,741.734375,5.7057057057057055,0.46597120689655175,241.16015625,2.6706231454005933,65.60908275862069,111.53861144349652,0,25,4,True
|
||||
pg-cpu,pg0.5,four-r60-repeat1,4,2.0,1024.0,60,0.5,512.0,60,120,20,6702,6669,33,55.575,55.85,0.9308333333333333,False,25.463,576.18,2086.454,,0.6517002565561196,412.0703125,4.8293089092422985,0.3220422288135593,267.37109375,33.30536912751678,52.56348644067796,23.609644087794603,20,2,4,True
|
||||
pg-cpu,pg0.5,four-r60-repeat2,4,2.0,1024.0,60,0.5,512.0,60,120,20,5940,5864,76,48.86666666666667,49.5,0.825,False,75.384,1570.711,3408.81,,0.7329807440781919,513.3828125,7.136997538966367,0.3908916525423729,277.22265625,54.67445742904842,69.41428050847458,43.96043944981507,47,3,3,True
|
||||
pg-cpu,pg0.5,four-r60-repeat3,4,2.0,1024.0,60,0.5,512.0,60,120,20,6490,6478,12,53.983333333333334,54.083333333333336,0.9013888888888889,False,25.401,825.578,2398.068,,0.6025514401016459,598.08203125,5.189456342668864,0.340874731092437,267.171875,38.232865400495456,52.90928487394958,10.25498626471663,1,3,4,True
|
||||
pg-cpu,pg1,four-r60-repeat1,4,2.0,1024.0,60,1.0,512.0,60,120,20,6613,6613,0,55.108333333333334,55.108333333333334,0.9184722222222222,False,17.841,208.134,803.385,,0.6611089620356754,410.28125,5.546357615894039,0.3187262966101695,118.35546875,2.9925187032418954,43.279896610169494,0.9549692434151781,0,0,4,True
|
||||
pg-cpu,pg1,four-r60-repeat2,4,2.0,1024.0,60,1.0,512.0,60,120,20,6191,6191,0,51.59166666666667,51.59166666666667,0.8598611111111111,False,25.284,296.119,886.275,,0.6961733670166016,490.05859375,5.037159372419488,0.38550305128205126,121.6171875,5.091819699499165,51.8048282051282,1.1744822766036151,0,0,4,True
|
||||
pg-cpu,pg1,four-r60-repeat3,4,2.0,1024.0,60,1.0,512.0,60,120,20,6278,6277,1,52.30833333333333,52.31666666666667,0.8719444444444444,False,24.799,551.315,1469.251,,0.6838688608643495,591.7265625,5.523495465787304,0.3920775338983051,190.1015625,8.00990916597853,52.54273220338983,10.537646633393535,0,0,4,True
|
||||
pool-limit,pool60,four-r100-repeat1,4,2.0,1024.0,60,1.0,512.0,100,60,15,3753,3265,488,54.416666666666664,62.55,0.6255,False,49.021,1712.812,3905.626,,0.8365694303748847,434.23828125,9.283387622149837,0.5837867719298245,274.10546875,24.006908462867013,67.08071052631578,147.1668448014528,418,7,2,True
|
||||
pool-limit,pool60,four-r100-repeat2,4,2.0,1024.0,60,1.0,512.0,100,60,15,5036,5036,0,83.93333333333334,83.93333333333334,0.8393333333333334,False,23.084,263.596,461.061,,0.8666951803432426,507.84375,7.540983606557377,0.4340725172413793,168.43359375,5.6951423785594635,53.356810344827586,0.22145806512495533,0,0,4,True
|
||||
pool-limit,pool60,four-r100-repeat3,4,2.0,1024.0,60,1.0,512.0,100,60,15,5146,5146,0,85.76666666666667,85.76666666666667,0.8576666666666667,False,26.017,288.733,710.623,,0.8637168989788998,594.375,6.688963210702341,0.44118370689655173,201.85546875,4.8903878583473865,54.0543,3.7378880537782533,0,0,4,True
|
||||
pool-limit,pool32,four-r100-repeat1,4,2.0,1024.0,32,1.0,512.0,100,60,15,4558,4558,0,75.96666666666667,75.96666666666667,0.7596666666666667,False,46.854,573.083,1464.894,,0.9994498117976172,417.15234375,9.75206611570248,0.4833926896551724,200.52734375,4.3478260869565215,61.618993103448275,4.1768588468573355,0,5,4,True
|
||||
pool-limit,pool32,four-r100-repeat2,4,2.0,1024.0,32,1.0,512.0,100,60,15,4927,4923,4,82.05,82.11666666666666,0.8211666666666667,False,22.419,389.339,1936.349,,0.8844784302484496,486.140625,7.8817733990147785,0.423828186440678,201.1953125,3.523489932885906,50.59918474576271,12.409012795600338,0,10,4,True
|
||||
pool-limit,pool32,four-r100-repeat3,4,2.0,1024.0,32,1.0,512.0,100,60,15,5062,5062,0,84.36666666666666,84.36666666666666,0.8436666666666667,False,20.843,373.895,1093.443,,0.829926970799795,589.52734375,5.509181969949917,0.4455904406779661,207.91015625,6.065573770491803,49.169627118644065,5.397192570060225,0,6,4,True
|
||||
pg-upper-cpu,pg1,two-r80-confirmation-4,2,2.0,1024.0,32,1.0,512.0,80,30,30,2401,2401,0,80.03333333333333,80.03333333333333,1.0004166666666667,True,28.122368,793.449891,1037.210309,793.449891,0.8285819189203447,251.90625,3.1645569620253164,0.4253926551724138,243.45703125,3.5830618892508146,46.79261724137931,0.7895206962982697,0,5,2,True
|
||||
pg-upper-cpu,pg1,two-r80-confirmation-5,2,2.0,1024.0,32,1.0,512.0,80,30,30,2369,2369,0,78.96666666666667,78.96666666666667,0.9870833333333333,True,72.155286,2026.0931697999993,4108.34764632,2026.0931697999993,0.8320800483649098,333.4140625,5.9405940594059405,0.445373724137931,262.19921875,3.8461538461538463,49.43855517241379,40.801660400828624,0,37,1,True
|
||||
pg-upper-cpu,pg1,two-r80-confirmation-6,2,2.0,1024.0,32,1.0,512.0,80,30,30,2401,2401,0,80.03333333333333,80.03333333333333,1.0004166666666667,True,35.408502,744.1695299999999,1372.70999,744.1695299999999,0.7643656472272428,393.8984375,2.5559105431309903,0.4581740384615385,270.90234375,5.190311418685121,53.7610923076923,7.759117931556264,0,19,2,True
|
||||
pg-upper-cpu,pg2,two-r80-confirmation-4,2,2.0,1024.0,32,2.0,512.0,80,30,30,2326,2326,0,77.53333333333333,77.53333333333333,0.9691666666666666,False,26.308654,2121.76918975,3430.25811825,2121.76918975,0.7933795254496,251.75390625,3.257328990228013,0.4538005769230769,297.5078125,0.0,46.54303846153846,22.08989025991319,0,19,2,True
|
||||
pg-upper-cpu,pg2,two-r80-confirmation-5,2,2.0,1024.0,32,2.0,512.0,80,30,30,2306,1978,328,65.93333333333334,76.86666666666666,0.9608333333333333,False,128.46333049999998,1639.3791287499998,4067.77854259,1555.304007,0.7760643068565013,329.14453125,2.9605263157894735,0.47419825925925924,307.85546875,0.0,58.73266296296297,33.51751995267285,0,26,0,True
|
||||
pg-upper-cpu,pg2,two-r80-confirmation-6,2,2.0,1024.0,32,2.0,512.0,80,30,30,2399,2399,0,79.96666666666667,79.96666666666667,0.9995833333333334,True,21.56194,712.7126198999999,1247.7373979799997,712.7126198999999,0.7646030358560834,354.8828125,4.220779220779221,0.46384507142857145,313.56640625,0.0,46.342039285714286,1.1575000698940525,0,1,2,True
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
@@ -0,0 +1,23 @@
|
||||
events {}
|
||||
|
||||
http {
|
||||
upstream api {
|
||||
least_conn;
|
||||
zone api 64k;
|
||||
include /etc/nginx/ready-upstreams.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
upstreams=/etc/nginx/ready-upstreams.conf
|
||||
printf 'server 127.0.0.1:9 down;\n' > "$upstreams"
|
||||
|
||||
check_ready() {
|
||||
state=/tmp/nginx-readiness.state
|
||||
probes=/tmp/nginx-readiness-probes
|
||||
mkdir -p "$probes"
|
||||
: > "$state"
|
||||
sleep 2
|
||||
while :; do
|
||||
next="${upstreams}.next"
|
||||
previous="${upstreams}.previous"
|
||||
: > "${state}.next"
|
||||
printf 'server 127.0.0.1:9 down;\n' > "$next"
|
||||
|
||||
addresses=$(getent ahostsv4 api | awk '{print $1}' | sort -u)
|
||||
for address in $addresses; do
|
||||
(
|
||||
if wget -q -T 1 -t 1 -O /dev/null "http://${address}:8080/health/ready"; then
|
||||
printf '1\n'
|
||||
else
|
||||
printf '0\n'
|
||||
fi > "$probes/$address"
|
||||
) &
|
||||
done
|
||||
wait
|
||||
|
||||
for address in $addresses; do
|
||||
record=$(awk -v address="$address" '$1 == address {print $2, $3, $4}' "$state")
|
||||
set -- ${record:-0 0 0}
|
||||
ready=$1
|
||||
successes=$2
|
||||
failures=$3
|
||||
if [ "$(cat "$probes/$address")" = 1 ]; then
|
||||
failures=0
|
||||
if [ "$successes" -lt 2 ]; then successes=$((successes + 1)); fi
|
||||
if [ "$successes" -ge 2 ]; then ready=1; fi
|
||||
else
|
||||
successes=0
|
||||
if [ "$failures" -lt 2 ]; then failures=$((failures + 1)); fi
|
||||
if [ "$failures" -ge 2 ]; then ready=0; fi
|
||||
fi
|
||||
printf '%s %s %s %s\n' "$address" "$ready" "$successes" "$failures" >> "${state}.next"
|
||||
if [ "$ready" -eq 1 ]; then
|
||||
printf 'server %s:8080 max_fails=2 fail_timeout=2s;\n' "$address" >> "$next"
|
||||
fi
|
||||
done
|
||||
mv "${state}.next" "$state"
|
||||
|
||||
if cmp -s "$next" "$upstreams"; then
|
||||
rm -f "$next"
|
||||
else
|
||||
cp "$upstreams" "$previous"
|
||||
mv "$next" "$upstreams"
|
||||
if nginx -t -q && nginx -s reload; then
|
||||
rm -f "$previous"
|
||||
else
|
||||
mv "$previous" "$upstreams"
|
||||
printf 'Could not reload Nginx readiness upstreams\n' >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
check_ready &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { Config } from 'jest';
|
||||
import { pathsToModuleNameMapper } from 'ts-jest';
|
||||
import ts from 'typescript';
|
||||
|
||||
const { config: tsconfig } = ts.readConfigFile(
|
||||
'./tsconfig.json',
|
||||
ts.sys.readFile,
|
||||
);
|
||||
const paths = tsconfig?.compilerOptions?.paths ?? {};
|
||||
|
||||
const config: Config = {
|
||||
moduleFileExtensions: ['js', 'json', 'ts'],
|
||||
rootDir: '.',
|
||||
testRegex: '.*\\.spec\\.ts$',
|
||||
transform: {
|
||||
'^.+\\.(t|j)s$': 'ts-jest',
|
||||
},
|
||||
moduleNameMapper: {
|
||||
...pathsToModuleNameMapper(paths, { prefix: '<rootDir>/' }),
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
collectCoverageFrom: [
|
||||
'src/**/*.(t|j)s',
|
||||
'libs/**/*.(t|j)s',
|
||||
'apps/**/*.(t|j)s',
|
||||
],
|
||||
coverageDirectory: './coverage',
|
||||
testEnvironment: 'node',
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/nest-cli",
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src",
|
||||
"compilerOptions": {
|
||||
"deleteOutDir": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"name": "mtusi2026lab1-fork",
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"author": "",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"scripts": {
|
||||
"db:generate": "prisma generate",
|
||||
"db:validate": "prisma validate",
|
||||
"db:migrate": "prisma migrate dev",
|
||||
"db:deploy": "prisma migrate deploy",
|
||||
"test:persistence": "pnpm build && node --test test/persistence.mjs",
|
||||
"test:cache": "pnpm build && node --test test/cache.mjs",
|
||||
"test:resilience": "pnpm build && node --test test/resilience.mjs",
|
||||
"test:idempotency": "pnpm build && node --test test/idempotency.mjs",
|
||||
"test:pagination": "pnpm build && node --test test/message-pagination.mjs",
|
||||
"prebuild": "pnpm db:generate",
|
||||
"prestart:dev": "pnpm db:generate",
|
||||
"prestart": "pnpm db:generate",
|
||||
"pretest": "pnpm db:generate",
|
||||
"build": "nest build",
|
||||
"deploy": "nest deploy",
|
||||
"format": "prettier --write \"src/**/*.ts\"",
|
||||
"start": "nest start",
|
||||
"start:dev": "nest start --watch",
|
||||
"start:debug": "nest start --debug --watch",
|
||||
"start:prod": "node dist/main",
|
||||
"lint": "oxlint --type-aware src/",
|
||||
"test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js",
|
||||
"test:watch": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --watch",
|
||||
"test:cov": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --coverage",
|
||||
"test:debug": "node --inspect-brk --experimental-vm-modules -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/static": "^10.1.4",
|
||||
"@nestjs/common": "^12.0.1",
|
||||
"@nestjs/core": "^12.0.1",
|
||||
"@nestjs/platform-fastify": "12.0.1",
|
||||
"@nestjs/swagger": "^12.0.1",
|
||||
"@prisma/adapter-pg": "^7.10.0",
|
||||
"@prisma/client": "^7.10.0",
|
||||
"@scalar/nestjs-api-reference": "^1.2.19",
|
||||
"dotenv": "^18.0.7",
|
||||
"fastify": "5.12.1",
|
||||
"ioredis": "^6.0.0",
|
||||
"nestjs-pino": "^5.2.0",
|
||||
"pg": "^8.23.1",
|
||||
"pino-http": "^11.0.0",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nestjs/cli": "^12.0.0",
|
||||
"@nestjs/mau": "^0.2.6",
|
||||
"@nestjs/schematics": "^12.0.0",
|
||||
"@nestjs/testing": "^12.0.1",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^24.0.0",
|
||||
"@types/pg": "^8.23.1",
|
||||
"@types/supertest": "^7.0.0",
|
||||
"jest": "^30.0.0",
|
||||
"oxlint": "^1.58.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"prettier": "^3.4.2",
|
||||
"prisma": "7.10.0",
|
||||
"source-map-support": "^0.5.21",
|
||||
"supertest": "^7.0.0",
|
||||
"ts-jest": "^29.4.0",
|
||||
"ts-loader": "^9.5.2",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsconfig-paths": "^4.2.0",
|
||||
"typescript": "^6.0.2"
|
||||
}
|
||||
}
|
||||
Generated
+7698
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
import 'dotenv/config';
|
||||
import { defineConfig } from 'prisma/config';
|
||||
|
||||
export default defineConfig({
|
||||
schema: 'prisma/schema.prisma',
|
||||
migrations: { path: 'prisma/migrations' },
|
||||
datasource: { url: process.env.DATABASE_URL },
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
CREATE SCHEMA IF NOT EXISTS "public";
|
||||
|
||||
CREATE TABLE "users" (
|
||||
"id" UUID NOT NULL,
|
||||
"username" VARCHAR(32) NOT NULL,
|
||||
"display_name" VARCHAR(100) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "users_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE TABLE "chats" (
|
||||
"id" UUID NOT NULL,
|
||||
"title" VARCHAR(100) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "chats_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE TABLE "chat_members" (
|
||||
"chat_id" UUID NOT NULL,
|
||||
"user_id" UUID NOT NULL,
|
||||
"position" INTEGER NOT NULL,
|
||||
|
||||
CONSTRAINT "chat_members_pkey" PRIMARY KEY ("chat_id","user_id")
|
||||
);
|
||||
|
||||
CREATE TABLE "messages" (
|
||||
"id" UUID NOT NULL,
|
||||
"chat_id" UUID NOT NULL,
|
||||
"sender_id" UUID NOT NULL,
|
||||
"text" VARCHAR(4000) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "messages_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "users_username_key" ON "users"("username");
|
||||
|
||||
CREATE INDEX "chats_created_at_id_idx" ON "chats"("created_at", "id");
|
||||
|
||||
CREATE INDEX "chat_members_user_id_chat_id_idx" ON "chat_members"("user_id", "chat_id");
|
||||
|
||||
CREATE UNIQUE INDEX "chat_members_chat_id_position_key" ON "chat_members"("chat_id", "position");
|
||||
|
||||
CREATE INDEX "messages_chat_id_created_at_id_idx" ON "messages"("chat_id", "created_at", "id");
|
||||
|
||||
CREATE INDEX "messages_sender_id_idx" ON "messages"("sender_id");
|
||||
|
||||
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "messages" ADD CONSTRAINT "messages_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "messages" ADD CONSTRAINT "messages_sender_id_fkey" FOREIGN KEY ("sender_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE "users" ADD COLUMN "password_hash" TEXT;
|
||||
|
||||
CREATE TABLE "sessions" (
|
||||
"id" UUID NOT NULL,
|
||||
"user_id" UUID NOT NULL,
|
||||
"token_hash" CHAR(64) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"expires_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
"revoked_at" TIMESTAMPTZ(3),
|
||||
|
||||
CONSTRAINT "sessions_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "sessions_token_hash_key" ON "sessions"("token_hash");
|
||||
|
||||
CREATE INDEX "sessions_user_id_idx" ON "sessions"("user_id");
|
||||
|
||||
CREATE INDEX "sessions_expires_at_idx" ON "sessions"("expires_at");
|
||||
|
||||
ALTER TABLE "sessions" ADD CONSTRAINT "sessions_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE "users" ADD COLUMN "chats_epoch" BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE "chats" ADD COLUMN "messages_epoch" BIGINT NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE "chat_message_epochs" (
|
||||
"chat_id" UUID NOT NULL,
|
||||
"bucket" SMALLINT NOT NULL,
|
||||
"epoch" BIGINT NOT NULL DEFAULT 0,
|
||||
CONSTRAINT "chat_message_epochs_pkey" PRIMARY KEY ("chat_id", "bucket"),
|
||||
CONSTRAINT "chat_message_epochs_bucket_check" CHECK ("bucket" BETWEEN 0 AND 15),
|
||||
CONSTRAINT "chat_message_epochs_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
INSERT INTO "chat_message_epochs" ("chat_id", "bucket", "epoch")
|
||||
SELECT "id", 0, "messages_epoch" FROM "chats";
|
||||
|
||||
ALTER TABLE "chats" DROP COLUMN "messages_epoch";
|
||||
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE "message_idempotency" (
|
||||
"user_id" UUID NOT NULL,
|
||||
"chat_id" UUID NOT NULL,
|
||||
"key" VARCHAR(128) NOT NULL,
|
||||
"message_id" UUID,
|
||||
"expires_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
CONSTRAINT "message_idempotency_pkey" PRIMARY KEY ("user_id", "chat_id", "key"),
|
||||
CONSTRAINT "message_idempotency_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
CONSTRAINT "message_idempotency_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
CONSTRAINT "message_idempotency_message_id_fkey" FOREIGN KEY ("message_id") REFERENCES "messages"("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX "message_idempotency_expires_at_idx" ON "message_idempotency"("expires_at");
|
||||
CREATE INDEX "message_idempotency_chat_id_idx" ON "message_idempotency"("chat_id");
|
||||
CREATE INDEX "message_idempotency_message_id_idx" ON "message_idempotency"("message_id");
|
||||
@@ -0,0 +1 @@
|
||||
provider = "postgresql"
|
||||
@@ -0,0 +1,106 @@
|
||||
generator client {
|
||||
provider = "prisma-client"
|
||||
output = "../src/generated/prisma"
|
||||
moduleFormat = "cjs"
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
}
|
||||
|
||||
model User {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
username String @unique @db.VarChar(32)
|
||||
displayName String @map("display_name") @db.VarChar(100)
|
||||
passwordHash String? @map("password_hash")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
chatsEpoch BigInt @default(0) @map("chats_epoch")
|
||||
memberships ChatMember[]
|
||||
messages Message[]
|
||||
sessions Session[]
|
||||
idempotencyRecords MessageIdempotency[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
|
||||
model Session {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
userId String @map("user_id") @db.Uuid
|
||||
tokenHash String @unique @map("token_hash") @db.Char(64)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
expiresAt DateTime @map("expires_at") @db.Timestamptz(3)
|
||||
revokedAt DateTime? @map("revoked_at") @db.Timestamptz(3)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([userId])
|
||||
@@index([expiresAt])
|
||||
@@map("sessions")
|
||||
}
|
||||
|
||||
model Chat {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
title String @db.VarChar(100)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
messageEpochs ChatMessageEpoch[]
|
||||
members ChatMember[]
|
||||
messages Message[]
|
||||
idempotencyRecords MessageIdempotency[]
|
||||
|
||||
@@index([createdAt, id])
|
||||
@@map("chats")
|
||||
}
|
||||
|
||||
model ChatMember {
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
userId String @map("user_id") @db.Uuid
|
||||
position Int
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Restrict)
|
||||
|
||||
@@id([chatId, userId])
|
||||
@@unique([chatId, position])
|
||||
@@index([userId, chatId])
|
||||
@@map("chat_members")
|
||||
}
|
||||
|
||||
model ChatMessageEpoch {
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
bucket Int @db.SmallInt
|
||||
epoch BigInt @default(0)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@id([chatId, bucket])
|
||||
@@map("chat_message_epochs")
|
||||
}
|
||||
|
||||
model Message {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
senderId String @map("sender_id") @db.Uuid
|
||||
text String @db.VarChar(4000)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
sender User @relation(fields: [senderId], references: [id], onDelete: Restrict)
|
||||
idempotencyRecords MessageIdempotency[]
|
||||
|
||||
@@index([chatId, createdAt, id])
|
||||
@@index([senderId])
|
||||
@@map("messages")
|
||||
}
|
||||
|
||||
model MessageIdempotency {
|
||||
userId String @map("user_id") @db.Uuid
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
key String @db.VarChar(128)
|
||||
messageId String? @map("message_id") @db.Uuid
|
||||
expiresAt DateTime @map("expires_at") @db.Timestamptz(3)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
message Message? @relation(fields: [messageId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@id([userId, chatId, key])
|
||||
@@index([chatId])
|
||||
@@index([messageId])
|
||||
@@index([expiresAt])
|
||||
@@map("message_idempotency")
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compatibility entry point for the consolidated report figures."""
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
if __name__ == '__main__':
|
||||
helper = Path(__file__).with_name('report-evidence.py')
|
||||
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
|
||||
runpy.run_path(str(helper), run_name='__main__')
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check saved source snapshots, run completion and actual container budgets."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('directory', type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.directory
|
||||
manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
|
||||
sources = {name: hashlib.sha256((root / 'sources' / name).read_bytes()).hexdigest() == digest
|
||||
for name, digest in manifest['source_sha256'].items()}
|
||||
budgets = {}
|
||||
images = set()
|
||||
for profile, containers in manifest['containers'].items():
|
||||
apis = [c for c in containers if '-api-' in c['name']]
|
||||
images.update(c['image'] for c in apis)
|
||||
budgets[profile] = {'replicas': len(apis), 'cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
|
||||
'ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
|
||||
'total_pool': sum(int(c['pool_max']) for c in apis)}
|
||||
runs = {}
|
||||
groups = Counter()
|
||||
for label in manifest['runs']:
|
||||
path = root / label
|
||||
summary = json.loads((path / 'summary.json').read_text(encoding='utf-8'))
|
||||
execution = json.loads((path / 'execution.json').read_text(encoding='utf-8'))
|
||||
profile = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
|
||||
expected = budgets[profile]['replicas']
|
||||
rate = summary['profile']['offered_rps']
|
||||
if summary['profile'].get('phase') == 'confirmation':
|
||||
groups[f'{profile}/r{rate}'] += 1
|
||||
runs[label] = {'returncode': execution['returncode'],
|
||||
'diagnostics_complete': summary.get('diagnostics_complete', False),
|
||||
'replicas_match': summary['profile']['expected_instances'] == expected,
|
||||
'criterion': summary.get('criterion')}
|
||||
requested = manifest.get('requested_confirmation_rates')
|
||||
complete_groups = all(count == manifest['repetitions'] for count in groups.values())
|
||||
if requested:
|
||||
complete_groups = complete_groups and all(
|
||||
groups[f'{profile}/r{rate}'] == manifest['repetitions']
|
||||
for profile in manifest['profiles'] for rate in requested)
|
||||
checks = {'finished': bool(manifest.get('finished')), 'source_snapshots_match': all(sources.values()),
|
||||
'one_api_image': len(images) == 1, 'all_runs_complete': bool(runs) and all(
|
||||
r['returncode'] == 0 and r['diagnostics_complete'] and r['replicas_match'] for r in runs.values()),
|
||||
'confirmation_groups_complete': bool(groups) and complete_groups,
|
||||
'pool_budget_matches': all(b['total_pool'] == manifest['pool_budget'] for b in budgets.values())}
|
||||
audit = {'checks_pass': all(checks.values()), 'checks': checks, 'budgets': budgets,
|
||||
'source_snapshots': sources, 'confirmation_groups': dict(groups), 'runs': runs}
|
||||
(root / 'audit.json').write_text(json.dumps(audit, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps({'checks_pass': audit['checks_pass'], 'checks': checks, 'budgets': budgets,
|
||||
'runs': len(runs)}, indent=2))
|
||||
if not audit['checks_pass']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Synthetic lab3 scaling benchmark. Runs against an already configured stack.
|
||||
* The client is outside the Docker resource budget; no automatic POST retries.
|
||||
*/
|
||||
|
||||
import { execFile as execFileCallback } from 'node:child_process';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
||||
import { promisify } from 'node:util';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function options(argv) {
|
||||
const defaults = {
|
||||
'base-url': 'http://127.0.0.1:8080',
|
||||
label: 'unlabelled',
|
||||
rate: '100',
|
||||
'duration-seconds': '30',
|
||||
'warmup-seconds': '5',
|
||||
'max-in-flight': '64',
|
||||
'read-percent': '80',
|
||||
'seed-messages': '100',
|
||||
'history-limit': '50',
|
||||
'timeout-ms': '5000',
|
||||
'docker-stats-interval-seconds': '2',
|
||||
};
|
||||
const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']);
|
||||
const values = { ...defaults };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const name = argv[i];
|
||||
if (name === '--help') {
|
||||
console.log(`Usage: node scripts/benchmark-scaling.mjs [options]
|
||||
--base-url URL Nginx URL (default: http://127.0.0.1:8080)
|
||||
--label NAME Configuration label for later comparisons
|
||||
--rate N Scheduled HTTP requests per second (default: 100)
|
||||
--duration-seconds N Measured phase (default: 30)
|
||||
--warmup-seconds N Read-only warmup (default: 5)
|
||||
--max-in-flight N Client-side outstanding request cap (default: 64)
|
||||
--read-percent N Percentage of history GETs (default: 80)
|
||||
--seed-messages N Initial messages in a fresh chat (default: 100)
|
||||
--history-limit N Messages per GET page (default: 50)
|
||||
--timeout-ms N Per-request deadline (default: 5000)
|
||||
--docker-stats Sample CPU/RAM of current Compose containers
|
||||
--docker-stats-interval-seconds N Sampling interval (default: 2)
|
||||
--expected-instances N Require N distinct X-Instance-Id values
|
||||
--output PATH Save JSON result; stdout always prints summary
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
if (!name.startsWith('--') || !allowed.has(name.slice(2))) {
|
||||
throw new Error(`unknown option: ${name}`);
|
||||
}
|
||||
const key = name.slice(2);
|
||||
if (key === 'docker-stats') {
|
||||
values[key] = true;
|
||||
continue;
|
||||
}
|
||||
const value = argv[++i];
|
||||
if (value === undefined || value.startsWith('--')) {
|
||||
throw new Error(`missing value for ${name}`);
|
||||
}
|
||||
values[key] = value;
|
||||
}
|
||||
const number = (key, min, max) => {
|
||||
const value = Number(values[key]);
|
||||
if (!Number.isFinite(value) || value < min || value > max) {
|
||||
throw new Error(`${key} must be ${min}..${max}`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const parsed = {
|
||||
baseUrl: values['base-url'].replace(/\/$/, ''),
|
||||
label: values.label,
|
||||
rate: number('rate', 1, 1000),
|
||||
durationSeconds: number('duration-seconds', 1, 3600),
|
||||
warmupSeconds: number('warmup-seconds', 0, 600),
|
||||
maxInFlight: number('max-in-flight', 1, 4096),
|
||||
readPercent: number('read-percent', 0, 100),
|
||||
seedMessages: number('seed-messages', 0, 10000),
|
||||
historyLimit: number('history-limit', 1, 100),
|
||||
timeoutMs: number('timeout-ms', 100, 120000),
|
||||
dockerStats: Boolean(values['docker-stats']),
|
||||
dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60),
|
||||
expectedInstances: values['expected-instances'] === undefined
|
||||
? null : Number(values['expected-instances']),
|
||||
output: values.output,
|
||||
};
|
||||
if (!/^https?:\/\//.test(parsed.baseUrl)) {
|
||||
throw new Error('base-url must begin with http:// or https://');
|
||||
}
|
||||
if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) {
|
||||
throw new Error('label must use letters, digits, dot, underscore or dash');
|
||||
}
|
||||
if (parsed.rate * parsed.durationSeconds > 200000) {
|
||||
throw new Error('at most 200000 measured request slots are allowed per run');
|
||||
}
|
||||
for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages',
|
||||
'historyLimit', 'timeoutMs']) {
|
||||
if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`);
|
||||
}
|
||||
if (parsed.expectedInstances !== null &&
|
||||
(!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 ||
|
||||
parsed.expectedInstances > 16)) {
|
||||
throw new Error('expected-instances must be an integer from 1 to 16');
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) {
|
||||
const headers = { Accept: 'application/json' };
|
||||
if (authorization) headers.Authorization = authorization;
|
||||
if (key) headers['Idempotency-Key'] = key;
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
const response = await fetch(baseUrl + path, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(timeoutMs ?? 10000),
|
||||
});
|
||||
const payload = await response.json().catch(() => null);
|
||||
return { status: response.status, payload, instance: response.headers.get('x-instance-id') };
|
||||
}
|
||||
|
||||
async function bootstrap(config) {
|
||||
const suffix = randomUUID().replaceAll('-', '').slice(0, 12);
|
||||
const password = `T3st-${randomUUID()}`;
|
||||
const users = [];
|
||||
for (const label of ['alice', 'bob']) {
|
||||
const username = `bench_${label}_${suffix}`;
|
||||
const created = await api(config.baseUrl, 'POST', '/api/v1/users', {
|
||||
body: { username, display_name: `Benchmark ${label}`, password },
|
||||
});
|
||||
if (created.status !== 201 || !created.payload?.id) {
|
||||
throw new Error(`setup: user registration returned ${created.status}`);
|
||||
}
|
||||
const basic = Buffer.from(`${username}:${password}`).toString('base64');
|
||||
const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', {
|
||||
authorization: `Basic ${basic}`,
|
||||
});
|
||||
if (session.status !== 201 || !session.payload?.token) {
|
||||
throw new Error(`setup: session creation returned ${session.status}`);
|
||||
}
|
||||
users.push({ id: created.payload.id, token: session.payload.token });
|
||||
}
|
||||
const authorization = `Bearer ${users[0].token}`;
|
||||
const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', {
|
||||
body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) },
|
||||
authorization,
|
||||
});
|
||||
if (chat.status !== 201 || !chat.payload?.id) {
|
||||
throw new Error(`setup: chat creation returned ${chat.status}`);
|
||||
}
|
||||
const path = `/api/v1/chats/${chat.payload.id}/messages`;
|
||||
for (let index = 0; index < config.seedMessages; index += 1) {
|
||||
const seeded = await api(config.baseUrl, 'POST', path, {
|
||||
body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(),
|
||||
});
|
||||
if (seeded.status !== 201 || !seeded.payload?.id) {
|
||||
throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`);
|
||||
}
|
||||
}
|
||||
return { authorization, path };
|
||||
}
|
||||
|
||||
function percentile(sorted, fraction) {
|
||||
if (!sorted.length) return null;
|
||||
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3));
|
||||
}
|
||||
|
||||
function summarize(events, durationSeconds, slots, dropped, late) {
|
||||
const groups = {};
|
||||
for (const kind of ['read', 'write', 'all']) {
|
||||
const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind);
|
||||
const successes = chosen.filter((event) => event.ok);
|
||||
const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b);
|
||||
const statuses = {};
|
||||
const errors = {};
|
||||
const instances = {};
|
||||
for (const event of chosen) {
|
||||
const status = event.status === null ? 'network' : String(event.status);
|
||||
statuses[status] = (statuses[status] ?? 0) + 1;
|
||||
if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1;
|
||||
if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1;
|
||||
}
|
||||
groups[kind] = {
|
||||
attempted: chosen.length,
|
||||
successful: successes.length,
|
||||
errors: chosen.length - successes.length,
|
||||
successful_rps: Number((successes.length / durationSeconds).toFixed(3)),
|
||||
status_counts: statuses,
|
||||
error_counts: errors,
|
||||
instance_counts: instances,
|
||||
successful_latency_ms: {
|
||||
p50: percentile(latencies, 0.5),
|
||||
p95: percentile(latencies, 0.95),
|
||||
p99: percentile(latencies, 0.99),
|
||||
},
|
||||
};
|
||||
}
|
||||
return { scheduled_slots: slots, client_dropped_slots: dropped,
|
||||
scheduler_late_slots: late, ...groups };
|
||||
}
|
||||
|
||||
async function runPhase(config, context, durationSeconds, readPercent) {
|
||||
const slots = Math.floor(config.rate * durationSeconds);
|
||||
const intervalMs = 1000 / config.rate;
|
||||
const started = performance.now();
|
||||
const startedEpochMs = Date.now();
|
||||
const events = [];
|
||||
const pending = new Set();
|
||||
let dropped = 0;
|
||||
let late = 0;
|
||||
for (let sequence = 0; sequence < slots; sequence += 1) {
|
||||
const scheduledAt = started + sequence * intervalMs;
|
||||
const waitMs = scheduledAt - performance.now();
|
||||
if (waitMs > 0) await sleep(waitMs);
|
||||
const actualAt = performance.now();
|
||||
if (actualAt - scheduledAt > intervalMs) {
|
||||
late += 1;
|
||||
dropped += 1;
|
||||
continue;
|
||||
}
|
||||
if (pending.size >= config.maxInFlight) {
|
||||
dropped += 1;
|
||||
continue;
|
||||
}
|
||||
const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write';
|
||||
const task = (async () => {
|
||||
let status = null;
|
||||
let instance = null;
|
||||
let error = null;
|
||||
try {
|
||||
const result = kind === 'read'
|
||||
? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, {
|
||||
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
||||
})
|
||||
: await api(config.baseUrl, 'POST', context.path, {
|
||||
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
||||
key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` },
|
||||
});
|
||||
status = result.status;
|
||||
instance = result.instance;
|
||||
if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`;
|
||||
else if (!instance) error = 'missing_instance_id';
|
||||
else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page';
|
||||
else if (kind === 'write' && !result.payload?.id) error = 'invalid_message';
|
||||
} catch (caught) {
|
||||
error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse';
|
||||
}
|
||||
events.push({ sequence, kind, ok: error === null, status, error, instance,
|
||||
scheduled_ms: Number((scheduledAt - started).toFixed(3)),
|
||||
start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)),
|
||||
latency_ms: Number((performance.now() - actualAt).toFixed(3)) });
|
||||
})();
|
||||
pending.add(task);
|
||||
void task.finally(() => pending.delete(task));
|
||||
}
|
||||
const scheduledEnd = started + durationSeconds * 1000;
|
||||
if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now());
|
||||
await Promise.all(pending);
|
||||
return { events, started_epoch_ms: startedEpochMs,
|
||||
scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000,
|
||||
completed_epoch_ms: Date.now(),
|
||||
summary: summarize(events, durationSeconds, slots, dropped, late),
|
||||
drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) };
|
||||
}
|
||||
|
||||
function bytesToMiB(value) {
|
||||
const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim());
|
||||
if (!match) return null;
|
||||
const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12,
|
||||
KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 };
|
||||
return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3));
|
||||
}
|
||||
|
||||
async function dockerSampler(intervalSeconds) {
|
||||
const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], {
|
||||
timeout: 15000, maxBuffer: 1024 * 1024,
|
||||
});
|
||||
const ids = stdout.trim().split(/\s+/).filter(Boolean);
|
||||
if (!ids.length) throw new Error('docker-stats: no running Compose containers');
|
||||
let stopped = false;
|
||||
const samples = [];
|
||||
const errors = [];
|
||||
const task = (async () => {
|
||||
while (!stopped) {
|
||||
const started = performance.now();
|
||||
try {
|
||||
const result = await execFile('docker',
|
||||
['stats', '--no-stream', '--format', '{{json .}}', ...ids],
|
||||
{ timeout: 15000, maxBuffer: 1024 * 1024 });
|
||||
for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) {
|
||||
const row = JSON.parse(line);
|
||||
samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name,
|
||||
cpu_percent: Number(row.CPUPerc.replace('%', '')),
|
||||
memory_mib: bytesToMiB(row.MemUsage.split('/')[0]),
|
||||
memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]),
|
||||
pids: Number(row.PIDs) });
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error?.code || error?.name || 'docker_stats_error');
|
||||
}
|
||||
const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started));
|
||||
if (!stopped && waitMs) await sleep(waitMs);
|
||||
}
|
||||
})();
|
||||
return { stop: async () => { stopped = true; await task; return { samples, errors }; } };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const config = options(process.argv.slice(2));
|
||||
const context = await bootstrap(config);
|
||||
const warmup = config.warmupSeconds
|
||||
? await runPhase(config, context, config.warmupSeconds, 100)
|
||||
: null;
|
||||
const sampler = config.dockerStats
|
||||
? await dockerSampler(config.dockerStatsIntervalSeconds)
|
||||
: null;
|
||||
const delay = monitorEventLoopDelay({ resolution: 20 });
|
||||
delay.enable();
|
||||
const cpuBefore = process.cpuUsage();
|
||||
const memoryBefore = process.memoryUsage().rss;
|
||||
let measured;
|
||||
let dockerStats = null;
|
||||
try {
|
||||
measured = await runPhase(config, context, config.durationSeconds, config.readPercent);
|
||||
} finally {
|
||||
if (sampler) dockerStats = await sampler.stop();
|
||||
delay.disable();
|
||||
}
|
||||
const cpu = process.cpuUsage(cpuBefore);
|
||||
const result = {
|
||||
label: config.label,
|
||||
base_url: config.baseUrl,
|
||||
profile: {
|
||||
arrival_model: 'scheduled open loop; late or over-capacity slots dropped',
|
||||
offered_rps: config.rate,
|
||||
duration_seconds: config.durationSeconds,
|
||||
warmup_seconds: config.warmupSeconds,
|
||||
warmup_profile: 'read only',
|
||||
read_percent: config.readPercent,
|
||||
write_percent: 100 - config.readPercent,
|
||||
initial_messages: config.seedMessages,
|
||||
history_limit: config.historyLimit,
|
||||
max_in_flight: config.maxInFlight,
|
||||
timeout_ms: config.timeoutMs,
|
||||
automatic_retries: false,
|
||||
expected_instances: config.expectedInstances,
|
||||
},
|
||||
warmup: warmup?.summary ?? null,
|
||||
measurement: measured.summary,
|
||||
drain_seconds: measured.drain_seconds,
|
||||
timing: {
|
||||
measurement_started_epoch_ms: measured.started_epoch_ms,
|
||||
measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms,
|
||||
measurement_completed_epoch_ms: measured.completed_epoch_ms,
|
||||
},
|
||||
client: {
|
||||
cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)),
|
||||
rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)),
|
||||
rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)),
|
||||
event_loop_delay_ms: {
|
||||
mean: Number((delay.mean / 1e6).toFixed(3)),
|
||||
p99: Number((delay.percentile(99) / 1e6).toFixed(3)),
|
||||
},
|
||||
},
|
||||
docker_stats: dockerStats,
|
||||
request_samples: measured.events,
|
||||
notes: [
|
||||
'Setup and read-only warmup are excluded from measured RPS.',
|
||||
'RPS uses the scheduled measurement window; requests may complete during drain.',
|
||||
'HTTP failures, timeouts and client-dropped slots are reported separately.',
|
||||
'One chat is used per run; writes increase its history during measurement.',
|
||||
'The Node client runs outside Docker CPU/RAM limits and can itself saturate.',
|
||||
'A timed-out POST might have committed; this benchmark never retries it.',
|
||||
],
|
||||
};
|
||||
const json = JSON.stringify(result, null, 2) + '\n';
|
||||
if (config.output) {
|
||||
await mkdir(dirname(config.output), { recursive: true });
|
||||
await writeFile(config.output, json, 'utf8');
|
||||
}
|
||||
console.log(JSON.stringify({ label: result.label, profile: result.profile,
|
||||
measurement: result.measurement, drain_seconds: result.drain_seconds,
|
||||
client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0,
|
||||
output: config.output ?? null }, null, 2));
|
||||
const observed = Object.keys(result.measurement.all.instance_counts).length;
|
||||
if (config.expectedInstances !== null && observed !== config.expectedInstances) {
|
||||
console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
if (!result.measurement.all.successful) process.exitCode = 1;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`benchmark failed: ${error?.message ?? String(error)}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure idle PostgreSQL CPU with/without the observer after a scaling sweep."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(["docker", *args], cwd=ROOT, text=True, encoding="utf-8", timeout=30)
|
||||
|
||||
|
||||
def snapshot(container):
|
||||
values = {k: int(v) for k, v in (line.split() for line in docker("exec", container, "cat", "/sys/fs/cgroup/cpu.stat").splitlines())}
|
||||
return {"time": time.time(), "cpu": values}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seconds", type=int, default=30)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
container = docker("compose", "ps", "-q", "postgres").strip()
|
||||
results = []
|
||||
for phase in ("without-before", "with-observer", "without-after"):
|
||||
child = None
|
||||
files = []
|
||||
stop = f"/tmp/scaling-calibration-{uuid.uuid4().hex}.stop"
|
||||
try:
|
||||
if phase == "with-observer":
|
||||
files = [(args.output / "postgres.jsonl").open("wb"), (args.output / "postgres.stderr").open("wb")]
|
||||
child = subprocess.Popen(["docker", "exec", "-i", container, "sh", "-s", "postgres", stop],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=files[0], stderr=files[1])
|
||||
child.stdin.write((ROOT / "scripts/diagnostics/observer.sh").read_bytes())
|
||||
child.stdin.close()
|
||||
time.sleep(3)
|
||||
before = snapshot(container)
|
||||
time.sleep(args.seconds)
|
||||
after = snapshot(container)
|
||||
elapsed = after["time"] - before["time"]
|
||||
row = {"phase": phase, "before": before, "after": after,
|
||||
"mean_cpu_cores": (after["cpu"]["usage_usec"]-before["cpu"]["usage_usec"])/elapsed/1e6}
|
||||
results.append(row)
|
||||
print(f"{phase}: CPU={row['mean_cpu_cores']:.4f} cores", flush=True)
|
||||
finally:
|
||||
if child is not None:
|
||||
docker("exec", container, "touch", stop)
|
||||
child.wait(timeout=15)
|
||||
docker("cp", f"{container}:{stop}.pg.jsonl", str(args.output / "postgres-sql.jsonl"))
|
||||
docker("exec", container, "rm", "-f", stop + ".pg.jsonl")
|
||||
for file in files:
|
||||
file.close()
|
||||
overhead = results[1]["mean_cpu_cores"] - (results[0]["mean_cpu_cores"] + results[2]["mean_cpu_cores"])/2
|
||||
report = {"phases": results, "estimated_observer_cpu_cores": overhead,
|
||||
"notes": ["Idle calibration with the running stack's normal readiness probes; no HTTP load generator.",
|
||||
"Difference includes host variation, monitor clients and monitoring SQL; not a correction factor for loaded runs."]}
|
||||
(args.output / "summary.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
print(f"Estimated observer CPU={overhead:.4f} cores", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab3 checks and retain one compact result without request logs or secrets."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def capture(command, timeout=180):
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout)
|
||||
|
||||
|
||||
def snapshot():
|
||||
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
|
||||
result.check_returncode()
|
||||
ids = result.stdout.split()
|
||||
inspected = capture(['docker', 'inspect', *ids], 30)
|
||||
inspected.check_returncode()
|
||||
rows = []
|
||||
for c in json.loads(inspected.stdout):
|
||||
service = c['Config']['Labels'].get('com.docker.compose.service')
|
||||
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
|
||||
continue
|
||||
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
|
||||
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
|
||||
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
|
||||
'health': c['State'].get('Health', {}).get('Status'),
|
||||
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
|
||||
'pool_max': pool, 'command': c['Config']['Cmd'],
|
||||
'ports': c['HostConfig'].get('PortBindings'),
|
||||
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
|
||||
for m in c['Mounts'] if m['Type'] == 'volume']})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
|
||||
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
|
||||
args = parser.parse_args()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if args.retry_failed:
|
||||
report = json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if report.get('source_sha256'):
|
||||
report['source_before_recheck_sha256'] = report['source_sha256']
|
||||
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
|
||||
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
|
||||
else:
|
||||
previous = (json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if args.output.exists() else None)
|
||||
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
|
||||
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
|
||||
if previous:
|
||||
report['previous_validation'] = {
|
||||
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
|
||||
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
|
||||
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
|
||||
for c in previous['checks']],
|
||||
}
|
||||
selected = None
|
||||
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||||
for name in ('src/main.ts', 'src/app.controller.ts',
|
||||
'src/common/http/fastify-adapter.ts',
|
||||
'infra/start-nginx.sh', 'scripts/stop-one.sh',
|
||||
'scripts/restart.sh', 'test/check_shutdown.py')}
|
||||
|
||||
def save():
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
def run(name, command, timeout=180, details=None, parse_json=False):
|
||||
if selected is not None and name not in selected and name != 'final-smoke':
|
||||
return None
|
||||
print(f'START {name}', flush=True)
|
||||
started = time.monotonic()
|
||||
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
|
||||
try:
|
||||
result = capture(command, timeout)
|
||||
item['exit_code'] = result.returncode
|
||||
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
|
||||
lines = (result.stdout + '\n' + result.stderr).splitlines()
|
||||
item['summary'] = [line for line in lines if line.startswith(
|
||||
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
|
||||
if details is not None and details.exists():
|
||||
item['details'] = json.loads(details.read_text(encoding='utf-8'))
|
||||
elif parse_json:
|
||||
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
|
||||
try:
|
||||
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
item.update(status='FAIL', error=type(exc).__name__)
|
||||
item['seconds'] = round(time.monotonic() - started, 3)
|
||||
previous = next((c for c in report['checks'] if c['name'] == name), None)
|
||||
if previous is not None:
|
||||
item['previous_attempts'] = previous.get('previous_attempts', []) + [
|
||||
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
|
||||
report['checks'][report['checks'].index(previous)] = item
|
||||
else:
|
||||
report['checks'].append(item)
|
||||
save()
|
||||
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
|
||||
for line in item.get('summary', [])[-3:]:
|
||||
print(line, flush=True)
|
||||
return item
|
||||
|
||||
def restore():
|
||||
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
|
||||
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
|
||||
result.check_returncode()
|
||||
time.sleep(6)
|
||||
|
||||
py = [sys.executable, '-X', 'utf8']
|
||||
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
|
||||
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
|
||||
save()
|
||||
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
|
||||
temp = Path(tmp)
|
||||
run('contract', py + ['scripts/check_contract.py'])
|
||||
run('lint', [pnpm, 'lint'])
|
||||
run('build', [pnpm, 'build'], 240)
|
||||
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
|
||||
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
|
||||
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
|
||||
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
|
||||
'-v', f'{ROOT / "infra"}:/source:ro',
|
||||
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
|
||||
'nginx:1.30.6-alpine', '/test.sh'], 60)
|
||||
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
|
||||
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
|
||||
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
|
||||
if selected is None or selected & {'persistence', 'public-failover'}:
|
||||
try:
|
||||
if selected is None or 'persistence' in selected:
|
||||
before = snapshot()
|
||||
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
|
||||
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
|
||||
after = snapshot()
|
||||
item['before'] = before
|
||||
item['after'] = after
|
||||
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
|
||||
same = {c['id']: c for c in after if c['service'] != 'migrate'}
|
||||
item['containers_restarted'] = restarted.keys() == same.keys() and all(
|
||||
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
|
||||
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
|
||||
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
|
||||
if not item['containers_restarted'] or not item['volumes_preserved']:
|
||||
item['status'] = 'FAIL'
|
||||
save()
|
||||
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
|
||||
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
|
||||
parse_json=True)
|
||||
finally:
|
||||
restore()
|
||||
for scenario in ('resources', 'failover', 'rolling'):
|
||||
result_file = temp / f'grade4-{scenario}.json'
|
||||
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
|
||||
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
|
||||
'--output', str(result_file)], 300, details=result_file)
|
||||
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
|
||||
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
|
||||
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
|
||||
if selected is None or 'migration-compatibility' in selected:
|
||||
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
|
||||
if available.returncode == 0:
|
||||
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
|
||||
else:
|
||||
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
|
||||
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
|
||||
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
report['final'] = snapshot()
|
||||
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
|
||||
report['finished'] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
|
||||
if not report['all_executed_checks_pass']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
# Диагностическая серия масштабирования
|
||||
|
||||
Из корня репозитория, при работающем Docker и собранном образе API:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/run-scaling-diagnostics.py --client container --output .artifacts/scaling/diagnostics-r100
|
||||
```
|
||||
|
||||
Значения по умолчанию: профили one/two/four, только 100 запланированных
|
||||
запросов/с, три повтора на профиль, 15 с прогрева и 60 с измерения, 80% чтений истории и 20%
|
||||
записей. Пул берётся из Compose: по умолчанию 15 на реплику для one/two,
|
||||
8 для four; фактические значения сохраняются в `manifest.json`.
|
||||
В каждом прогоне — новый чат и 30 исходных
|
||||
сообщений. Всего 9 прогонов: 11 минут 15 секунд прогрева и измерений плюс время запуска
|
||||
контейнеров и сбора данных (ориентировочно около 15 минут, не жёсткий тайм-аут). Меньшие нагрузки
|
||||
по умолчанию исключены; при необходимости их можно явно задать через `--rates`.
|
||||
|
||||
Нагрузка для профиля перестаёт повышаться, если минимум в двух из трёх повторов
|
||||
встречается один из признаков: больше 5% слотов пропущено из-за опоздания генератора,
|
||||
больше 5% пропущено из-за лимита одновременно выполняемых запросов, больше 1% ошибок
|
||||
среди отправленных запросов или p95 успешных ответов превышает 1000 мс. Это границы
|
||||
эксперимента, а не установленный SLA и не доказательство насыщения сервера.
|
||||
|
||||
Для полной матрицы всех запрошенных ступеней добавьте `--full-matrix`. Тогда
|
||||
превышения порогов записываются в `threshold_crossings` и не прерывают профиль.
|
||||
Точки с большими пропусками генератора не доказывают максимальную ёмкость API.
|
||||
Ошибки запуска, отсутствие результата или неполные диагностические данные всё
|
||||
ещё останавливают серию с восстановлением обычного профиля two.
|
||||
|
||||
Сбор инструментированного пула включается только через `test/compose.diagnostics.yaml`.
|
||||
Preload сохраняет callback/Promise-интерфейсы pg.Pool и раз в секунду пишет
|
||||
`/tmp/scaling-pool.jsonl` внутри API. Очередь и занятые соединения наблюдаются каждые
|
||||
100 мс; краткие очереди могут не попасть в выборку, поэтому дополнительно измеряется
|
||||
время каждого получения соединения. Оно включает создание новых соединений и
|
||||
задержки event loop, а не только ожидание очереди. Cчётчики приобретений накопительные,
|
||||
max-показатели относятся к интервалу записи.
|
||||
|
||||
PostgreSQL, Nginx и Redis наблюдаются через один постоянный docker exec на сервис
|
||||
с паузой 2 с между выборками. Наблюдатель версии 3 выполняет SQL через одно
|
||||
постоянное psql-соединение (`\watch 2`); результат находится в `postgres-sql.jsonl`.
|
||||
Сохраняются cgroup CPU/memory, ожидания активных
|
||||
запросов PostgreSQL, число простаивающих транзакций, состав upstream Nginx и события
|
||||
reload. Тексты SQL, токены, пароли и сообщения не записываются. Наблюдатели потребляют
|
||||
часть ресурсов тех же контейнеров; это одинаковая методика для всех профилей.
|
||||
|
||||
В каталоге прогона:
|
||||
|
||||
- `benchmark.json`: исходные HTTP-замеры и границы измеряемого окна в epoch ms;
|
||||
- `api-*.jsonl`: пул, event loop и cgroup API (включают также прогрев и предыдущие
|
||||
повторы того же профиля; summary выбирает только текущее измеряемое окно);
|
||||
- `postgres/nginx/redis.jsonl`: наблюдения остальных сервисов;
|
||||
- `postgres-sql.jsonl`: SQL-снимки постоянного psql-наблюдателя версии 3;
|
||||
- `*.stderr`, `nginx-events.log`: ошибки наблюдателей и события Nginx;
|
||||
- `summary.json`: показатели в измеряемом окне; скорости CPU рассчитаны из
|
||||
приращений счётчиков, память — максимум по выборкам;
|
||||
- `execution.json`: команда и временные границы прогона.
|
||||
|
||||
`manifest.json` в корне фиксирует настройки, причины остановки нагрузки, ID образов,
|
||||
хеши сборщиков, окружение Docker и лимиты контейнеров. После завершения или исключения runner восстанавливает профиль
|
||||
two без диагностического preload. При принудительном завершении процесса это можно
|
||||
сделать вручную:
|
||||
|
||||
```powershell
|
||||
docker compose -f compose.yaml -f configs/two.yaml up -d --no-build --no-deps --wait api
|
||||
```
|
||||
|
||||
Проверки сборщика:
|
||||
|
||||
```powershell
|
||||
node --test test/pool-diagnostics.mjs test/benchmark-scaling.mjs
|
||||
python -X utf8 test/diagnostic-summary.py
|
||||
```
|
||||
|
||||
Для отдельной проверки влияния Windows-клиента и публикации порта можно запустить
|
||||
тот же генератор в Linux-контейнере:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/run-scaling-diagnostics.py --client container --rates 100 --repetitions 3 --output .artifacts/scaling/diagnostics-linux-client-r100
|
||||
```
|
||||
|
||||
Клиент использует тот же собранный образ (только Node и benchmark-скрипт), не имеет
|
||||
лимита CPU/RAM и обращается к `http://nginx` по сети Compose. Это другой путь HTTP:
|
||||
без Windows port forwarding, но с конкуренцией за ресурсы той же Docker VM. Такой
|
||||
контроль помогает проверить влияние клиентского пути, но не заменяет внешний
|
||||
генератор на отдельной машине. Не запускайте его одновременно с основной серией.
|
||||
|
||||
После нагрузки можно оценить накладные расходы наблюдателя PostgreSQL на простое:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/calibrate-scaling-observer.py --output .artifacts/scaling/observer-calibration
|
||||
```
|
||||
|
||||
Выполняются три фазы по 30 с: без наблюдателя, с наблюдателем и снова без него.
|
||||
Это оценка собственных расходов сборщика, а не поправочный коэффициент к нагрузке.
|
||||
|
||||
После завершения серии соберите таблицу и графики. Matplotlib можно запустить
|
||||
через временное окружение uv, не меняя зависимости API:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/summarize-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
|
||||
uv run --no-project --with "matplotlib>=3.9,<4" python scripts/plot-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
|
||||
```
|
||||
|
||||
Графики `performance`, `resources`, `delivered-load` сохраняются в PNG и SVG.
|
||||
Линии показывают медианы повторов, затенение — диапазон min/max, а не доверительный
|
||||
интервал. Отдельный график задержек и ошибок по фактически отправленным RPS помогает
|
||||
учесть пропуски генератора. Превышение номинальной нагрузки не означает, что сервер
|
||||
фактически получил столько запросов.
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/bin/sh
|
||||
# One persistent docker exec per service; stdout contains only numeric diagnostics.
|
||||
set -eu
|
||||
role=$1
|
||||
stop=$2
|
||||
watch_pid=''
|
||||
cleanup() {
|
||||
if [ -n "$watch_pid" ]; then
|
||||
kill "$watch_pid" 2>/dev/null || true
|
||||
wait "$watch_pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 0' TERM INT
|
||||
printf '{"kind":"observer","version":3,"time_ms":%s000}\n' "$(date +%s)"
|
||||
if [ "$role" = postgres ]; then
|
||||
PGAPPNAME=scaling-observer psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" > "${stop}.pg.jsonl" <<'SQL' &
|
||||
SELECT json_build_object(
|
||||
'kind', 'postgres', 'time_ms', (extract(epoch FROM clock_timestamp())*1000)::bigint,
|
||||
'activity', (SELECT coalesce(json_agg(a), '[]'::json) FROM (
|
||||
SELECT state, wait_event_type, wait_event, count(*) AS count,
|
||||
max(extract(epoch FROM clock_timestamp()-query_start)) FILTER (WHERE state = 'active') AS max_active_seconds
|
||||
FROM pg_stat_activity WHERE datname=current_database() AND backend_type='client backend'
|
||||
AND application_name <> 'scaling-observer'
|
||||
GROUP BY state,wait_event_type,wait_event
|
||||
) a),
|
||||
'db', (SELECT row_to_json(d) FROM (
|
||||
SELECT xact_commit,xact_rollback,blks_read,blks_hit,temp_files,temp_bytes,deadlocks,stats_reset
|
||||
FROM pg_stat_database WHERE datname=current_database()
|
||||
) d)
|
||||
)::jsonb
|
||||
\watch 2
|
||||
SQL
|
||||
watch_pid=$!
|
||||
fi
|
||||
while [ ! -f "$stop" ]; do
|
||||
stamp="$(date +%s)000"
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"cgroup\",\"time_ms\":%s", stamp}
|
||||
FILENAME ~ /cpu.stat$/ {printf ",\"cpu_%s\":%s", $1, $2}
|
||||
FILENAME ~ /memory.events$/ {printf ",\"memory_%s\":%s", $1, $2}
|
||||
FILENAME ~ /memory.current$/ {printf ",\"memory_bytes\":%s", $1}
|
||||
END {print "}"}
|
||||
' /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.events /sys/fs/cgroup/memory.current
|
||||
if [ "$role" = postgres ]; then
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"host\",\"time_ms\":%s", stamp}
|
||||
FILENAME ~ /pressure\// {
|
||||
name=FILENAME; sub(/^.*\//,"",name)
|
||||
for (i=2;i<=NF;i++) {
|
||||
split($i,value,"=")
|
||||
if (value[1] == "total") printf ",\"%s_%s_total_usec\":%s",name,$1,value[2]
|
||||
}
|
||||
}
|
||||
FILENAME ~ /meminfo$/ && ($1 == "MemAvailable:" || $1 == "SwapFree:" || $1 == "SwapTotal:") {
|
||||
name=$1; sub(/:$/,"",name); printf ",\"%s_kib\":%s",name,$2
|
||||
}
|
||||
END {print "}"}
|
||||
' /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /proc/meminfo
|
||||
fi
|
||||
if [ "$role" = nginx ]; then
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"upstream\",\"time_ms\":%s,\"servers\":[", stamp; sep=""}
|
||||
/^server .*:8080 / {printf "%s\"%s\"", sep, $2; sep=","}
|
||||
END {print "]}"}
|
||||
' /etc/nginx/ready-upstreams.conf
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
rm -f "$stop"
|
||||
@@ -0,0 +1,73 @@
|
||||
// Loaded only by test/compose.diagnostics.yaml. No HTTP endpoint or credentials.
|
||||
const { Pool } = require('pg');
|
||||
const fs = require('node:fs');
|
||||
const { performance, monitorEventLoopDelay } = require('node:perf_hooks');
|
||||
|
||||
const states = new Map();
|
||||
const originalConnect = Pool.prototype.connect;
|
||||
Pool.prototype.connect = function (...args) {
|
||||
let state = states.get(this);
|
||||
if (!state) {
|
||||
state = { acquisitions: 0, errors: 0, acquire_ms: 0, interval_max_ms: 0,
|
||||
interval_max_waiting: 0, interval_max_busy: 0 };
|
||||
states.set(this, state);
|
||||
}
|
||||
const started = performance.now();
|
||||
const finish = (error) => {
|
||||
const elapsed = performance.now() - started;
|
||||
state.acquisitions += 1;
|
||||
state.errors += Number(Boolean(error));
|
||||
state.acquire_ms += elapsed;
|
||||
state.interval_max_ms = Math.max(state.interval_max_ms, elapsed);
|
||||
};
|
||||
if (typeof args[0] === 'function') {
|
||||
const callback = args[0];
|
||||
args[0] = (error, client, release) => {
|
||||
finish(error);
|
||||
callback(error, client, release);
|
||||
};
|
||||
return originalConnect.apply(this, args);
|
||||
}
|
||||
return originalConnect.apply(this, args).then(
|
||||
(client) => { finish(null); return client; },
|
||||
(error) => { finish(error); throw error; },
|
||||
);
|
||||
};
|
||||
|
||||
const output = '/tmp/scaling-pool.jsonl';
|
||||
fs.writeFileSync(output, '');
|
||||
const delay = monitorEventLoopDelay({ resolution: 20 });
|
||||
delay.enable();
|
||||
function counters(path) {
|
||||
try {
|
||||
return Object.fromEntries(fs.readFileSync(path, 'utf8').trim().split('\n')
|
||||
.map((line) => { const [key, value] = line.split(/\s+/); return [key, Number(value)]; }));
|
||||
} catch { return null; }
|
||||
}
|
||||
let lastWrite = performance.now();
|
||||
const timer = setInterval(() => {
|
||||
for (const [pool, state] of states) {
|
||||
state.interval_max_waiting = Math.max(state.interval_max_waiting, pool.waitingCount);
|
||||
state.interval_max_busy = Math.max(state.interval_max_busy, pool.totalCount - pool.idleCount);
|
||||
}
|
||||
if (performance.now() - lastWrite < 1000) return;
|
||||
lastWrite = performance.now();
|
||||
const pools = [...states].map(([pool, state]) => ({
|
||||
...state, total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount,
|
||||
}));
|
||||
let memory = null;
|
||||
try { memory = Number(fs.readFileSync('/sys/fs/cgroup/memory.current', 'utf8')); } catch {}
|
||||
fs.appendFileSync(output, JSON.stringify({ time_ms: Date.now(), pid: process.pid,
|
||||
instance: process.env.INSTANCE_ID, pools, cpu: counters('/sys/fs/cgroup/cpu.stat'),
|
||||
memory_bytes: memory, memory_events: counters('/sys/fs/cgroup/memory.events'),
|
||||
rss_bytes: process.memoryUsage().rss, process_cpu: process.cpuUsage(),
|
||||
event_loop_delay_ms: { mean: delay.mean / 1e6, p99: delay.percentile(99) / 1e6 },
|
||||
}) + '\n');
|
||||
delay.reset();
|
||||
for (const state of states.values()) {
|
||||
state.interval_max_ms = 0;
|
||||
state.interval_max_waiting = 0;
|
||||
state.interval_max_busy = 0;
|
||||
}
|
||||
}, 100);
|
||||
timer.unref();
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One read-only snapshot of SQL classes, never SQL text or parameters."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
SQL = """
|
||||
SELECT jsonb_build_object('time_ms',(extract(epoch FROM clock_timestamp())*1000)::bigint,
|
||||
'activity', (SELECT coalesce(jsonb_agg(a),'[]'::jsonb) FROM (
|
||||
SELECT state,wait_event_type,wait_event,
|
||||
CASE
|
||||
WHEN query LIKE '%INSERT INTO "public"."chat_message_epochs"%' THEN 'message_and_epoch_write'
|
||||
WHEN query LIKE '%message_idempotency%' THEN 'idempotency'
|
||||
WHEN lower(ltrim(query)) LIKE 'commit%' THEN 'commit'
|
||||
WHEN query LIKE '%token_hash%' THEN 'authentication'
|
||||
WHEN query LIKE '%chat_message_epochs%' THEN 'epoch_read'
|
||||
WHEN query LIKE '%messages%' THEN 'message_read'
|
||||
ELSE 'other'
|
||||
END AS operation,
|
||||
count(*) AS count,
|
||||
max(extract(epoch FROM clock_timestamp()-query_start)) AS max_query_seconds,
|
||||
max(cardinality(pg_blocking_pids(pid))) AS max_blockers
|
||||
FROM pg_stat_activity
|
||||
WHERE datname=current_database() AND backend_type='client backend'
|
||||
AND application_name <> 'scaling-observer' AND state='active'
|
||||
GROUP BY state,wait_event_type,wait_event,operation
|
||||
) a));
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "-e", "PGAPPNAME=scaling-observer",
|
||||
"postgres", "psql", "-X", "-qAt", "-U", "app", "-d", "chat", "-c", SQL],
|
||||
capture_output=True, text=True, encoding="utf-8", timeout=30, check=True)
|
||||
data = json.loads(result.stdout)
|
||||
args.output.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(data))
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot completed diagnostic runs using matplotlib; reads results.csv and manifest.json."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--phase", help="Only plot this CSV phase, e.g. confirmation")
|
||||
parser.add_argument("--all-latency", action="store_true", help="Use all-request p95, including failures")
|
||||
parser.add_argument("--valid-delivery-only", action="store_true", help="Exclude runs with unverified arrival rate")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
with (args.directory / "results.csv").open(encoding="utf-8") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
if args.phase:
|
||||
rows = [row for row in rows if row.get("phase") == args.phase]
|
||||
if args.valid_delivery_only:
|
||||
rows = [row for row in rows if row.get("delivery_valid") in ("True", "true", "1")]
|
||||
if not rows:
|
||||
parser.error("no results")
|
||||
for row in rows:
|
||||
row['configuration'] = row.get('configuration') or next(
|
||||
(profile for profile in manifest.get('profiles', []) if row['label'].startswith(profile + '-r')),
|
||||
str(row['replicas']))
|
||||
for key in ("replicas", "offered_rps"):
|
||||
row[key] = int(row[key])
|
||||
for key in ("successful_rps", "attempted_rps", "p95_ms", "drop_percent", "acquisition_mean_ms",
|
||||
"pool_max_waiting", "api_cpu_cores", "api_max_throttle_percent",
|
||||
"pg_cpu_cores", "pg_throttle_percent"):
|
||||
row[key] = float(row[key] or 'nan')
|
||||
if args.all_latency:
|
||||
if not row.get("all_request_p95_ms"):
|
||||
parser.error("all-request latency is missing from this dataset")
|
||||
row["p95_ms"] = float(row["all_request_p95_ms"])
|
||||
summary = json.loads((args.directory / row["label"] / "summary.json").read_text(encoding="utf-8"))
|
||||
m = summary["measurement"]["all"]
|
||||
row["error_percent"] = 100*m["errors"]/max(m["attempted"], 1)
|
||||
pg = next(c for group in manifest["containers"].values() for c in group if "-postgres-" in c["name"])
|
||||
pg_cpus = pg["nano_cpus"]/1e9
|
||||
memory_mib = pg["memory_bytes"]/1024**2
|
||||
configurations = sorted({r['configuration'] for r in rows})
|
||||
palette = ['#2563eb', '#059669', '#d97706', '#9333ea', '#db2777', '#0891b2', '#64748b']
|
||||
colors = {configuration: palette[i % len(palette)] for i, configuration in enumerate(configurations)}
|
||||
budgets = {}
|
||||
labels = {}
|
||||
for configuration in configurations:
|
||||
apis = [c for c in manifest.get('containers', {}).get(configuration, []) if '-api-' in c['name']]
|
||||
replicas = next(r['replicas'] for r in rows if r['configuration'] == configuration)
|
||||
if apis:
|
||||
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
|
||||
memory = sum(c['memory_bytes'] for c in apis) / 1024**3
|
||||
budgets[configuration] = (cpu, memory)
|
||||
labels[configuration] = f'{replicas} API ({cpu:g} CPU / {memory:g} GiB)'
|
||||
else:
|
||||
labels[configuration] = f'{configuration}: {replicas} API'
|
||||
budget_description = '; '.join(f'{cpu:g} CPU / {memory:g} GiB'
|
||||
for cpu, memory in sorted(set(budgets.values()))) or 'see manifest'
|
||||
|
||||
def draw(ax, field, title, ylabel, log=False):
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
samples = [[r[field] for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
values = [statistics.median(v) for v in samples]
|
||||
ax.plot(rates, values, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
ax.fill_between(rates, [min(v) for v in samples], [max(v) for v in samples],
|
||||
color=colors[configuration], alpha=0.12)
|
||||
ax.set(title=title, xlabel="Scheduled requests/s", ylabel=ylabel)
|
||||
ax.grid(alpha=0.25)
|
||||
if log:
|
||||
ax.set_yscale("log")
|
||||
else:
|
||||
ax.set_ylim(bottom=0)
|
||||
|
||||
def save(fig, name, caption):
|
||||
if args.valid_delivery_only:
|
||||
caption += "\nUnderfed runs excluded; sample counts can differ between points."
|
||||
fig.text(0.5, 0.015, caption, ha="center", fontsize=9)
|
||||
fig.tight_layout(rect=(0, 0.055, 1, 0.94))
|
||||
for extension in ("png", "svg"):
|
||||
fig.savefig(args.directory / f"{name}.{extension}", dpi=160, bbox_inches="tight", pad_inches=0.25)
|
||||
plt.close(fig)
|
||||
|
||||
plt.rcParams.update({"font.size": 10, "axes.titlesize": 11})
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
fig.suptitle(f"PostgreSQL {pg_cpus:g} CPU / {memory_mib:g} MiB; API budgets: {budget_description}", fontsize=12)
|
||||
draw(axes[0, 0], "successful_rps", "Successful throughput", "Successful requests/s")
|
||||
rates = sorted({r["offered_rps"] for r in rows})
|
||||
axes[0, 0].plot(rates, rates, color="#475569", linestyle="--", alpha=0.7, label="Scheduled")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], max(rates)*1.05))
|
||||
axes[0, 0].legend()
|
||||
draw(axes[0, 1], "p95_ms", "Latency of all requests" if args.all_latency else "Latency of successful requests", "p95, ms (log scale)", log=True)
|
||||
if args.all_latency:
|
||||
axes[0, 1].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[1, 0], "error_percent", "Errors among attempted requests", "Errors, %")
|
||||
axes[1, 0].axhline(1, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 0].set_ylim(0, max(axes[1, 0].get_ylim()[1], 1.05))
|
||||
draw(axes[1, 1], "drop_percent", "Slots dropped by client", "Dropped scheduled slots, %")
|
||||
drop_limit = 2 if args.valid_delivery_only else 5
|
||||
axes[1, 1].axhline(drop_limit, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 1].set_ylim(0, max(axes[1, 1].get_ylim()[1], drop_limit*1.05))
|
||||
save(fig, "performance", "Lines: medians of repeats; shading: min-max. High client-drop points do not establish server capacity.")
|
||||
|
||||
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
|
||||
fig.suptitle("Resource and connection-pool diagnostics", fontsize=14)
|
||||
draw(axes[0, 0], "pg_cpu_cores", "PostgreSQL CPU usage", "CPU cores")
|
||||
axes[0, 0].axhline(pg_cpus, color="#64748b", linestyle="--", linewidth=1, label="CPU limit")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], pg_cpus*1.05))
|
||||
draw(axes[0, 1], "api_cpu_cores", "Total API CPU usage", "CPU cores")
|
||||
for configuration, (cpu, _) in budgets.items():
|
||||
axes[0, 1].axhline(cpu, color=colors[configuration], linestyle='--', linewidth=1,
|
||||
label=f'{configuration} limit: {cpu:g} CPU')
|
||||
if budgets:
|
||||
axes[0, 1].set_ylim(0, max(axes[0, 1].get_ylim()[1], max(cpu for cpu, _ in budgets.values()) * 1.05))
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[0, 2], "pg_throttle_percent", "PostgreSQL CPU throttling", "Throttled periods, %")
|
||||
draw(axes[1, 0], "api_max_throttle_percent", "Most throttled API replica", "Throttled periods, %")
|
||||
draw(axes[1, 1], "acquisition_mean_ms", "Mean connection acquisition", "Acquisition, ms (log scale)", log=True)
|
||||
draw(axes[1, 2], "pool_max_waiting", "Largest observed pool queue", "Waiting requests")
|
||||
axes[0, 0].legend()
|
||||
save(fig, "resources", "Lines: medians of repeats; shading: min-max. Queue sampled every 100 ms; acquisition includes connection setup.")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
|
||||
fig.suptitle("Latency and errors against the workload actually delivered", fontsize=14)
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
groups = [[r for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
x = [statistics.median(r["attempted_rps"] for r in group) for group in groups]
|
||||
for axis, field, title, ylabel in (
|
||||
(axes[0], "p95_ms", "Latency", "p95, ms (log scale)"),
|
||||
(axes[1], "error_percent", "Errors", "Errors among attempts, %"),
|
||||
):
|
||||
y = [statistics.median(r[field] for r in group) for group in groups]
|
||||
axis.plot(x, y, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
for rate, xp, yp in zip(rates, x, y):
|
||||
axis.annotate(str(rate), (xp, yp), xytext=(4, 4), textcoords="offset points", fontsize=8)
|
||||
axis.set(title=title, xlabel="Actually attempted requests/s", ylabel=ylabel)
|
||||
axis.grid(alpha=0.25)
|
||||
axes[0].set_yscale("log")
|
||||
if args.all_latency:
|
||||
axes[0].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[1].set_ylim(bottom=0)
|
||||
axes[0].legend()
|
||||
save(fig, "delivered-load", "Points: medians of repeats. Small labels: scheduled RPS. Client shares the Docker VM; no server-capacity claim.")
|
||||
print(f"Wrote performance/resources/delivered-load PNG and SVG for {len(rows)} runs to {args.directory}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compatibility entry point for the consolidated report figures."""
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
if __name__ == '__main__':
|
||||
helper = Path(__file__).with_name('report-evidence.py')
|
||||
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
|
||||
runpy.run_path(str(helper), run_name='__main__')
|
||||
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep report evidence in one JSON, one CSV, two figures and checks.json."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERIES = [
|
||||
('api-cpu', None, 'api-small-growth-20261029'),
|
||||
('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'),
|
||||
('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'),
|
||||
('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'),
|
||||
('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'),
|
||||
('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'),
|
||||
('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'),
|
||||
]
|
||||
|
||||
|
||||
def digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def flatten(bundle):
|
||||
rows = []
|
||||
for run in bundle['runs']:
|
||||
profile, measurement = run['profile'], run['measurement']
|
||||
all_requests = measurement['all']
|
||||
diagnostics = run['diagnostics']
|
||||
resources, pools = diagnostics['resources'], diagnostics['pools']
|
||||
manifest = bundle['manifests'][run['manifest']]
|
||||
containers = manifest['containers'][run['configuration']]
|
||||
apis = [c for c in containers if '-api-' in c['name']]
|
||||
pg = next(c for c in containers if '-postgres-' in c['name'])
|
||||
seconds, rate = profile['duration_seconds'], profile['offered_rps']
|
||||
acquired = sum(p['acquisitions'] for p in pools.values())
|
||||
fraction = all_requests['attempted'] / (seconds * rate)
|
||||
latency = all_requests['successful_latency_ms']
|
||||
all_latency = run.get('all_latency_ms') or {}
|
||||
rows.append({
|
||||
'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'],
|
||||
'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
|
||||
'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
|
||||
'pool_total': sum(int(c['pool_max']) for c in apis),
|
||||
'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2,
|
||||
'offered_rps': rate, 'duration_seconds': seconds,
|
||||
'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']),
|
||||
'attempted': all_requests['attempted'], 'successful': all_requests['successful'],
|
||||
'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds,
|
||||
'attempted_rps': all_requests['attempted'] / seconds,
|
||||
'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02,
|
||||
'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'],
|
||||
'all_request_p95_ms': all_latency.get('p95'),
|
||||
'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')),
|
||||
'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')),
|
||||
'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100,
|
||||
'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'),
|
||||
'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'),
|
||||
'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100,
|
||||
'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100,
|
||||
'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1),
|
||||
'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()),
|
||||
'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0),
|
||||
'nginx_min_ready': diagnostics['nginx']['min_ready'],
|
||||
'diagnostics_complete': diagnostics['diagnostics_complete'],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def collect(source, output):
|
||||
manifests, runs = {}, []
|
||||
for dataset, variant, directory in SERIES:
|
||||
path = source / directory
|
||||
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
|
||||
manifests[directory] = manifest
|
||||
for label in manifest['runs']:
|
||||
location = path / label
|
||||
benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8'))
|
||||
summary = json.loads((location / 'summary.json').read_text(encoding='utf-8'))
|
||||
configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
|
||||
runs.append({
|
||||
'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration,
|
||||
'manifest': directory, 'label': label,
|
||||
'profile': benchmark['profile'], 'measurement': benchmark['measurement'],
|
||||
'timing': benchmark.get('timing'), 'client': benchmark.get('client'),
|
||||
'all_latency_ms': benchmark.get('all_latency_ms'),
|
||||
'criterion': summary.get('criterion'),
|
||||
'diagnostics': {key: summary[key] for key in
|
||||
('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')},
|
||||
'source_hashes': {'benchmark': digest(location / 'benchmark.json'),
|
||||
'summary': digest(location / 'summary.json')},
|
||||
})
|
||||
sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts]
|
||||
+ list((ROOT / 'prisma').rglob('*.sql'))
|
||||
+ list((ROOT / 'configs').glob('*.yaml'))
|
||||
+ [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')])
|
||||
host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json'
|
||||
host = json.loads(host_path.read_text(encoding='utf-8'))
|
||||
bundle = {
|
||||
'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(),
|
||||
'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
|
||||
'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.',
|
||||
'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.',
|
||||
'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']},
|
||||
'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources},
|
||||
'manifests': manifests, 'runs': runs,
|
||||
}
|
||||
(output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return bundle
|
||||
|
||||
|
||||
def verify(bundle, rows):
|
||||
counts = Counter(r['dataset'] for r in rows)
|
||||
assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts
|
||||
assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54
|
||||
assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows)
|
||||
assert all(r['diagnostics_complete'] for r in rows)
|
||||
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
||||
assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix)
|
||||
assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values())
|
||||
for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}),
|
||||
('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]:
|
||||
assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name)
|
||||
assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60}
|
||||
assert len(bundle['manifests']) == 7
|
||||
|
||||
|
||||
def plot(rows, output):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
for cpu, color in ((1, '#9333ea'), (2, '#059669')):
|
||||
for ax, rate, field, title, ylabel in (
|
||||
(axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'),
|
||||
(axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'),
|
||||
(axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'),
|
||||
(axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'),
|
||||
):
|
||||
groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)]
|
||||
scale = 100 if field == 'delivery_fraction' else 1
|
||||
values = [statistics.median(r[field] for r in g) * scale for g in groups]
|
||||
ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB')
|
||||
ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups],
|
||||
[max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1)
|
||||
ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
||||
ax.grid(alpha=.25)
|
||||
ax.legend(fontsize=8)
|
||||
axes[0, 0].axhline(1000, color='#64748b', linestyle='--')
|
||||
axes[1, 1].axhline(98, color='#64748b', linestyle='--')
|
||||
fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13)
|
||||
fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8)
|
||||
fig.tight_layout(rect=(0, .04, 1, .95))
|
||||
fig.savefig(output / 'scaling.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
panels = [
|
||||
('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40),
|
||||
('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60),
|
||||
('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100),
|
||||
('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80),
|
||||
]
|
||||
for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels):
|
||||
for i, variant in enumerate(variants):
|
||||
group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate]
|
||||
assert len(group) == 3
|
||||
color = '#9333ea' if i == 0 else '#059669'
|
||||
for j, r in enumerate(group):
|
||||
# Older Node series use a different scheduler: show their observations without a retroactive k6 classification.
|
||||
underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid']
|
||||
ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55)
|
||||
value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group)
|
||||
if dataset == 'pool-limit' else statistics.median(r[field] for r in group))
|
||||
ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2)
|
||||
ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value),
|
||||
xytext=(12, 8), textcoords='offset points', fontsize=9)
|
||||
ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4))
|
||||
ax.grid(axis='y', alpha=.25)
|
||||
if field == 'all_request_p95_ms':
|
||||
ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1)
|
||||
maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate)
|
||||
ax.set_ylim(0, maximum * 1.2)
|
||||
fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13)
|
||||
fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8)
|
||||
fig.tight_layout(rect=(0, .04, 1, .95))
|
||||
fig.savefig(output / 'improvements.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup')
|
||||
parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
|
||||
parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json')
|
||||
parser.add_argument('--plot', action='store_true')
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8'))
|
||||
rows = flatten(bundle)
|
||||
verify(bundle, rows)
|
||||
with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=rows[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
if args.plot:
|
||||
plot(rows, args.output)
|
||||
print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
if defined GIT_BASH goto run
|
||||
set "GIT_BASH=%ProgramFiles%\Git\bin\bash.exe"
|
||||
if exist "%GIT_BASH%" goto run
|
||||
echo Git Bash was not found. Set GIT_BASH to the full path to bash.exe. 1>&2
|
||||
exit /b 2
|
||||
:run
|
||||
"%GIT_BASH%" "%~dp0restart.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_dir"
|
||||
|
||||
for service in api nginx postgres redis; do
|
||||
if [ -z "$(docker compose ps --all --quiet "$service")" ]; then
|
||||
printf '%s\n' "Missing container for $service; start the Compose stack first." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
replicas=$(docker compose ps --all --quiet api | wc -l | tr -d ' ')
|
||||
if [ "$replicas" -lt 1 ]; then
|
||||
printf 'No API replicas found.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker compose stop --timeout 30 nginx api postgres redis
|
||||
docker compose up --detach --no-recreate --no-build --no-deps --wait --wait-timeout 45 postgres redis
|
||||
docker compose up --detach --no-recreate --no-build --no-deps --scale "api=$replicas" --wait --wait-timeout 45 api
|
||||
docker compose up --detach --no-recreate --no-build --no-deps nginx
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4,
|
||||
'one-small': 1, 'two-small': 2, 'four-small': 4}
|
||||
spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py')
|
||||
diag = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diag)
|
||||
|
||||
|
||||
def call(args, timeout=180, env=None):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout, env=env)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def watch(container, role, directory):
|
||||
stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop'
|
||||
output = (directory/f'{role}.jsonl').open('wb')
|
||||
error = (directory/f'{role}.stderr').open('wb')
|
||||
process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error)
|
||||
process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes())
|
||||
process.stdin.close()
|
||||
return container, role, stop, process, output, error
|
||||
|
||||
|
||||
def stop_watch(worker, directory):
|
||||
container, role, stop, process, output, error = worker
|
||||
try:
|
||||
call(['docker', 'exec', container, 'touch', stop], timeout=15)
|
||||
process.wait(timeout=15)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
output.close()
|
||||
error.close()
|
||||
if role == 'postgres':
|
||||
call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')])
|
||||
call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl'])
|
||||
|
||||
|
||||
def classify(benchmark):
|
||||
m = benchmark['measurement']
|
||||
count = m['all']['attempted']
|
||||
ratio = count/max(m['scheduled_slots'], 1)
|
||||
error_fraction = m['all']['errors']/max(count, 1)
|
||||
p95 = benchmark['all_latency_ms']['p95']
|
||||
delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02
|
||||
slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000
|
||||
return {'delivery_fraction': ratio, 'error_fraction': error_fraction,
|
||||
'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo,
|
||||
'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
parser.add_argument('--image', default='grafana/k6:1.3.0')
|
||||
parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four'])
|
||||
parser.add_argument('--probe-seconds', type=int, default=15)
|
||||
parser.add_argument('--confirm-seconds', type=int, default=30)
|
||||
parser.add_argument('--warmup-seconds', type=int, default=10,
|
||||
help='Mixed-load warmup before each confirmation run')
|
||||
parser.add_argument('--calibration-only', action='store_true')
|
||||
parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search')
|
||||
parser.add_argument('--no-expand', action='store_true',
|
||||
help='With --confirm-rates, run only the requested rates for a controlled comparison')
|
||||
args = parser.parse_args()
|
||||
if args.no_expand and not args.confirm_rates:
|
||||
parser.error('--no-expand requires --confirm-rates')
|
||||
if args.warmup_seconds < 1:
|
||||
parser.error('--warmup-seconds must be positive')
|
||||
out = args.output.resolve()
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles,
|
||||
'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds,
|
||||
'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5,
|
||||
'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {},
|
||||
'stopped_profiles': {}, 'observer_version': 3,
|
||||
'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms',
|
||||
'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.',
|
||||
'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.',
|
||||
'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.',
|
||||
'Probe durations differ; use confirmation rows for reported bounds.'],
|
||||
'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in
|
||||
['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py',
|
||||
'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh',
|
||||
'compose.yaml', 'test/compose.diagnostics.yaml',
|
||||
*[f'configs/{profile}.yaml' for profile in args.profiles]]}}
|
||||
save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8')
|
||||
manifest['requested_confirmation_rates'] = args.confirm_rates
|
||||
manifest['expand_confirmation_rates'] = not args.no_expand
|
||||
for name in manifest['source_sha256']:
|
||||
target = out/'sources'/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((ROOT/name).read_bytes())
|
||||
save()
|
||||
nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0]
|
||||
network = next(iter(nginx['NetworkSettings']['Networks']))
|
||||
api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0]
|
||||
api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image']
|
||||
suffix = uuid.uuid4().hex
|
||||
generator = 'lab3-saturation-client-' + suffix
|
||||
mock = 'lab3-saturation-mock-' + suffix
|
||||
generator_watch = None
|
||||
counter = 0
|
||||
manifest['generator_container'] = generator
|
||||
manifest['mock_container'] = mock
|
||||
save()
|
||||
|
||||
def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'):
|
||||
nonlocal counter
|
||||
if (out/'CANCEL').exists():
|
||||
raise InterruptedError('Cancellation requested; restoring ordinary profile')
|
||||
counter += 1
|
||||
label = f'{profile}-r{rate}-{phase}-{counter}'
|
||||
directory = out/label
|
||||
directory.mkdir()
|
||||
print(f'START {label} ({seconds}s)', flush=True)
|
||||
envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm),
|
||||
'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase,
|
||||
'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))}
|
||||
command = ['docker', 'exec']
|
||||
for k, v in envs.items():
|
||||
command += ['-e', f'{k}={v}']
|
||||
command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js']
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=seconds+warm+150)
|
||||
(directory/'execution.json').write_text(json.dumps({'returncode': result.returncode,
|
||||
'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n')
|
||||
(directory/'k6.stdout').write_text(result.stdout, encoding='utf-8')
|
||||
(directory/'k6.stderr').write_text(result.stderr, encoding='utf-8')
|
||||
if result.returncode or not (directory/'benchmark.json').exists():
|
||||
raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}')
|
||||
b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8'))
|
||||
if not b['timing']['measurement_started_epoch_ms']:
|
||||
raise RuntimeError(f'{label}: no measured requests')
|
||||
verdict = classify(b)
|
||||
(directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n')
|
||||
print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} "
|
||||
f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms "
|
||||
f"errors={b['measurement']['all']['errors']}", flush=True)
|
||||
if profile != 'calibration':
|
||||
manifest['runs'].append(label)
|
||||
manifest['rates'] = sorted(set(manifest['rates']+[rate]))
|
||||
save()
|
||||
time.sleep(1 if verdict['classification'] == 'pass' else 5)
|
||||
return label, verdict
|
||||
|
||||
try:
|
||||
call(['docker', 'run', '-d', '--name', generator, '--network', network,
|
||||
'-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results',
|
||||
'--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done'])
|
||||
manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image']
|
||||
manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version'])
|
||||
manifest['docker'] = json.loads(call(['docker', 'info', '--format',
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}']))
|
||||
generator_watch = watch(generator, 'generator', out)
|
||||
call(['docker', 'run', '-d', '--name', mock, '--network', network,
|
||||
'-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro',
|
||||
'--entrypoint', 'node', api_image, '/mock.cjs'])
|
||||
calibrations = []
|
||||
for calibration_rate in [400, 200, 100]:
|
||||
label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080')
|
||||
calibrations.append({'label': label, 'rate': calibration_rate, **calibration})
|
||||
if calibration['classification'] == 'pass': break
|
||||
manifest['calibration_attempts'] = calibrations
|
||||
manifest['calibration'] = {'label': label, **calibration}
|
||||
low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080')
|
||||
manifest['calibration_low_rate'] = {'label': low_label, **low_calibration}
|
||||
save()
|
||||
call(['docker', 'rm', '-f', mock])
|
||||
if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass':
|
||||
raise RuntimeError('Generator calibration failed; do not claim API saturation')
|
||||
if args.calibration_only:
|
||||
return
|
||||
for profile in args.profiles:
|
||||
profile_started = datetime.now(timezone.utc).isoformat()
|
||||
n = PROFILE_REPLICAS[profile]
|
||||
print(f'SWITCH {profile}, pool={32//n} per replica', flush=True)
|
||||
env = {**os.environ, 'DB_POOL_MAX': str(32//n)}
|
||||
call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env)
|
||||
time.sleep(6)
|
||||
containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']}
|
||||
details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]]))
|
||||
manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'],
|
||||
'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'],
|
||||
'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details]
|
||||
profile_dir = out/f'observers-{profile}'
|
||||
profile_dir.mkdir()
|
||||
workers = []
|
||||
first = len(manifest['runs'])
|
||||
save()
|
||||
try:
|
||||
for role in ['postgres', 'nginx', 'redis']:
|
||||
workers.append(watch(containers[role][0], role, profile_dir))
|
||||
low = None
|
||||
high = None
|
||||
for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]):
|
||||
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = rate
|
||||
else:
|
||||
high = rate
|
||||
break
|
||||
if low is None and not args.confirm_rates:
|
||||
for rate in [20, 10]:
|
||||
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = rate; break
|
||||
high = rate
|
||||
if low is not None and high is not None:
|
||||
for _ in range(2):
|
||||
if high-low <= 20: break
|
||||
middle = ((low+high)//20)*10
|
||||
_, v = run(profile, middle, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = middle
|
||||
else: high = middle
|
||||
confirmations = {}
|
||||
for repeat in range(3):
|
||||
for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])):
|
||||
_, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)
|
||||
confirmations.setdefault(rate, []).append(v)
|
||||
for _ in range(0 if args.no_expand else 2):
|
||||
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
||||
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
||||
if good and bad and max(good) < min(bad): break
|
||||
if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break
|
||||
rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20
|
||||
if rate in confirmations: break
|
||||
confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)]
|
||||
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
||||
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
||||
manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None,
|
||||
'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations,
|
||||
'bracketed': bool(good and bad and max(good) < min(bad))}
|
||||
print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. "
|
||||
f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True)
|
||||
finally:
|
||||
for worker in workers: stop_watch(worker, profile_dir)
|
||||
log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started,
|
||||
containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=30)
|
||||
events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines()
|
||||
if any(term in line for term in ['worker_connections', 'no live upstreams',
|
||||
'upstream timed out', 'download timed out', 'signal process started'])]
|
||||
(profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8')
|
||||
for index, container in enumerate(containers['api'], 1):
|
||||
call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')])
|
||||
for label in manifest['runs'][first:]:
|
||||
directory = out/label
|
||||
for p in profile_dir.glob('*.jsonl'):
|
||||
(directory/p.name).write_bytes(p.read_bytes())
|
||||
(directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes())
|
||||
b = json.loads((directory/'benchmark.json').read_text())
|
||||
summary = diag.summarize(directory, b)
|
||||
samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup'
|
||||
and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']]
|
||||
summary['resources']['generator'] = diag.resource_summary(samples)
|
||||
summary['all_latency_ms'] = b['all_latency_ms']
|
||||
summary['criterion'] = json.loads((directory/'criterion.json').read_text())
|
||||
summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values())
|
||||
(directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
|
||||
save()
|
||||
finally:
|
||||
if generator_watch: stop_watch(generator_watch, out)
|
||||
for name in [mock, generator]:
|
||||
subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30)
|
||||
if not args.calibration_only:
|
||||
print('RESTORE normal two profile, original pool setting', flush=True)
|
||||
call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'])
|
||||
manifest['finished'] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import hashlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, timeout=180):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def compose(profile, diagnostics=True):
|
||||
args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"]
|
||||
if diagnostics:
|
||||
args += ["-f", "test/compose.diagnostics.yaml"]
|
||||
return args
|
||||
|
||||
|
||||
def rows(path):
|
||||
data = []
|
||||
if path.exists():
|
||||
text = path.read_text(encoding="utf-8")
|
||||
decoder = json.JSONDecoder()
|
||||
position = 0
|
||||
while position < len(text):
|
||||
if text[position].isspace():
|
||||
position += 1
|
||||
continue
|
||||
try:
|
||||
value, position = decoder.raw_decode(text, position)
|
||||
data.append(value)
|
||||
except json.JSONDecodeError:
|
||||
break # Last record can be in progress when a running observer is copied.
|
||||
return data
|
||||
|
||||
|
||||
def delta(first, last, name):
|
||||
return max(0, last.get(name, 0) - first.get(name, 0))
|
||||
|
||||
|
||||
def resource_summary(samples):
|
||||
if len(samples) < 2:
|
||||
return {"samples": len(samples)}
|
||||
first, last = samples[0], samples[-1]
|
||||
elapsed = (last["time_ms"] - first["time_ms"]) / 1000
|
||||
periods = delta(first, last, "cpu_nr_periods")
|
||||
return {
|
||||
"samples": len(samples), "coverage_seconds": elapsed,
|
||||
"max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])),
|
||||
"mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1),
|
||||
"throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6,
|
||||
"throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1),
|
||||
"peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2,
|
||||
"oom_kills": delta(first, last, "memory_oom_kill"),
|
||||
}
|
||||
|
||||
|
||||
def summarize(directory, benchmark):
|
||||
start = benchmark["timing"]["measurement_started_epoch_ms"]
|
||||
end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"]
|
||||
within = lambda data: [r for r in data if start <= r["time_ms"] <= end]
|
||||
resources = {}
|
||||
pools = {}
|
||||
for path in sorted(directory.glob("api-*.jsonl")):
|
||||
data = within(rows(path))
|
||||
flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()},
|
||||
**{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}}
|
||||
for r in data]
|
||||
resources[path.stem] = resource_summary(flattened)
|
||||
if len(data) < 2:
|
||||
continue
|
||||
first, last = data[0]["pools"], data[-1]["pools"]
|
||||
acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first)
|
||||
acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first)
|
||||
intervals = [p for r in data for p in r["pools"]]
|
||||
totals = [sum(p["total"] for p in r["pools"]) for r in data]
|
||||
pools[path.stem] = {
|
||||
"samples": len(data), "acquisitions": acquisitions,
|
||||
"acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first),
|
||||
"mean_acquisition_ms": acquire_ms / max(acquisitions, 1),
|
||||
"max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0),
|
||||
"max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0),
|
||||
"queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1),
|
||||
"max_total": max((p["total"] for p in intervals), default=0),
|
||||
"observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])),
|
||||
"observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])),
|
||||
"event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data),
|
||||
}
|
||||
pg = within(rows(directory / "postgres.jsonl"))
|
||||
pg_samples = [r for r in pg if r["kind"] == "postgres"]
|
||||
pg_samples += within(rows(directory / "postgres-sql.jsonl"))
|
||||
pg_samples.sort(key=lambda r: r["time_ms"])
|
||||
host_samples = [r for r in pg if r["kind"] == "host"]
|
||||
host = {"samples": len(host_samples)}
|
||||
if len(host_samples) > 1:
|
||||
host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000
|
||||
host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1)
|
||||
for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")})
|
||||
host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024
|
||||
host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024
|
||||
waits = {}
|
||||
max_active = max_idle_tx = 0
|
||||
for sample in pg_samples:
|
||||
max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active"))
|
||||
max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction"))
|
||||
for a in sample["activity"]:
|
||||
if a["state"] == "active":
|
||||
key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}"
|
||||
waits[key] = waits.get(key, 0) + a["count"]
|
||||
for role in ("postgres", "nginx", "redis"):
|
||||
resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"])
|
||||
upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"]
|
||||
changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:]))
|
||||
m = benchmark["measurement"]
|
||||
return {
|
||||
"label": benchmark["label"], "profile": benchmark["profile"],
|
||||
"measurement": m, "client": benchmark["client"],
|
||||
"resources": resources, "pools": pools,
|
||||
"host": host,
|
||||
"postgres": {"samples": len(pg_samples), "max_active": max_active,
|
||||
"max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits},
|
||||
"nginx": {"samples": len(upstream), "upstream_changes": changes,
|
||||
"min_ready": min((len(r["servers"]) for r in upstream), default=None),
|
||||
"max_ready": max((len(r["servers"]) for r in upstream), default=None)},
|
||||
"limits_reached": {
|
||||
"generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05,
|
||||
"client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05,
|
||||
"request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01,
|
||||
"latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_one(config, profile, rate, repeat, containers):
|
||||
label = f"{profile}-r{rate}-repeat{repeat}"
|
||||
directory = config.output / label
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
workers = []
|
||||
stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop"
|
||||
observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes()
|
||||
client_container = None
|
||||
try:
|
||||
for role in ("postgres", "nginx", "redis"):
|
||||
output = (directory / f"{role}.jsonl").open("wb")
|
||||
errors = (directory / f"{role}.stderr").open("wb")
|
||||
child = subprocess.Popen(["docker", "exec", "-i", containers[role][0],
|
||||
"sh", "-s", role, stop], cwd=ROOT,
|
||||
stdin=subprocess.PIPE, stdout=output, stderr=errors)
|
||||
workers.append((role, child, output, errors))
|
||||
child.stdin.write(observer)
|
||||
child.stdin.close()
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
target = directory / "benchmark.json"
|
||||
args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url,
|
||||
"--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds),
|
||||
"--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30",
|
||||
"--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]),
|
||||
"--output", str(target)]
|
||||
if config.client == "container":
|
||||
nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0]
|
||||
api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0]
|
||||
networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"])
|
||||
if len(networks) != 1:
|
||||
raise RuntimeError("expected one common API/nginx network")
|
||||
client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex
|
||||
args[args.index("--base-url") + 1] = config.container_url
|
||||
args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json"
|
||||
args[1] = "/benchmark.mjs"
|
||||
args = ["docker", "run", "--name", client_container, "--no-healthcheck",
|
||||
"--network", next(iter(networks)),
|
||||
"-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro",
|
||||
"--entrypoint", "node", api["Image"], *args[1:]]
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace",
|
||||
timeout=config.duration_seconds + config.warmup_seconds + 120)
|
||||
(directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8")
|
||||
if client_container is not None:
|
||||
command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)])
|
||||
if not target.exists():
|
||||
raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}")
|
||||
benchmark = json.loads(target.read_text(encoding="utf-8"))
|
||||
for index, container in enumerate(containers["api"], 1):
|
||||
command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")])
|
||||
ended = datetime.now(timezone.utc).isoformat()
|
||||
# Docker puts nginx stderr notices on CLI stderr; collect both streams.
|
||||
log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started,
|
||||
"--until", ended, containers["nginx"][0]], cwd=ROOT,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
|
||||
diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines()
|
||||
if any(key in line for key in ("signal process started", "download timed out",
|
||||
"no live upstreams", "upstream timed out"))]
|
||||
(directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8")
|
||||
(directory / "execution.json").write_text(json.dumps({"returncode": result.returncode,
|
||||
"started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8")
|
||||
finally:
|
||||
if client_container is not None:
|
||||
# Only the uniquely named disposable generator created by this run.
|
||||
subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30)
|
||||
for role, child, output, errors in workers:
|
||||
try:
|
||||
command(["docker", "exec", containers[role][0], "touch", stop], timeout=15)
|
||||
child.wait(timeout=15)
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
child.terminate()
|
||||
child.wait(timeout=10)
|
||||
output.close()
|
||||
errors.close()
|
||||
if role == "postgres":
|
||||
command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")])
|
||||
command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"])
|
||||
summary = summarize(directory, benchmark)
|
||||
summary["diagnostics_complete"] = (
|
||||
len(summary["pools"]) == len(containers["api"])
|
||||
and summary["postgres"]["samples"] > 0
|
||||
and summary["nginx"]["samples"] > 0
|
||||
and all(v["samples"] > 0 for v in summary["resources"].values())
|
||||
)
|
||||
(directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
if not summary["diagnostics_complete"]:
|
||||
raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files")
|
||||
m = summary["measurement"]
|
||||
print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms "
|
||||
f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} "
|
||||
f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
|
||||
parser.add_argument("--client", choices=("host", "container"), default="host")
|
||||
parser.add_argument("--container-url", default="http://nginx")
|
||||
parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"])
|
||||
parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)")
|
||||
parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)")
|
||||
parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)")
|
||||
parser.add_argument("--repetitions", type=int, default=3)
|
||||
parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
config = parser.parse_args()
|
||||
if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates):
|
||||
parser.error("invalid duration, repetitions or rates")
|
||||
config.output = config.output.resolve()
|
||||
config.output.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles,
|
||||
"client": config.client,
|
||||
"rates": config.rates, "duration_seconds": config.duration_seconds,
|
||||
"warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions,
|
||||
"full_matrix": config.full_matrix,
|
||||
"stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix
|
||||
else "stop a profile after >=2 repeats at one rate exceed any threshold: ") +
|
||||
"scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms",
|
||||
"sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s",
|
||||
"observer_version": 3,
|
||||
"source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in (
|
||||
"scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh",
|
||||
"scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")},
|
||||
"notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.",
|
||||
"A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.",
|
||||
"Acquisition latency includes opening new connections; it is not SQL execution time.",
|
||||
"Observers consume resources inside the existing container budgets.",
|
||||
"Instrumentation stores no tokens, passwords, message text or SQL parameters."],
|
||||
"containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}}
|
||||
manifest_path = config.output / "manifest.json"
|
||||
save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
||||
save()
|
||||
try:
|
||||
for profile in config.profiles:
|
||||
print(f"SWITCH {profile}", flush=True)
|
||||
command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
||||
time.sleep(6) # Allow readiness's two successes and nginx reload to settle.
|
||||
containers = {}
|
||||
for role in ("api", "postgres", "nginx", "redis"):
|
||||
containers[role] = command(compose(profile) + ["ps", "-q", role]).split()
|
||||
if not containers[role]:
|
||||
raise RuntimeError(f"no running {role}")
|
||||
details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]]))
|
||||
running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()]
|
||||
manifest["environments"][profile] = {
|
||||
"docker": json.loads(command(["docker", "info", "--format",
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])),
|
||||
"running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running],
|
||||
}
|
||||
manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"],
|
||||
"nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"],
|
||||
"pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details]
|
||||
save()
|
||||
for rate in config.rates:
|
||||
batch = []
|
||||
for repeat in range(1, config.repetitions + 1):
|
||||
print(f"START {profile} rate={rate} repeat={repeat}", flush=True)
|
||||
summary = run_one(config, profile, rate, repeat, containers)
|
||||
batch.append(summary)
|
||||
manifest["runs"].append(summary["label"])
|
||||
save()
|
||||
reasons = [key for key in batch[0]["limits_reached"]
|
||||
if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)]
|
||||
if reasons:
|
||||
manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons})
|
||||
if not config.full_matrix:
|
||||
manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons}
|
||||
print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True)
|
||||
save()
|
||||
if not config.full_matrix:
|
||||
break
|
||||
finally:
|
||||
print("RESTORE two (without diagnostic preload)", flush=True)
|
||||
command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
||||
manifest["finished"] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
// Generator calibration only: no application or database state.
|
||||
const http = require('node:http');
|
||||
const item = { id: '00000000-0000-4000-8000-000000000001',
|
||||
chat_id: '00000000-0000-4000-8000-000000000002',
|
||||
sender_id: '00000000-0000-4000-8000-000000000003',
|
||||
text: 'Synthetic benchmark payload '.repeat(3), created_at: '2026-01-01T00:00:00.000Z' };
|
||||
const page = JSON.stringify({ items: Array.from({ length: 50 }, () => item), next_cursor: null });
|
||||
const keys = new Set();
|
||||
http.createServer(async (req, res) => {
|
||||
for await (const chunk of req) { /* consume POST body */ }
|
||||
if (req.method === 'POST') {
|
||||
const key = req.headers['idempotency-key'];
|
||||
if (!key || keys.has(key)) {
|
||||
res.writeHead(409, { 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
|
||||
res.end(JSON.stringify({ error: 'calibration detected a repeated write key' }));
|
||||
return;
|
||||
}
|
||||
keys.add(key);
|
||||
}
|
||||
res.writeHead(req.method === 'GET' ? 200 : 201,
|
||||
{ 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
|
||||
res.end(req.method === 'GET' ? page : JSON.stringify(item));
|
||||
}).listen(8080, '0.0.0.0');
|
||||
@@ -0,0 +1,166 @@
|
||||
import http from 'k6/http';
|
||||
import encoding from 'k6/encoding';
|
||||
import execution from 'k6/execution';
|
||||
import { check, fail, sleep } from 'k6';
|
||||
import { Counter, Trend } from 'k6/metrics';
|
||||
|
||||
const base = __ENV.BASE_URL || 'http://nginx';
|
||||
const rate = Number(__ENV.RATE || 100);
|
||||
const seconds = Number(__ENV.SECONDS || 30);
|
||||
const warmup = Number(__ENV.WARMUP || 10);
|
||||
const expected = Number(__ENV.EXPECTED || 1);
|
||||
const calibration = __ENV.CALIBRATION === '1';
|
||||
const attempts = new Counter('bench_attempts');
|
||||
const successes = new Counter('bench_successes');
|
||||
const failures = new Counter('bench_failures');
|
||||
const latency = new Trend('bench_latency', true);
|
||||
const successLatency = new Trend('bench_success_latency', true);
|
||||
const epoch = new Trend('bench_start_epoch');
|
||||
const operations = Object.fromEntries(['read', 'write'].map((op) => [op, {
|
||||
attempts: new Counter(`bench_${op}_attempts`),
|
||||
successes: new Counter(`bench_${op}_successes`),
|
||||
failures: new Counter(`bench_${op}_failures`),
|
||||
latency: new Trend(`bench_${op}_latency`, true),
|
||||
}]));
|
||||
const statuses = Object.fromEntries(['200', '201', 'other', 'network'].map((key) => [key, new Counter(`bench_status_${key}`)]));
|
||||
|
||||
const arrival = { executor: 'constant-arrival-rate', rate, timeUnit: '1s',
|
||||
preAllocatedVUs: 128, maxVUs: 128 };
|
||||
export const options = {
|
||||
scenarios: {
|
||||
warmup: { ...arrival, exec: 'warmupLoad', duration: `${warmup}s`, gracefulStop: '5s' },
|
||||
measurement: { ...arrival, exec: 'default', startTime: `${warmup}s`,
|
||||
duration: `${seconds}s`, gracefulStop: '10s' },
|
||||
},
|
||||
summaryTrendStats: ['min', 'med', 'avg', 'max', 'p(95)', 'p(99)'],
|
||||
setupTimeout: '120s',
|
||||
};
|
||||
|
||||
function request(method, path, token, body, key) {
|
||||
const headers = { Accept: 'application/json' };
|
||||
if (token) headers.Authorization = token;
|
||||
if (key) headers['Idempotency-Key'] = key;
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
return http.request(method, base + path, body === undefined ? null : JSON.stringify(body), {
|
||||
headers, timeout: '5s', tags: { name: path.includes('/messages') ? 'messages' : 'setup' },
|
||||
});
|
||||
}
|
||||
|
||||
function required(response, status, field) {
|
||||
let data;
|
||||
try { data = response.json(); } catch { fail(`setup: invalid JSON, status ${response.status}`); }
|
||||
if (response.status !== status || !data?.[field]) fail(`setup: expected ${status}/${field}, got ${response.status}`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export function setup() {
|
||||
const salt = `${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9)}`;
|
||||
let token = 'Bearer calibration';
|
||||
let path = '/messages';
|
||||
if (!calibration) {
|
||||
const users = ['alice', 'bob'].map((label) => {
|
||||
const username = `bench_${label}_${salt.replaceAll('-', '')}`;
|
||||
const password = `T3st-${salt}-Aa1!`;
|
||||
const user = required(request('POST', '/api/v1/users', null,
|
||||
{ username, display_name: `Benchmark ${label}`, password }), 201, 'id');
|
||||
const auth = `Basic ${encoding.b64encode(`${username}:${password}`)}`;
|
||||
const session = required(request('POST', '/api/v1/auth/sessions', auth), 201, 'token');
|
||||
return { id: user.id, token: session.token };
|
||||
});
|
||||
token = `Bearer ${users[0].token}`;
|
||||
const chat = required(request('POST', '/api/v1/chats', token,
|
||||
{ title: 'Saturation benchmark', member_ids: users.map((u) => u.id) }), 201, 'id');
|
||||
path = `/api/v1/chats/${chat.id}/messages`;
|
||||
for (let i = 0; i < 30; i++) required(request('POST', path, token,
|
||||
{ text: `seed-${salt}-${i}` }, `seed-${salt}-${i}`), 201, 'id');
|
||||
}
|
||||
const seen = new Set();
|
||||
const until = Date.now() + 20000;
|
||||
do {
|
||||
const response = request('GET', `${path}?limit=50`, token);
|
||||
if (response.status !== 200) fail(`warmup status ${response.status}`);
|
||||
const id = response.headers['X-Instance-Id'];
|
||||
if (id) seen.add(id);
|
||||
if (seen.size === expected) break;
|
||||
sleep(0.02);
|
||||
} while (Date.now() < until);
|
||||
if (seen.size !== expected) fail(`warmup observed ${seen.size} instances, expected ${expected}`);
|
||||
return { token, path, salt };
|
||||
}
|
||||
|
||||
export function warmupLoad(data) { iteration(data, false); }
|
||||
export default function (data) { iteration(data, true); }
|
||||
|
||||
function iteration(data, measuring) {
|
||||
if (measuring && execution.scenario.iterationInTest === 0) epoch.add(Date.now());
|
||||
const sequence = execution.scenario.iterationInTest;
|
||||
const identity = `${execution.scenario.name}-${sequence}`;
|
||||
const op = ((sequence * 37 + 17) % 100) < 80 ? 'read' : 'write';
|
||||
if (measuring) { attempts.add(1); operations[op].attempts.add(1); }
|
||||
const response = op === 'read'
|
||||
? request('GET', `${data.path}?limit=50`, data.token)
|
||||
: request('POST', data.path, data.token, { text: `bench-${identity}-${data.salt}` }, `${data.salt}-${identity}`);
|
||||
if (!measuring) return;
|
||||
let payload;
|
||||
try { payload = response.json(); } catch { payload = null; }
|
||||
const instance = response.headers['X-Instance-Id'];
|
||||
const ok = response.status === (op === 'read' ? 200 : 201) && Boolean(instance)
|
||||
&& (op === 'read' ? Array.isArray(payload?.items) : Boolean(payload?.id));
|
||||
const elapsed = response.timings.duration + response.timings.blocked;
|
||||
latency.add(elapsed);
|
||||
successes.add(ok ? 1 : 0);
|
||||
failures.add(ok ? 0 : 1);
|
||||
operations[op].successes.add(ok ? 1 : 0);
|
||||
operations[op].failures.add(ok ? 0 : 1);
|
||||
if (ok) { successLatency.add(elapsed); operations[op].latency.add(elapsed); }
|
||||
statuses[String(response.status)]?.add(1);
|
||||
if (![200, 201].includes(response.status)) statuses[response.status === 0 ? 'network' : 'other'].add(1);
|
||||
if (instance) check(response, { [`instance:${instance}`]: () => true });
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
const count = (name) => data.metrics[name]?.values.count || 0;
|
||||
const times = (name) => ({ p50: data.metrics[name]?.values.med ?? null,
|
||||
p95: data.metrics[name]?.values['p(95)'] ?? null,
|
||||
p99: data.metrics[name]?.values['p(99)'] ?? null });
|
||||
const start = data.metrics.bench_start_epoch?.values.min;
|
||||
const totalDropped = count('dropped_iterations');
|
||||
const dropped = Math.max(0, rate * seconds - count('bench_attempts'));
|
||||
const instances = Object.fromEntries((data.root_group.checks || []).filter((c) => c.name.startsWith('instance:'))
|
||||
.map((c) => [c.name.slice(9), c.passes]));
|
||||
const group = (op) => {
|
||||
const prefix = op === 'all' ? 'bench' : `bench_${op}`;
|
||||
return { attempted: count(`${prefix}_attempts`), successful: count(`${prefix}_successes`),
|
||||
errors: count(`${prefix}_failures`), successful_rps: count(`${prefix}_successes`) / seconds,
|
||||
successful_latency_ms: times(op === 'all' ? 'bench_success_latency' : `${prefix}_latency`),
|
||||
status_counts: {}, error_counts: {}, instance_counts: op === 'all' ? instances : {} };
|
||||
};
|
||||
const result = {
|
||||
label: __ENV.LABEL, profile: { arrival_model: 'k6 constant-arrival-rate; one HTTP request per iteration',
|
||||
offered_rps: rate, duration_seconds: seconds, warmup_seconds: warmup, warmup_profile: 'same 80/20 workload at target rate',
|
||||
read_percent: 80, write_percent: 20, initial_messages: calibration ? 0 : 30, history_limit: 50,
|
||||
max_in_flight: 128, timeout_ms: 5000, automatic_retries: false, expected_instances: expected,
|
||||
phase: __ENV.PHASE || 'probe' },
|
||||
measurement: { scheduled_slots: rate * seconds, client_dropped_slots: dropped,
|
||||
scheduler_late_slots: 0, read: group('read'), write: group('write'), all: group('all') },
|
||||
timing: { measurement_started_epoch_ms: start,
|
||||
measurement_scheduled_end_epoch_ms: start + seconds * 1000,
|
||||
measurement_completed_epoch_ms: Date.now() },
|
||||
client: { engine: 'k6', scheduler_lateness_measured: false,
|
||||
raw_dropped_iterations_including_warmup: totalDropped,
|
||||
peak_vus: data.metrics.vus?.values.max, max_vus: 128 },
|
||||
all_latency_ms: times('bench_latency'), raw_metrics: data.metrics,
|
||||
notes: ['Setup and in-scenario mixed warmup excluded from custom bench metrics.',
|
||||
'Native warmup and measurement scenarios; no Date.now gating of measured requests.',
|
||||
'Latency uses native HTTP duration plus blocked time, not subtraction of wall-clock timestamps.',
|
||||
'Measurement missing slots are planned minus actual starts; raw k6 dropped iterations also include warmup.',
|
||||
'scheduler_late_slots is a compatibility placeholder, not a k6 measurement.',
|
||||
'Dropped iterations and actual delivered fraction must be checked.',
|
||||
'No credentials or message bodies are exported.'],
|
||||
};
|
||||
result.measurement.all.status_counts = Object.fromEntries(['200', '201', 'other', 'network']
|
||||
.map((status) => [status, count(`bench_status_${status}`)]));
|
||||
return { [__ENV.OUTPUT]: JSON.stringify(result, null, 2),
|
||||
stdout: JSON.stringify({ label: result.label, attempted: result.measurement.all.attempted,
|
||||
errors: result.measurement.all.errors, p95: result.all_latency_ms.p95, dropped }) + '\n' };
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if not defined TARGET_INSTANCE_ID (
|
||||
echo TARGET_INSTANCE_ID is required 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
for /f "delims=" %%I in ('docker compose ps --all --quiet api') do (
|
||||
for /f "delims=" %%H in ('docker inspect --format "{{.Config.Hostname}}" %%I') do (
|
||||
if "%%H"=="%TARGET_INSTANCE_ID%" (
|
||||
docker stop --time 2 %%I >nul
|
||||
if errorlevel 1 exit /b 1
|
||||
exit /b 0
|
||||
)
|
||||
)
|
||||
)
|
||||
echo No API container matches X-Instance-Id 1>&2
|
||||
exit /b 2
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_dir"
|
||||
|
||||
target=${TARGET_INSTANCE_ID:?test provides target API instance ID}
|
||||
|
||||
containers=$(docker compose ps --all --quiet api)
|
||||
if [ -z "$containers" ]; then
|
||||
printf 'No API containers found\n' >&2
|
||||
exit 2
|
||||
fi
|
||||
for entry in $(docker inspect --format '{{.Config.Hostname}}:{{.Id}}' $containers); do
|
||||
instance=${entry%%:*}
|
||||
container=${entry#*:}
|
||||
if [ "$instance" = "$target" ]; then
|
||||
docker stop --time 30 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'No API container matches X-Instance-Id\n' >&2
|
||||
exit 2
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create CSV and Markdown from completed diagnostic runs (can run during a sweep)."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import importlib.util
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--refresh", action="store_true", help="Recompute summaries from raw records")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
results = []
|
||||
raw_summaries = []
|
||||
if args.refresh:
|
||||
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).with_name("run-scaling-diagnostics.py"))
|
||||
diagnostics = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diagnostics)
|
||||
for label in manifest["runs"]:
|
||||
path = args.directory / label / "summary.json"
|
||||
if not path.exists():
|
||||
continue
|
||||
s = json.loads(path.read_text(encoding="utf-8"))
|
||||
if args.refresh:
|
||||
benchmark = json.loads((path.parent / "benchmark.json").read_text(encoding="utf-8"))
|
||||
refreshed = diagnostics.summarize(path.parent, benchmark)
|
||||
refreshed["diagnostics_complete"] = s.get("diagnostics_complete", False)
|
||||
s = refreshed
|
||||
path.write_text(json.dumps(s, indent=2) + "\n", encoding="utf-8")
|
||||
raw_summaries.append(s)
|
||||
m, p, r = s["measurement"], s["pools"], s["resources"]
|
||||
acquisitions = sum(v["acquisitions"] for v in p.values())
|
||||
results.append({
|
||||
"label": label, "replicas": s["profile"]["expected_instances"],
|
||||
"configuration": next((profile for profile in manifest.get("profiles", [])
|
||||
if label.startswith(profile + "-r")), str(s["profile"]["expected_instances"])),
|
||||
"phase": s["profile"].get("phase", "measurement"),
|
||||
"delivery_valid": s.get("criterion", {}).get("delivery_valid"),
|
||||
"duration_seconds": s["profile"]["duration_seconds"],
|
||||
"offered_rps": s["profile"]["offered_rps"], "successful_rps": m["all"]["successful_rps"],
|
||||
"attempted_rps": m["all"]["attempted"] / s["profile"]["duration_seconds"],
|
||||
"p50_ms": m["all"]["successful_latency_ms"]["p50"],
|
||||
"p95_ms": m["all"]["successful_latency_ms"]["p95"],
|
||||
"all_request_p95_ms": s.get("all_latency_ms", {}).get("p95"),
|
||||
"p99_ms": m["all"]["successful_latency_ms"]["p99"],
|
||||
"errors": m["all"]["errors"], "dropped": m["client_dropped_slots"],
|
||||
"drop_percent": 100*m["client_dropped_slots"]/m["scheduled_slots"],
|
||||
"late_percent": 100*m["scheduler_late_slots"]/m["scheduled_slots"],
|
||||
"acquisition_mean_ms": sum(v["mean_acquisition_ms"]*v["acquisitions"] for v in p.values()) / max(acquisitions, 1),
|
||||
"acquisition_max_ms": max((v["max_acquisition_ms"] for v in p.values()), default=0),
|
||||
"pool_max_waiting": max((v["max_waiting"] for v in p.values()), default=0),
|
||||
"pool_observed_connection_increases": sum(v.get("observed_connection_increases", 0) for v in p.values()),
|
||||
"pool_observed_connection_decreases": sum(v.get("observed_connection_decreases", 0) for v in p.values()),
|
||||
"api_cpu_cores": sum(v.get("mean_cpu_cores", 0) for k, v in r.items() if k.startswith("api-")),
|
||||
"api_max_throttle_percent": 100*max((v.get("throttled_period_fraction", 0) for k, v in r.items() if k.startswith("api-")), default=0),
|
||||
"pg_cpu_cores": r["postgres"].get("mean_cpu_cores"),
|
||||
"pg_throttle_percent": 100*r["postgres"].get("throttled_period_fraction", 0),
|
||||
"nginx_cpu_cores": r["nginx"].get("mean_cpu_cores"),
|
||||
"redis_cpu_cores": r["redis"].get("mean_cpu_cores"),
|
||||
"host_cpu_pressure_percent": 100*s.get("host", {}).get("cpu_some_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_memory_pressure_percent": 100*s.get("host", {}).get("memory_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_io_pressure_percent": 100*s.get("host", {}).get("io_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_min_available_mib": s.get("host", {}).get("min_available_mib"),
|
||||
"host_max_swap_used_mib": s.get("host", {}).get("max_swap_used_mib"),
|
||||
"pg_max_active": s["postgres"]["max_active"],
|
||||
"pg_max_idle_tx": s["postgres"]["max_idle_in_transaction"],
|
||||
"pg_active_wait_observations": json.dumps(s["postgres"]["active_wait_observations"], sort_keys=True),
|
||||
"nginx_min_ready": s["nginx"]["min_ready"], "nginx_changes": s["nginx"]["upstream_changes"],
|
||||
"min_resource_coverage_seconds": min(v.get("coverage_seconds", 0) for v in r.values()),
|
||||
})
|
||||
if not results:
|
||||
print("No completed runs yet")
|
||||
return
|
||||
with (args.directory / "results.csv").open("w", encoding="utf-8", newline="") as output:
|
||||
writer = csv.DictWriter(output, fieldnames=results[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(results)
|
||||
pg_limits = next((c for containers in manifest.get("containers", {}).values()
|
||||
for c in containers if "-postgres-" in c.get("name", "")), None)
|
||||
pg_description = (f"PostgreSQL — {pg_limits['nano_cpus']/1e9:g} CPU/{pg_limits['memory_bytes']/1024**2:g} МиБ."
|
||||
if pg_limits else "Лимиты PostgreSQL не записаны в manifest.")
|
||||
pool_descriptions = [f"{profile}: " + "/".join(sorted({str(c['pool_max']) for c in containers
|
||||
if "-api-" in c.get("name", "") and c.get("pool_max") is not None}))
|
||||
for profile, containers in manifest.get("containers", {}).items()]
|
||||
pool_description = "; ".join(pool_descriptions) or "см. manifest"
|
||||
api_budgets = []
|
||||
for profile, containers in manifest.get("containers", {}).items():
|
||||
apis = [c for c in containers if "-api-" in c.get("name", "")]
|
||||
if apis:
|
||||
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
|
||||
memory = sum(c['memory_bytes'] for c in apis) / 1024**2
|
||||
api_budgets.append(f"{profile}: {cpu:g} CPU/{memory:g} МиБ")
|
||||
api_description = "; ".join(api_budgets) or "см. manifest"
|
||||
probe_description = (f"Разведочные прогоны: {manifest['probe_duration_seconds']} с после "
|
||||
f"{manifest['probe_warmup_seconds']} с прогрева. Указанная ниже длительность "
|
||||
"относится к подтверждающим прогонам. " if "probe_duration_seconds" in manifest else "")
|
||||
lines = ["# Диагностическая серия масштабирования", "",
|
||||
probe_description + f"Завершено прогонов: {len(results)}. Длительность измерения — {manifest['duration_seconds']} с, "
|
||||
f"прогрев — {manifest['warmup_seconds']} с. Пул на реплику — {pool_description}. Общий бюджет API — {api_description}; " + pg_description, "",
|
||||
("Клиент: Linux-контейнер через внутренний адрес Nginx; разделяет ресурсы Docker VM." if manifest.get("client") == "container"
|
||||
else "Клиент: Windows-хост через опубликованный порт Nginx."),
|
||||
f"Версия SQL-наблюдателя: {manifest.get('observer_version', '1/2, первоначальная серия')}. "
|
||||
"Серия с версиями 1/2 имеет повышенные накладные расходы наблюдения; см. калибровку в итоговой интерпретации.", "",
|
||||
"Методика и повторение: [scripts/diagnostics/README.md](../../../scripts/diagnostics/README.md). "
|
||||
"Исходные HTTP-данные, временные ряды и summary каждого прогона находятся в его подкаталоге. "
|
||||
"Полная таблица метрик: [results.csv](results.csv). Все настройки и причины остановки нагрузки: [manifest.json](manifest.json).", "",
|
||||
"Показатели ниже — результаты каждого прогона. CPU выражен в долях одного ядра; "
|
||||
"throttle — доля периодов CPU, в которых cgroup ограничивала выполнение. Получение соединения "
|
||||
"включает установку новых соединений и задержки event loop. Очередь наблюдалась каждые 100 мс. "
|
||||
"Наблюдатели имеют собственные накладные расходы; результаты не являются ненаблюдаемой максимальной ёмкостью API.", "",
|
||||
"| Прогон | Успешные RPS | p95, мс | Ошибки | Пропуски, % | Пул: среднее / max, мс | Max очередь | CPU API / PG | Throttle API max / PG, % | Upstream min / изменений |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for r in results:
|
||||
lines.append(f"| [{r['label']}]({r['label']}/summary.json) | {r['successful_rps']:.2f} | "
|
||||
f"{r['p95_ms']:.0f} | {r['errors']} | {r['drop_percent']:.1f} | "
|
||||
f"{r['acquisition_mean_ms']:.2f} / {r['acquisition_max_ms']:.0f} | {r['pool_max_waiting']} | "
|
||||
f"{r['api_cpu_cores']:.2f} / {r['pg_cpu_cores'] or 0:.2f} | "
|
||||
f"{r['api_max_throttle_percent']:.1f} / {r['pg_throttle_percent']:.1f} | "
|
||||
f"{r['nginx_min_ready']} / {r['nginx_changes']} |")
|
||||
lines += ["", "## Повторы на одном уровне нагрузки", "",
|
||||
"Здесь показаны медианы результатов отдельных прогонов. Медиана p95 не является p95 объединённых запросов.", "",
|
||||
"| Профиль | Задано RPS | Повторов | Медиана отправленных RPS | Медиана успешных RPS | Медиана p95, мс | Всего ошибок | Медиана пропусков, % |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for key in sorted({(r["configuration"], r["offered_rps"]) for r in results}):
|
||||
group = [r for r in results if (r["configuration"], r["offered_rps"]) == key]
|
||||
lines.append(f"| {key[0]} | {key[1]} | {len(group)} | "
|
||||
f"{statistics.median(r['attempted_rps'] for r in group):.2f} | "
|
||||
f"{statistics.median(r['successful_rps'] for r in group):.2f} | "
|
||||
f"{statistics.median(r['p95_ms'] for r in group):.0f} | "
|
||||
f"{sum(r['errors'] for r in group)} | "
|
||||
f"{statistics.median(r['drop_percent'] for r in group):.1f} |")
|
||||
lines += ["", "## Границы серии", ""]
|
||||
for profile, stop in manifest["stopped_profiles"].items():
|
||||
lines.append(f"- {profile}: повышение остановлено после {stop['rate']} запросов/с: {', '.join(stop['reasons'])}.")
|
||||
if manifest.get("full_matrix"):
|
||||
lines.append("Режим полной матрицы: превышения порогов не останавливают повышение нагрузки; все перечисленные точки запланированы к выполнению.")
|
||||
for profile, crossings in manifest.get("threshold_crossings", {}).items():
|
||||
for crossing in crossings:
|
||||
lines.append(f"- {profile}, {crossing['rate']} запросов/с: {', '.join(crossing['reasons'])}.")
|
||||
lines += ["", "Превышение порога задержки или пропусков не доказывает насыщение конкретного компонента. "
|
||||
"Для интерпретации необходимо сопоставить очередь пула, ограничения CPU, ожидания PostgreSQL и изменения upstream. "
|
||||
"SQL-тексты, токены и пароли в этой серии не сохраняются.", ""]
|
||||
if (args.directory / "findings.md").exists():
|
||||
lines += ["## Итоговая интерпретация", "",
|
||||
"[Выводы и графики](findings.md). Проверка полноты серии: [audit.json](audit.json).", ""]
|
||||
(args.directory / "README.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(results)} runs to {args.directory / 'results.csv'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = {'backend', 'postgres', 'redis'}
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(
|
||||
['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
).split()
|
||||
if len(ids) != len(SERVICES):
|
||||
raise RuntimeError('Start backend, postgres and redis before this test')
|
||||
template = (
|
||||
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
||||
'"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"running":{{json .State.Running}},"mounts":{{json .Mounts}}}'
|
||||
)
|
||||
output = subprocess.check_output(
|
||||
['docker', 'inspect', '--format', template, *ids],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
)
|
||||
result = {}
|
||||
for line in output.splitlines():
|
||||
entry = json.loads(line)
|
||||
service = entry.pop('service')
|
||||
entry['volumes'] = [
|
||||
{'name': mount['Name'], 'destination': mount['Destination']}
|
||||
for mount in entry.pop('mounts') if mount['Type'] == 'volume'
|
||||
]
|
||||
result[service] = entry
|
||||
if set(result) != SERVICES or not all(v['running'] for v in result.values()):
|
||||
raise RuntimeError('All three services must be running')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json')
|
||||
args = parser.parse_args()
|
||||
before = snapshot()
|
||||
hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh')
|
||||
if args.scenario == 'public':
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence']
|
||||
else:
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')]
|
||||
command += ['--action-script', str(hook), '--base-url', args.base_url]
|
||||
if args.ca_file:
|
||||
command += ['--ca-file', args.ca_file]
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
|
||||
print(proc.stdout, end='')
|
||||
print(proc.stderr, end='', file=sys.stderr)
|
||||
if proc.returncode:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps({
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'result': 'FAIL', 'exit_code': proc.returncode,
|
||||
'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(),
|
||||
'before': before,
|
||||
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return proc.returncode
|
||||
after = snapshot()
|
||||
for service in sorted(SERVICES):
|
||||
old, new = before[service], after[service]
|
||||
if old['id'] != new['id'] or old['started_at'] == new['started_at']:
|
||||
raise RuntimeError(f'{service}: expected a restart of the same container')
|
||||
if not old['volumes'] or old['volumes'] != new['volumes']:
|
||||
raise RuntimeError(f'{service}: persistent volumes changed or are missing')
|
||||
evidence = {
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'elapsed_seconds': round(time.monotonic() - started, 3),
|
||||
'result': 'PASS', 'test_output': proc.stdout.strip(),
|
||||
'before': before, 'after': after,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: backend, postgres and redis restarted; named volumes preserved.')
|
||||
print(f'Evidence: {args.output}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Post,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiBasicAuth,
|
||||
ApiBearerAuth,
|
||||
ApiCreatedResponse,
|
||||
ApiNoContentResponse,
|
||||
ApiOperation,
|
||||
ApiTags,
|
||||
ApiUnauthorizedResponse,
|
||||
} from '@nestjs/swagger';
|
||||
import { UserGuard, type AuthenticatedRequest } from '../User/user.guard';
|
||||
import { AuthService } from './auth.service';
|
||||
import { Session } from './auth.schema';
|
||||
|
||||
@ApiTags('auth')
|
||||
@Controller('api/v1/auth/sessions')
|
||||
export class AuthController {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(HttpStatus.CREATED)
|
||||
@ApiBasicAuth('basicAuth')
|
||||
@ApiOperation({
|
||||
operationId: 'createSession',
|
||||
summary: 'Вход',
|
||||
})
|
||||
@ApiCreatedResponse({ type: Session })
|
||||
@ApiUnauthorizedResponse({
|
||||
headers: {
|
||||
'WWW-Authenticate': {
|
||||
schema: { type: 'string' },
|
||||
example: 'Basic realm="messenger", charset="UTF-8"',
|
||||
},
|
||||
},
|
||||
})
|
||||
create(@Req() request: AuthenticatedRequest, @Body() body: unknown) {
|
||||
return this.auth.login(request.headers.authorization, body);
|
||||
}
|
||||
|
||||
@Delete('current')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@UseGuards(UserGuard)
|
||||
@ApiBearerAuth('bearerAuth')
|
||||
@ApiOperation({
|
||||
operationId: 'revokeSession',
|
||||
summary: 'Отзыв текущей сессии',
|
||||
})
|
||||
@ApiNoContentResponse()
|
||||
@ApiUnauthorizedResponse({
|
||||
headers: {
|
||||
'WWW-Authenticate': {
|
||||
schema: { type: 'string' },
|
||||
example: 'Bearer realm="messenger"',
|
||||
},
|
||||
},
|
||||
})
|
||||
logout(@Req() request: AuthenticatedRequest) {
|
||||
return this.auth.logout(request.currentSessionId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { HttpStatus } from '@nestjs/common';
|
||||
import { ApiException } from '../common/exceptions/api.exception';
|
||||
|
||||
export class AuthException extends ApiException {
|
||||
readonly challenge: string;
|
||||
|
||||
constructor(scheme: 'Basic' | 'Bearer') {
|
||||
super(
|
||||
HttpStatus.UNAUTHORIZED,
|
||||
'unauthorized',
|
||||
'Требуется действительная аутентификация',
|
||||
);
|
||||
this.challenge =
|
||||
scheme === 'Basic'
|
||||
? 'Basic realm="messenger", charset="UTF-8"'
|
||||
: 'Bearer realm="messenger"';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { User } from '../User/user.schema';
|
||||
|
||||
export class Session {
|
||||
token!: string;
|
||||
|
||||
@ApiProperty({ enum: ['Bearer'] })
|
||||
token_type!: 'Bearer';
|
||||
|
||||
@ApiProperty({ format: 'date-time' })
|
||||
expires_at!: string;
|
||||
|
||||
@ApiProperty({ type: User })
|
||||
user!: User;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { TextDecoder } from 'node:util';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { Prisma } from '../generated/prisma/client';
|
||||
import { UserService } from '../User/user.service';
|
||||
import type { User } from '../User/user.schema';
|
||||
import { invalidRequest, uuid, uuidPattern } from '../common/validation';
|
||||
import { ApiException } from '../common/exceptions/api.exception';
|
||||
import { AuthException } from './auth.exception';
|
||||
import { PasswordService, validPassword } from './password.service';
|
||||
import type { Session } from './auth.schema';
|
||||
|
||||
export interface ChatAccess {
|
||||
chatId: string;
|
||||
userId: string;
|
||||
messagesEpoch: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
private readonly ttlSeconds: number;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly users: UserService,
|
||||
private readonly passwords: PasswordService,
|
||||
) {
|
||||
this.ttlSeconds = Number(process.env.SESSION_TTL_SECONDS ?? 3600);
|
||||
if (
|
||||
!Number.isSafeInteger(this.ttlSeconds) ||
|
||||
this.ttlSeconds <= 0 ||
|
||||
Number.isNaN(new Date(Date.now() + this.ttlSeconds * 1000).getTime())
|
||||
) {
|
||||
throw new Error(
|
||||
'SESSION_TTL_SECONDS must be a positive integer representing a valid expiry',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async login(authorization: unknown, body: unknown): Promise<Session> {
|
||||
const { username, password } = this.basicCredentials(authorization);
|
||||
const user = await this.prisma.user.findUnique({ where: { username } });
|
||||
const verified = await this.passwords.verify(
|
||||
password,
|
||||
user?.passwordHash ?? null,
|
||||
);
|
||||
if (!user || !verified) throw new AuthException('Basic');
|
||||
if (body !== undefined)
|
||||
return invalidRequest('Вход через Basic не принимает тело запроса');
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const session = await this.prisma.session.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
tokenHash: this.tokenHash(token),
|
||||
expiresAt: new Date(Date.now() + this.ttlSeconds * 1000),
|
||||
},
|
||||
});
|
||||
return {
|
||||
token,
|
||||
token_type: 'Bearer',
|
||||
expires_at: session.expiresAt.toISOString(),
|
||||
user: this.users.serialize(user),
|
||||
};
|
||||
}
|
||||
|
||||
async authenticate(
|
||||
authorization: unknown,
|
||||
chatId?: unknown,
|
||||
): Promise<{
|
||||
user: User;
|
||||
sessionId: string;
|
||||
chatsEpoch: string;
|
||||
chatAccess?: ChatAccess;
|
||||
}> {
|
||||
const match =
|
||||
typeof authorization === 'string' &&
|
||||
/^Bearer +([A-Za-z0-9_-]{43})$/i.exec(authorization);
|
||||
if (!match) throw new AuthException('Bearer');
|
||||
const requestedChat =
|
||||
typeof chatId === 'string' && uuidPattern.test(chatId)
|
||||
? chatId.toLowerCase()
|
||||
: null;
|
||||
const [session] = await this.prisma.$queryRaw<
|
||||
{
|
||||
id: string;
|
||||
userId: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
createdAt: Date;
|
||||
chatsEpoch: bigint;
|
||||
chatId: string | null;
|
||||
messagesEpoch: bigint | null;
|
||||
}[]
|
||||
>(Prisma.sql`
|
||||
SELECT s.id, s.user_id AS "userId", u.chats_epoch AS "chatsEpoch",
|
||||
u.username, u.display_name AS "displayName", u.created_at AS "createdAt",
|
||||
${
|
||||
chatId === undefined
|
||||
? Prisma.sql`NULL::uuid AS "chatId", NULL::bigint AS "messagesEpoch"`
|
||||
: Prisma.sql`c.id AS "chatId", COALESCE((SELECT SUM(e.epoch) FROM "public"."chat_message_epochs" AS e WHERE e.chat_id = c.id), 0)::bigint AS "messagesEpoch"`
|
||||
}
|
||||
FROM sessions AS s
|
||||
JOIN users AS u ON u.id = s.user_id
|
||||
${
|
||||
chatId === undefined
|
||||
? Prisma.empty
|
||||
: Prisma.sql`
|
||||
LEFT JOIN chat_members AS cm
|
||||
ON cm.user_id = s.user_id AND cm.chat_id = ${requestedChat}::uuid
|
||||
LEFT JOIN chats AS c ON c.id = cm.chat_id
|
||||
`
|
||||
}
|
||||
WHERE s.token_hash = ${this.tokenHash(match[1])}
|
||||
AND s.revoked_at IS NULL AND s.expires_at > ${new Date()}
|
||||
LIMIT 1
|
||||
`);
|
||||
if (!session) throw new AuthException('Bearer');
|
||||
if (chatId !== undefined) {
|
||||
uuid(chatId, 'chat_id');
|
||||
if (session.chatId === null || session.messagesEpoch === null)
|
||||
throw new ApiException(404, 'not_found', 'Объект не найден');
|
||||
}
|
||||
return {
|
||||
user: this.users.serialize({ ...session, id: session.userId }),
|
||||
sessionId: session.id,
|
||||
chatsEpoch: session.chatsEpoch.toString(),
|
||||
chatAccess:
|
||||
session.chatId !== null && session.messagesEpoch !== null
|
||||
? {
|
||||
chatId: session.chatId,
|
||||
userId: session.userId,
|
||||
messagesEpoch: session.messagesEpoch.toString(),
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async logout(sessionId: string): Promise<void> {
|
||||
const now = new Date();
|
||||
const result = await this.prisma.session.updateMany({
|
||||
where: { id: sessionId, revokedAt: null, expiresAt: { gt: now } },
|
||||
data: { revokedAt: now },
|
||||
});
|
||||
if (result.count !== 1) throw new AuthException('Bearer');
|
||||
}
|
||||
|
||||
private tokenHash(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
private basicCredentials(authorization: unknown): {
|
||||
username: string;
|
||||
password: string;
|
||||
} {
|
||||
const match =
|
||||
typeof authorization === 'string' &&
|
||||
/^Basic +([A-Za-z0-9+/]+={0,2})$/i.exec(authorization);
|
||||
if (!match) throw new AuthException('Basic');
|
||||
try {
|
||||
const bytes = Buffer.from(match[1], 'base64');
|
||||
if (
|
||||
bytes.toString('base64').replace(/=+$/, '') !==
|
||||
match[1].replace(/=+$/, '')
|
||||
)
|
||||
throw new AuthException('Basic');
|
||||
const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
const colon = value.indexOf(':');
|
||||
const username = value.slice(0, colon);
|
||||
const password = value.slice(colon + 1);
|
||||
if (
|
||||
colon < 0 ||
|
||||
!/^[a-z0-9_]{3,32}$/.test(username) ||
|
||||
!validPassword(password)
|
||||
)
|
||||
throw new AuthException('Basic');
|
||||
return { username, password };
|
||||
} catch {
|
||||
throw new AuthException('Basic');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { argon2, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import { invalidRequest } from '../common/validation';
|
||||
|
||||
const argonParameters = {
|
||||
memory: 19456,
|
||||
passes: 2,
|
||||
parallelism: 1,
|
||||
tagLength: 32,
|
||||
};
|
||||
const argonPrefix = '$argon2id$v=19$m=19456,t=2,p=1';
|
||||
|
||||
function decodeBase64(value: string, length: number) {
|
||||
const decoded = Buffer.from(value, 'base64');
|
||||
return decoded.length === length &&
|
||||
decoded.toString('base64').replace(/=+$/, '') === value
|
||||
? decoded
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function argonHash(value: string | null) {
|
||||
const parts = value?.split('$');
|
||||
if (parts?.length !== 6 || parts.slice(0, 4).join('$') !== argonPrefix)
|
||||
return undefined;
|
||||
const salt = decodeBase64(parts[4], 16);
|
||||
const key = decodeBase64(parts[5], 32);
|
||||
return salt && key ? { salt, key } : undefined;
|
||||
}
|
||||
|
||||
export function validPassword(value: unknown): value is string {
|
||||
if (typeof value !== 'string' || value.trim().length === 0) return false;
|
||||
const length = Array.from(value).length;
|
||||
return length >= 12 && length <= 128;
|
||||
}
|
||||
|
||||
export function registrationPassword(value: unknown): string {
|
||||
if (!validPassword(value))
|
||||
return invalidRequest('Пароль должен содержать 12–128 Unicode-символов');
|
||||
return value;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PasswordService {
|
||||
async hash(password: string): Promise<string> {
|
||||
const salt = randomBytes(16);
|
||||
const key = await this.deriveArgon(password, salt);
|
||||
return `${argonPrefix}$${salt.toString('base64').replace(/=+$/, '')}$${key.toString('base64').replace(/=+$/, '')}`;
|
||||
}
|
||||
|
||||
async verify(password: string, storedHash: string | null): Promise<boolean> {
|
||||
const stored = argonHash(storedHash);
|
||||
const key = await this.deriveArgon(
|
||||
password,
|
||||
stored?.salt ?? Buffer.alloc(16),
|
||||
);
|
||||
const matches = timingSafeEqual(key, stored?.key ?? Buffer.alloc(32));
|
||||
return Boolean(stored && matches);
|
||||
}
|
||||
|
||||
private deriveArgon(password: string, salt: Buffer): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
argon2(
|
||||
'argon2id',
|
||||
{ ...argonParameters, message: password, nonce: salt },
|
||||
(error, key) => {
|
||||
if (error) reject(error);
|
||||
else resolve(key);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
Headers,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiBody,
|
||||
ApiBearerAuth,
|
||||
ApiCreatedResponse,
|
||||
ApiHeader,
|
||||
ApiOkResponse,
|
||||
ApiOperation,
|
||||
ApiQuery,
|
||||
ApiTags,
|
||||
} from '@nestjs/swagger';
|
||||
import { UserGuard, type AuthenticatedRequest } from '../User/user.guard';
|
||||
import {
|
||||
CreateChatDto,
|
||||
CreateMessageDto,
|
||||
Chat,
|
||||
Message,
|
||||
ChatPage,
|
||||
MessagePage,
|
||||
} from './chat.schema';
|
||||
import { ChatService } from './chat.service';
|
||||
|
||||
@ApiTags('chats')
|
||||
@ApiBearerAuth('bearerAuth')
|
||||
@UseGuards(UserGuard)
|
||||
@Controller('api/v1/chats')
|
||||
export class ChatController {
|
||||
constructor(private readonly chats: ChatService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(HttpStatus.CREATED)
|
||||
@ApiBody({ type: CreateChatDto })
|
||||
@ApiOperation({ operationId: 'createChat' })
|
||||
@ApiCreatedResponse({ type: Chat })
|
||||
create(@Req() request: AuthenticatedRequest, @Body() body: unknown) {
|
||||
return this.chats.create(body, request.currentUser.id);
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ operationId: 'listChats' })
|
||||
@ApiOkResponse({ type: ChatPage })
|
||||
@ApiQuery({
|
||||
name: 'limit',
|
||||
required: false,
|
||||
schema: { type: 'integer', minimum: 1, maximum: 100, default: 50 },
|
||||
})
|
||||
@ApiQuery({ name: 'cursor', required: false, type: String })
|
||||
list(
|
||||
@Req() request: AuthenticatedRequest,
|
||||
@Query('limit') limit: unknown,
|
||||
@Query('cursor') cursor: unknown,
|
||||
) {
|
||||
return this.chats.list(
|
||||
request.currentUser.id,
|
||||
limit,
|
||||
cursor,
|
||||
request.currentChatsEpoch,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':chat_id')
|
||||
@ApiOperation({ operationId: 'getChat' })
|
||||
@ApiOkResponse({ type: Chat })
|
||||
get(@Req() request: AuthenticatedRequest, @Param('chat_id') chatId: string) {
|
||||
return this.chats.get(chatId, request.currentUser.id);
|
||||
}
|
||||
|
||||
@Post(':chat_id/messages')
|
||||
@HttpCode(HttpStatus.CREATED)
|
||||
@ApiBody({ type: CreateMessageDto })
|
||||
@ApiHeader({
|
||||
name: 'Idempotency-Key',
|
||||
required: true,
|
||||
schema: {
|
||||
type: 'string',
|
||||
minLength: 1,
|
||||
maxLength: 128,
|
||||
pattern: '^[A-Za-z0-9._:-]+$',
|
||||
},
|
||||
})
|
||||
@ApiOperation({ operationId: 'createMessage' })
|
||||
@ApiCreatedResponse({ type: Message })
|
||||
createMessage(
|
||||
@Req() request: AuthenticatedRequest,
|
||||
@Param('chat_id') chatId: string,
|
||||
@Headers('idempotency-key') idempotencyKey: unknown,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
return this.chats.createMessage(
|
||||
chatId,
|
||||
body,
|
||||
request.currentUser.id,
|
||||
idempotencyKey,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':chat_id/messages')
|
||||
@ApiOperation({ operationId: 'listMessages' })
|
||||
@ApiOkResponse({ type: MessagePage })
|
||||
@ApiQuery({
|
||||
name: 'limit',
|
||||
required: false,
|
||||
schema: { type: 'integer', minimum: 1, maximum: 100, default: 50 },
|
||||
})
|
||||
@ApiQuery({ name: 'cursor', required: false, type: String })
|
||||
listMessages(
|
||||
@Req() request: AuthenticatedRequest,
|
||||
@Param('chat_id') chatId: string,
|
||||
@Query('limit') limit: unknown,
|
||||
@Query('cursor') cursor: unknown,
|
||||
) {
|
||||
return this.chats.listMessages(
|
||||
chatId,
|
||||
request.currentUser.id,
|
||||
limit,
|
||||
cursor,
|
||||
request.currentChatAccess,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { UserModule } from '../User/user.module';
|
||||
import { ChatController } from './chat.controller';
|
||||
import { ChatService } from './chat.service';
|
||||
|
||||
@Module({
|
||||
imports: [UserModule],
|
||||
controllers: [ChatController],
|
||||
providers: [ChatService],
|
||||
})
|
||||
export class ChatModule {}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
|
||||
export class CreateChatDto {
|
||||
@ApiProperty({ example: 'Проект по ВНП', maxLength: 100 })
|
||||
title!: string;
|
||||
|
||||
@ApiProperty({ type: [String], format: 'uuid' })
|
||||
member_ids!: string[];
|
||||
}
|
||||
|
||||
export class CreateMessageDto {
|
||||
@ApiProperty({ example: 'Привет, Боб!', maxLength: 4000 })
|
||||
text!: string;
|
||||
}
|
||||
|
||||
export class Chat {
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
id!: string;
|
||||
@ApiProperty({ maxLength: 100 })
|
||||
title!: string;
|
||||
@ApiProperty({ type: [String], format: 'uuid' })
|
||||
member_ids!: string[];
|
||||
@ApiProperty({ format: 'date-time' })
|
||||
created_at!: string;
|
||||
}
|
||||
|
||||
export class Message {
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
id!: string;
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
chat_id!: string;
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
sender_id!: string;
|
||||
@ApiProperty({ maxLength: 4000 })
|
||||
text!: string;
|
||||
@ApiProperty({ format: 'date-time' })
|
||||
created_at!: string;
|
||||
}
|
||||
|
||||
export class ChatPage {
|
||||
@ApiProperty({ type: [Chat] })
|
||||
items!: Chat[];
|
||||
@ApiProperty({ type: String, nullable: true })
|
||||
next_cursor!: string | null;
|
||||
}
|
||||
|
||||
export class MessagePage {
|
||||
@ApiProperty({ type: [Message] })
|
||||
items!: Message[];
|
||||
@ApiProperty({ type: String, nullable: true })
|
||||
next_cursor!: string | null;
|
||||
}
|
||||
|
||||
export interface Page<T> {
|
||||
items: T[];
|
||||
next_cursor: string | null;
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
import { HttpStatus, Injectable } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { ChatAccess } from '../Auth/auth.service';
|
||||
import {
|
||||
Prisma,
|
||||
type Message as StoredMessage,
|
||||
} from '../generated/prisma/client';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { ApiException } from '../common/exceptions/api.exception';
|
||||
import {
|
||||
invalidRequest,
|
||||
objectBody,
|
||||
pageLimit,
|
||||
requiredText,
|
||||
uuid,
|
||||
} from '../common/validation';
|
||||
import type { Chat, Message, Page } from './chat.schema';
|
||||
import { CacheService, CACHE_TTL } from '../cache/cache.service';
|
||||
|
||||
interface Cursor {
|
||||
scope: string;
|
||||
created_at: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
const chatInclude = {
|
||||
members: { orderBy: { position: 'asc' as const } },
|
||||
} satisfies Prisma.ChatInclude;
|
||||
type StoredChat = Prisma.ChatGetPayload<{ include: typeof chatInclude }>;
|
||||
|
||||
@Injectable()
|
||||
export class ChatService {
|
||||
private readonly idempotencyTtlSeconds: number;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly cache: CacheService,
|
||||
) {
|
||||
this.idempotencyTtlSeconds = Number(
|
||||
process.env.IDEMPOTENCY_TTL_SECONDS ?? 86400,
|
||||
);
|
||||
if (
|
||||
!Number.isSafeInteger(this.idempotencyTtlSeconds) ||
|
||||
this.idempotencyTtlSeconds < 86400 ||
|
||||
Number.isNaN(
|
||||
new Date(Date.now() + this.idempotencyTtlSeconds * 1000).getTime(),
|
||||
)
|
||||
) {
|
||||
throw new Error('IDEMPOTENCY_TTL_SECONDS must be at least 86400');
|
||||
}
|
||||
}
|
||||
|
||||
async create(value: unknown, userId: string): Promise<Chat> {
|
||||
const body = objectBody(value, ['title', 'member_ids']);
|
||||
const title = requiredText(body.title, 'title', 100);
|
||||
if (
|
||||
!Array.isArray(body.member_ids) ||
|
||||
body.member_ids.length < 1 ||
|
||||
body.member_ids.length > 100
|
||||
) {
|
||||
return invalidRequest('Некорректный member_ids');
|
||||
}
|
||||
const memberIds = body.member_ids.map((id: unknown) =>
|
||||
uuid(id, 'member_ids'),
|
||||
);
|
||||
if (
|
||||
new Set(memberIds).size !== memberIds.length ||
|
||||
!memberIds.includes(userId)
|
||||
) {
|
||||
return invalidRequest(
|
||||
'Список участников содержит дубликат или не содержит вызывающего',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const chat = await this.prisma.$transaction(async (tx) => {
|
||||
const count = await tx.user.count({ where: { id: { in: memberIds } } });
|
||||
if (count !== memberIds.length)
|
||||
return invalidRequest('Неизвестный участник чата');
|
||||
const created = await tx.chat.create({
|
||||
data: {
|
||||
title,
|
||||
members: {
|
||||
create: memberIds.map((userId, position) => ({
|
||||
userId,
|
||||
position,
|
||||
})),
|
||||
},
|
||||
},
|
||||
include: chatInclude,
|
||||
});
|
||||
await tx.$executeRaw(Prisma.sql`
|
||||
WITH locked AS MATERIALIZED (
|
||||
SELECT id FROM users
|
||||
WHERE id IN (${Prisma.join(memberIds.map((id) => Prisma.sql`${id}::uuid`))})
|
||||
ORDER BY id FOR UPDATE
|
||||
)
|
||||
UPDATE users AS u SET chats_epoch = u.chats_epoch + 1
|
||||
FROM locked WHERE u.id = locked.id
|
||||
`);
|
||||
return created;
|
||||
});
|
||||
return this.serializeChat(chat);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2003'
|
||||
) {
|
||||
return invalidRequest('Неизвестный участник чата');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
userId: string,
|
||||
limitValue: unknown,
|
||||
cursorValue: unknown,
|
||||
epoch: string,
|
||||
): Promise<Page<Chat>> {
|
||||
const limit = pageLimit(limitValue);
|
||||
const scope = `chats:${userId}`;
|
||||
const cursor = this.decodeCursor(cursorValue, scope);
|
||||
return this.cache.remember(
|
||||
scope,
|
||||
{ limit, cursor },
|
||||
CACHE_TTL.page,
|
||||
async () => {
|
||||
const chats = await this.prisma.chat.findMany({
|
||||
where: { members: { some: { userId } }, ...this.after(cursor) },
|
||||
include: chatInclude,
|
||||
orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
|
||||
take: limit + 1,
|
||||
});
|
||||
return this.page(
|
||||
chats.map((chat) => this.serializeChat(chat)),
|
||||
limit,
|
||||
scope,
|
||||
);
|
||||
},
|
||||
epoch,
|
||||
);
|
||||
}
|
||||
|
||||
async get(value: unknown, userId: string): Promise<Chat> {
|
||||
const { id } = await this.authorizeChat(value, userId);
|
||||
return this.getMetadata(id);
|
||||
}
|
||||
|
||||
private async authorizeChat(value: unknown, userId: string) {
|
||||
const id = uuid(value, 'chat_id');
|
||||
const [membership] = await this.prisma.$queryRaw<
|
||||
{ id: string; messagesEpoch: bigint }[]
|
||||
>(Prisma.sql`
|
||||
SELECT c.id, COALESCE((SELECT SUM(e.epoch) FROM "public"."chat_message_epochs" AS e WHERE e.chat_id = c.id), 0)::bigint AS "messagesEpoch"
|
||||
FROM chat_members AS cm JOIN chats AS c ON c.id = cm.chat_id
|
||||
WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid
|
||||
`);
|
||||
if (!membership) {
|
||||
throw new ApiException(
|
||||
HttpStatus.NOT_FOUND,
|
||||
'not_found',
|
||||
'Объект не найден',
|
||||
);
|
||||
}
|
||||
return membership;
|
||||
}
|
||||
|
||||
private getMetadata(id: string): Promise<Chat> {
|
||||
return this.cache.remember(
|
||||
`chat:${id}`,
|
||||
'metadata',
|
||||
CACHE_TTL.entity,
|
||||
async () => {
|
||||
const chat = await this.prisma.chat.findUnique({
|
||||
where: { id },
|
||||
include: chatInclude,
|
||||
});
|
||||
if (!chat)
|
||||
throw new ApiException(
|
||||
HttpStatus.NOT_FOUND,
|
||||
'not_found',
|
||||
'Объект не найден',
|
||||
);
|
||||
return this.serializeChat(chat);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async createMessage(
|
||||
chatId: unknown,
|
||||
value: unknown,
|
||||
userId: string,
|
||||
keyValue: unknown,
|
||||
): Promise<Message> {
|
||||
const id = uuid(chatId, 'chat_id');
|
||||
if (
|
||||
typeof keyValue !== 'string' ||
|
||||
!/^[A-Za-z0-9._:-]{1,128}$/.test(keyValue)
|
||||
) {
|
||||
return invalidRequest('Некорректный Idempotency-Key');
|
||||
}
|
||||
const body = objectBody(value, ['text']);
|
||||
const text = requiredText(body.text, 'text', 4000);
|
||||
await this.authorizeChat(id, userId);
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
await tx.$executeRaw(Prisma.sql`
|
||||
DELETE FROM message_idempotency
|
||||
WHERE user_id = ${userId}::uuid AND chat_id = ${id}::uuid
|
||||
AND "key" = ${keyValue} AND expires_at <= NOW()
|
||||
`);
|
||||
const claimed = await tx.$queryRaw<{ key: string }[]>(Prisma.sql`
|
||||
INSERT INTO message_idempotency (user_id, chat_id, "key", expires_at)
|
||||
VALUES (
|
||||
${userId}::uuid, ${id}::uuid, ${keyValue},
|
||||
NOW() + (${this.idempotencyTtlSeconds} * INTERVAL '1 second')
|
||||
)
|
||||
ON CONFLICT (user_id, chat_id, "key") DO NOTHING
|
||||
RETURNING "key"
|
||||
`);
|
||||
if (claimed.length === 0) {
|
||||
const [existing] = await tx.$queryRaw<StoredMessage[]>(Prisma.sql`
|
||||
SELECT m.id, m.chat_id AS "chatId", m.sender_id AS "senderId",
|
||||
m.text, m.created_at AS "createdAt"
|
||||
FROM message_idempotency AS i
|
||||
JOIN messages AS m ON m.id = i.message_id
|
||||
WHERE i.user_id = ${userId}::uuid AND i.chat_id = ${id}::uuid
|
||||
AND i."key" = ${keyValue}
|
||||
`);
|
||||
if (!existing) throw new Error('Idempotency record has no message');
|
||||
if (existing.text !== text) {
|
||||
throw new ApiException(
|
||||
HttpStatus.CONFLICT,
|
||||
'conflict',
|
||||
'Idempotency-Key уже использован с другим сообщением',
|
||||
);
|
||||
}
|
||||
return this.serializeMessage(existing);
|
||||
}
|
||||
|
||||
const messageId = randomUUID();
|
||||
const bucket = parseInt(messageId.slice(0, 2), 16) % 16;
|
||||
const [message] = await tx.$queryRaw<StoredMessage[]>(Prisma.sql`
|
||||
WITH updated AS (
|
||||
INSERT INTO "public"."chat_message_epochs" AS e (chat_id, bucket, epoch)
|
||||
SELECT cm.chat_id, ${bucket}::smallint, 1 FROM chat_members AS cm
|
||||
WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid
|
||||
ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = e.epoch + 1
|
||||
RETURNING chat_id
|
||||
)
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
SELECT ${messageId}::uuid, updated.chat_id, ${userId}::uuid, ${text} FROM updated
|
||||
RETURNING id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt"
|
||||
`);
|
||||
if (!message)
|
||||
throw new ApiException(
|
||||
HttpStatus.NOT_FOUND,
|
||||
'not_found',
|
||||
'Объект не найден',
|
||||
);
|
||||
await tx.$executeRaw(Prisma.sql`
|
||||
UPDATE message_idempotency SET message_id = ${message.id}::uuid
|
||||
WHERE user_id = ${userId}::uuid AND chat_id = ${id}::uuid
|
||||
AND "key" = ${keyValue}
|
||||
`);
|
||||
return this.serializeMessage(message);
|
||||
});
|
||||
}
|
||||
|
||||
async listMessages(
|
||||
chatId: unknown,
|
||||
userId: string,
|
||||
limitValue: unknown,
|
||||
cursorValue: unknown,
|
||||
access?: ChatAccess,
|
||||
): Promise<Page<Message>> {
|
||||
const id = uuid(chatId, 'chat_id');
|
||||
const chat =
|
||||
access?.chatId === id && access.userId === userId
|
||||
? { id, messagesEpoch: access.messagesEpoch }
|
||||
: await this.authorizeChat(id, userId);
|
||||
const limit = pageLimit(limitValue);
|
||||
const scope = `messages:${chat.id}`;
|
||||
const cursor = this.decodeCursor(cursorValue, scope);
|
||||
return this.cache.remember(
|
||||
scope,
|
||||
{ limit, cursor },
|
||||
CACHE_TTL.page,
|
||||
async () => {
|
||||
const messages = cursor
|
||||
? await this.prisma.$queryRaw<StoredMessage[]>(Prisma.sql`
|
||||
SELECT id, chat_id AS "chatId", sender_id AS "senderId",
|
||||
text, created_at AS "createdAt"
|
||||
FROM "public"."messages"
|
||||
WHERE chat_id = ${chat.id}::uuid
|
||||
AND (created_at, id) >
|
||||
(${new Date(cursor.created_at)}::timestamptz, ${cursor.id}::uuid)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ${limit + 1}
|
||||
`)
|
||||
: await this.prisma.message.findMany({
|
||||
where: { chatId: chat.id },
|
||||
orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
|
||||
take: limit + 1,
|
||||
});
|
||||
return this.page(
|
||||
messages.map((message) => this.serializeMessage(message)),
|
||||
limit,
|
||||
scope,
|
||||
);
|
||||
},
|
||||
chat.messagesEpoch.toString(),
|
||||
);
|
||||
}
|
||||
|
||||
private after(cursor: Cursor | null) {
|
||||
if (!cursor) return {};
|
||||
const createdAt = new Date(cursor.created_at);
|
||||
return {
|
||||
OR: [
|
||||
{ createdAt: { gt: createdAt } },
|
||||
{ createdAt, id: { gt: cursor.id } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
private serializeChat(chat: StoredChat): Chat {
|
||||
return {
|
||||
id: chat.id,
|
||||
title: chat.title,
|
||||
member_ids: chat.members.map((member) => member.userId),
|
||||
created_at: chat.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private serializeMessage(message: StoredMessage): Message {
|
||||
return {
|
||||
id: message.id,
|
||||
chat_id: message.chatId,
|
||||
sender_id: message.senderId,
|
||||
text: message.text,
|
||||
created_at: message.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private page<T extends { id: string; created_at: string }>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
scope: string,
|
||||
): Page<T> {
|
||||
const result = items.slice(0, limit);
|
||||
const last = result.at(-1);
|
||||
return {
|
||||
items: result,
|
||||
next_cursor:
|
||||
items.length > limit && last
|
||||
? Buffer.from(
|
||||
JSON.stringify({
|
||||
scope,
|
||||
created_at: last.created_at,
|
||||
id: last.id,
|
||||
}),
|
||||
).toString('base64url')
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
private decodeCursor(value: unknown, scope: string): Cursor | null {
|
||||
if (value === undefined) return null;
|
||||
if (typeof value !== 'string' || !/^[A-Za-z0-9_-]+$/.test(value))
|
||||
return invalidRequest('Некорректный cursor');
|
||||
try {
|
||||
const cursor: unknown = JSON.parse(
|
||||
Buffer.from(value, 'base64url').toString('utf8'),
|
||||
);
|
||||
if (
|
||||
typeof cursor !== 'object' ||
|
||||
cursor === null ||
|
||||
Array.isArray(cursor)
|
||||
)
|
||||
return invalidRequest('Некорректный cursor');
|
||||
const data = cursor as Record<string, unknown>;
|
||||
if (
|
||||
data.scope !== scope ||
|
||||
typeof data.created_at !== 'string' ||
|
||||
Number.isNaN(Date.parse(data.created_at)) ||
|
||||
typeof data.id !== 'string'
|
||||
) {
|
||||
return invalidRequest('Некорректный cursor');
|
||||
}
|
||||
return {
|
||||
scope,
|
||||
created_at: data.created_at,
|
||||
id: uuid(data.id, 'cursor'),
|
||||
};
|
||||
} catch {
|
||||
return invalidRequest('Некорректный cursor');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Post,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiBody,
|
||||
ApiBearerAuth,
|
||||
ApiCreatedResponse,
|
||||
ApiOkResponse,
|
||||
ApiOperation,
|
||||
ApiTags,
|
||||
} from '@nestjs/swagger';
|
||||
import { UserGuard, type AuthenticatedRequest } from './user.guard';
|
||||
import { CreateUserDto, User } from './user.schema';
|
||||
import { UserService } from './user.service';
|
||||
|
||||
@ApiTags('users')
|
||||
@Controller('api/v1/users')
|
||||
export class UserController {
|
||||
constructor(private readonly users: UserService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(HttpStatus.CREATED)
|
||||
@ApiBody({ type: CreateUserDto })
|
||||
@ApiOperation({ operationId: 'registerUser' })
|
||||
@ApiCreatedResponse({ type: User })
|
||||
create(@Body() body: unknown) {
|
||||
return this.users.create(body);
|
||||
}
|
||||
|
||||
@Get('me')
|
||||
@UseGuards(UserGuard)
|
||||
@ApiBearerAuth('bearerAuth')
|
||||
@ApiOperation({ operationId: 'currentUser' })
|
||||
@ApiOkResponse({ type: User })
|
||||
me(@Req() request: AuthenticatedRequest) {
|
||||
return request.currentUser;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
|
||||
import type { FastifyRequest } from 'fastify';
|
||||
import { AuthService, type ChatAccess } from '../Auth/auth.service';
|
||||
import type { User } from './user.schema';
|
||||
|
||||
export type AuthenticatedRequest = FastifyRequest & {
|
||||
currentUser: User;
|
||||
currentSessionId: string;
|
||||
currentChatsEpoch: string;
|
||||
currentChatAccess?: ChatAccess;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class UserGuard implements CanActivate {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest<AuthenticatedRequest>();
|
||||
const chatId =
|
||||
request.method === 'GET' &&
|
||||
request.routeOptions.url === '/api/v1/chats/:chat_id/messages'
|
||||
? (request.params as { chat_id: string }).chat_id
|
||||
: undefined;
|
||||
const session = await this.auth.authenticate(
|
||||
request.headers.authorization,
|
||||
chatId,
|
||||
);
|
||||
request.currentUser = session.user;
|
||||
request.currentSessionId = session.sessionId;
|
||||
request.currentChatsEpoch = session.chatsEpoch;
|
||||
request.currentChatAccess = session.chatAccess;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { UserController } from './user.controller';
|
||||
import { UserGuard } from './user.guard';
|
||||
import { UserService } from './user.service';
|
||||
import { AuthController } from '../Auth/auth.controller';
|
||||
import { AuthService } from '../Auth/auth.service';
|
||||
import { PasswordService } from '../Auth/password.service';
|
||||
|
||||
@Module({
|
||||
controllers: [UserController, AuthController],
|
||||
providers: [UserService, UserGuard, AuthService, PasswordService],
|
||||
exports: [UserService, UserGuard, AuthService],
|
||||
})
|
||||
export class UserModule {}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
|
||||
export class CreateUserDto {
|
||||
@ApiProperty({ example: 'alice', pattern: '^[a-z0-9_]{3,32}$' })
|
||||
username!: string;
|
||||
|
||||
@ApiProperty({ example: 'Алиса', maxLength: 100 })
|
||||
display_name!: string;
|
||||
|
||||
@ApiProperty({
|
||||
writeOnly: true,
|
||||
minLength: 12,
|
||||
maxLength: 128,
|
||||
example: 'Example-only-not-a-real-secret-123',
|
||||
})
|
||||
password!: string;
|
||||
}
|
||||
|
||||
export class User {
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
id!: string;
|
||||
@ApiProperty({ pattern: '^[a-z0-9_]{3,32}$' })
|
||||
username!: string;
|
||||
@ApiProperty({ minLength: 1, maxLength: 100 })
|
||||
display_name!: string;
|
||||
@ApiProperty({ format: 'date-time' })
|
||||
created_at!: string;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { HttpStatus, Injectable } from '@nestjs/common';
|
||||
import { Prisma, type User as StoredUser } from '../generated/prisma/client';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { ApiException } from '../common/exceptions/api.exception';
|
||||
import { objectBody, requiredText } from '../common/validation';
|
||||
import {
|
||||
PasswordService,
|
||||
registrationPassword,
|
||||
} from '../Auth/password.service';
|
||||
import type { User } from './user.schema';
|
||||
import { CacheService, CACHE_TTL } from '../cache/cache.service';
|
||||
|
||||
@Injectable()
|
||||
export class UserService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly passwords: PasswordService,
|
||||
private readonly cache: CacheService,
|
||||
) {}
|
||||
|
||||
async create(value: unknown): Promise<User> {
|
||||
const body = objectBody(value, ['username', 'display_name', 'password']);
|
||||
const username = body.username;
|
||||
if (typeof username !== 'string' || !/^[a-z0-9_]{3,32}$/.test(username)) {
|
||||
throw new ApiException(
|
||||
HttpStatus.BAD_REQUEST,
|
||||
'invalid_request',
|
||||
'Некорректный username',
|
||||
);
|
||||
}
|
||||
const displayName = requiredText(body.display_name, 'display_name', 100);
|
||||
const password = registrationPassword(body.password);
|
||||
const passwordHash = await this.passwords.hash(password);
|
||||
try {
|
||||
return this.serialize(
|
||||
await this.prisma.user.create({
|
||||
data: { username, displayName, passwordHash },
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2002'
|
||||
) {
|
||||
throw new ApiException(
|
||||
HttpStatus.CONFLICT,
|
||||
'conflict',
|
||||
'Username уже занят',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async get(id: string): Promise<User | undefined> {
|
||||
return this.cache.remember(
|
||||
`user:${id}`,
|
||||
'profile',
|
||||
CACHE_TTL.entity,
|
||||
async () => {
|
||||
const user = await this.prisma.user.findUnique({ where: { id } });
|
||||
return user ? this.serialize(user) : undefined;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
serialize(
|
||||
user: Pick<StoredUser, 'id' | 'username' | 'displayName' | 'createdAt'>,
|
||||
): User {
|
||||
return {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
display_name: user.displayName,
|
||||
created_at: user.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
import { PrismaService } from './prisma/prisma.service';
|
||||
|
||||
describe('AppController', () => {
|
||||
let appController: AppController;
|
||||
const queryRaw = jest.fn();
|
||||
|
||||
beforeEach(async () => {
|
||||
queryRaw.mockReset();
|
||||
const app: TestingModule = await Test.createTestingModule({
|
||||
controllers: [AppController],
|
||||
providers: [
|
||||
AppService,
|
||||
{ provide: PrismaService, useValue: { $queryRaw: queryRaw } },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
appController = app.get<AppController>(AppController);
|
||||
});
|
||||
|
||||
describe('root', () => {
|
||||
it('should return "Hello World!"', () => {
|
||||
expect(appController.getHello()).toBe('Hello World!');
|
||||
});
|
||||
});
|
||||
|
||||
it('reports ready when PostgreSQL is reachable', async () => {
|
||||
queryRaw.mockResolvedValue([{ '?column?': 1 }]);
|
||||
await expect(appController.ready()).resolves.toEqual({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('returns 503 when PostgreSQL is unavailable', async () => {
|
||||
queryRaw.mockRejectedValue(new Error('connection refused'));
|
||||
await expect(appController.ready()).rejects.toMatchObject({ status: 503 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Controller, Get, HttpStatus } from '@nestjs/common';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { AppService } from './app.service';
|
||||
import { PrismaService } from './prisma/prisma.service';
|
||||
import { ApiException } from './common/exceptions/api.exception';
|
||||
import { DrainingFastifyAdapter } from './common/http/fastify-adapter';
|
||||
|
||||
@Controller()
|
||||
export class AppController {
|
||||
constructor(
|
||||
private readonly appService: AppService,
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly adapterHost: HttpAdapterHost,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
getHello(): string {
|
||||
return this.appService.getHello();
|
||||
}
|
||||
|
||||
@Get('health/live')
|
||||
live() {
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
@Get('health/ready')
|
||||
async ready() {
|
||||
this.assertNotDraining();
|
||||
try {
|
||||
await this.prisma.$queryRaw`SELECT 1`;
|
||||
} catch {
|
||||
throw new ApiException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE,
|
||||
'unavailable',
|
||||
'База данных недоступна',
|
||||
);
|
||||
}
|
||||
this.assertNotDraining();
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
private assertNotDraining(): void {
|
||||
const adapter = this.adapterHost.httpAdapter;
|
||||
if (adapter instanceof DrainingFastifyAdapter && adapter.isDraining) {
|
||||
throw new ApiException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE,
|
||||
'unavailable',
|
||||
'API завершает работу',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { APP_FILTER } from '@nestjs/core';
|
||||
import { LoggerModule } from 'nestjs-pino';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
import { AllExceptionsFilter } from './common/filters/all-exceptions.filter';
|
||||
import { UserModule } from './User/user.module';
|
||||
import { ChatModule } from './Chat/chat.module';
|
||||
import { PrismaModule } from './prisma/prisma.module';
|
||||
import { CacheModule } from './cache/cache.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
PrismaModule,
|
||||
CacheModule,
|
||||
UserModule,
|
||||
ChatModule,
|
||||
LoggerModule.forRoot({
|
||||
pinoHttp: {
|
||||
level: process.env.NODE_ENV === 'production' ? 'info' : 'debug',
|
||||
|
||||
transport:
|
||||
process.env.NODE_ENV !== 'production'
|
||||
? {
|
||||
target: 'pino-pretty',
|
||||
options: {
|
||||
colorize: true,
|
||||
singleLine: true,
|
||||
translateTime: 'SYS:standard',
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
|
||||
redact: {
|
||||
paths: [
|
||||
'req.headers.authorization',
|
||||
'req.headers.cookie',
|
||||
'res.headers.set-cookie',
|
||||
],
|
||||
censor: '[REDACTED]',
|
||||
},
|
||||
|
||||
autoLogging: {
|
||||
ignore: (request) => request.url === '/health',
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
controllers: [AppController],
|
||||
providers: [
|
||||
AppService,
|
||||
{
|
||||
provide: APP_FILTER,
|
||||
useClass: AllExceptionsFilter,
|
||||
},
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class AppService {
|
||||
getHello(): string {
|
||||
return 'Hello World!';
|
||||
}
|
||||
}
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { CacheService } from './cache.service';
|
||||
|
||||
@Global()
|
||||
@Module({ providers: [CacheService], exports: [CacheService] })
|
||||
export class CacheModule {}
|
||||
Vendored
+149
@@ -0,0 +1,149 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
type OnModuleDestroy,
|
||||
type OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
export const CACHE_TTL = { entity: 300, page: 30 } as const;
|
||||
const generationTtl = 600;
|
||||
const storeIfCurrent = `
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
|
||||
end
|
||||
return false`;
|
||||
|
||||
@Injectable()
|
||||
export class CacheService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(CacheService.name);
|
||||
private readonly redis?: Redis;
|
||||
private readonly prefix = process.env.CACHE_PREFIX ?? 'messenger:cache:v1';
|
||||
private warned = false;
|
||||
|
||||
constructor() {
|
||||
if (process.env.CACHE_URL) {
|
||||
this.redis = new Redis(process.env.CACHE_URL, {
|
||||
lazyConnect: true,
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 300,
|
||||
maxRetriesPerRequest: 0,
|
||||
enableOfflineQueue: false,
|
||||
autoResendUnfulfilledCommands: false,
|
||||
retryStrategy: () => 1000,
|
||||
});
|
||||
this.redis.on('error', () => this.unavailable());
|
||||
this.redis.on('ready', () => {
|
||||
this.warned = false;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
await this.redis?.connect();
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
this.redis?.disconnect();
|
||||
}
|
||||
|
||||
async remember<T>(
|
||||
namespace: string,
|
||||
parameters: unknown,
|
||||
ttl: number,
|
||||
load: () => Promise<T>,
|
||||
epoch?: string,
|
||||
): Promise<T> {
|
||||
if (!this.redis || this.redis.status !== 'ready') return load();
|
||||
if (epoch !== undefined) {
|
||||
const digest = createHash('sha256')
|
||||
.update(JSON.stringify(parameters))
|
||||
.digest('hex');
|
||||
const key = `${this.prefix}:${namespace}:epoch:${epoch}:${digest}`;
|
||||
try {
|
||||
const value = await this.redis.get(key);
|
||||
if (value !== null) return JSON.parse(value) as T;
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
const result = await load();
|
||||
const serialized = JSON.stringify(result);
|
||||
if (serialized !== undefined && this.redis.status === 'ready') {
|
||||
try {
|
||||
await this.redis.set(key, serialized, 'EX', ttl);
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
const versionKey = `${this.prefix}:${namespace}:generation`;
|
||||
let version: string | null = null;
|
||||
let dataKey = '';
|
||||
try {
|
||||
await this.redis.set(versionKey, randomUUID(), 'EX', generationTtl, 'NX');
|
||||
version = await this.redis.get(versionKey);
|
||||
if (version) {
|
||||
const digest = createHash('sha256')
|
||||
.update(JSON.stringify(parameters))
|
||||
.digest('hex');
|
||||
dataKey = `${this.prefix}:${namespace}:${version}:${digest}`;
|
||||
const value = await this.redis.get(dataKey);
|
||||
if (value !== null) return JSON.parse(value) as T;
|
||||
}
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
|
||||
const result = await load();
|
||||
const serialized = JSON.stringify(result);
|
||||
if (version && serialized !== undefined && this.redis.status === 'ready') {
|
||||
try {
|
||||
await this.redis.eval(
|
||||
storeIfCurrent,
|
||||
2,
|
||||
versionKey,
|
||||
dataKey,
|
||||
version,
|
||||
serialized,
|
||||
ttl,
|
||||
);
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async invalidate(...namespaces: string[]): Promise<void> {
|
||||
if (!this.redis || this.redis.status !== 'ready' || namespaces.length === 0)
|
||||
return;
|
||||
try {
|
||||
const transaction = this.redis.multi();
|
||||
for (const namespace of new Set(namespaces)) {
|
||||
transaction.set(
|
||||
`${this.prefix}:${namespace}:generation`,
|
||||
randomUUID(),
|
||||
'EX',
|
||||
generationTtl,
|
||||
);
|
||||
}
|
||||
const results = await transaction.exec();
|
||||
if (results?.some(([error]) => error !== null)) this.unavailable();
|
||||
} catch {
|
||||
this.unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
private unavailable(): void {
|
||||
if (!this.warned) {
|
||||
this.logger.warn('Redis cache unavailable');
|
||||
this.warned = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import {
|
||||
HttpException,
|
||||
HttpExceptionOptions,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
|
||||
export class ApiException extends HttpException {
|
||||
constructor(
|
||||
status: HttpStatus,
|
||||
code: string,
|
||||
message: string,
|
||||
options?: HttpExceptionOptions,
|
||||
) {
|
||||
super({ code, message }, status, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import {
|
||||
ArgumentsHost,
|
||||
Catch,
|
||||
ExceptionFilter,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { Prisma } from '../../generated/prisma/client';
|
||||
import { ApiException } from '../exceptions/api.exception';
|
||||
import { AuthException } from '../../Auth/auth.exception';
|
||||
|
||||
interface ExceptionPayload {
|
||||
code?: unknown;
|
||||
}
|
||||
|
||||
const defaultCodes: Partial<Record<HttpStatus, string>> = {
|
||||
[HttpStatus.BAD_REQUEST]: 'invalid_request',
|
||||
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
|
||||
[HttpStatus.FORBIDDEN]: 'forbidden',
|
||||
[HttpStatus.NOT_FOUND]: 'not_found',
|
||||
[HttpStatus.CONFLICT]: 'conflict',
|
||||
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
|
||||
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
|
||||
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
|
||||
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
|
||||
};
|
||||
|
||||
const defaultMessages: Partial<Record<HttpStatus, string>> = {
|
||||
[HttpStatus.BAD_REQUEST]: 'Некорректный запрос',
|
||||
[HttpStatus.UNAUTHORIZED]: 'Требуется авторизация',
|
||||
[HttpStatus.FORBIDDEN]: 'Доступ запрещён',
|
||||
[HttpStatus.NOT_FOUND]: 'Ресурс не найден',
|
||||
[HttpStatus.METHOD_NOT_ALLOWED]: 'Метод не поддерживается',
|
||||
[HttpStatus.CONFLICT]: 'Конфликт данных',
|
||||
[HttpStatus.PAYLOAD_TOO_LARGE]: 'Тело запроса слишком большое',
|
||||
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'Неподдерживаемый формат данных',
|
||||
[HttpStatus.TOO_MANY_REQUESTS]: 'Слишком много запросов',
|
||||
[HttpStatus.INTERNAL_SERVER_ERROR]: 'Внутренняя ошибка сервера',
|
||||
[HttpStatus.SERVICE_UNAVAILABLE]: 'Сервис временно недоступен',
|
||||
};
|
||||
|
||||
const databaseUnavailableCodes = new Set([
|
||||
'P1000',
|
||||
'P1001',
|
||||
'P1002',
|
||||
'P1008',
|
||||
'P1017',
|
||||
'P2024',
|
||||
'P2037',
|
||||
'ECONNREFUSED',
|
||||
'ECONNRESET',
|
||||
'ETIMEDOUT',
|
||||
'EPIPE',
|
||||
'ENOTFOUND',
|
||||
'EAI_AGAIN',
|
||||
'EACCES',
|
||||
'08000',
|
||||
'08001',
|
||||
'08003',
|
||||
'08004',
|
||||
'08006',
|
||||
'53300',
|
||||
'57P01',
|
||||
'57P02',
|
||||
'57P03',
|
||||
'57014',
|
||||
]);
|
||||
const driverTimeoutMessages = [
|
||||
'Query read timeout',
|
||||
'timeout exceeded when trying to connect',
|
||||
'Connection terminated unexpectedly',
|
||||
'Connection terminated due to connection timeout',
|
||||
];
|
||||
|
||||
function databaseUnavailable(exception: unknown): boolean {
|
||||
if (exception instanceof Prisma.PrismaClientInitializationError) return true;
|
||||
if (exception instanceof HttpException || !(exception instanceof Error))
|
||||
return false;
|
||||
const error = exception as Error & { code?: unknown };
|
||||
if (
|
||||
typeof error.code === 'string' &&
|
||||
databaseUnavailableCodes.has(error.code)
|
||||
)
|
||||
return true;
|
||||
if (exception instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
const originalCode = exception.meta?.code;
|
||||
if (
|
||||
typeof originalCode === 'string' &&
|
||||
databaseUnavailableCodes.has(originalCode)
|
||||
)
|
||||
return true;
|
||||
const adapter = exception.meta?.driverAdapterError;
|
||||
if (adapter !== null && typeof adapter === 'object' && 'cause' in adapter) {
|
||||
const cause = adapter.cause;
|
||||
if (cause !== null && typeof cause === 'object') {
|
||||
if ('kind' in cause && cause.kind === 'ConnectionClosed') return true;
|
||||
if (
|
||||
'originalCode' in cause &&
|
||||
typeof cause.originalCode === 'string' &&
|
||||
databaseUnavailableCodes.has(cause.originalCode)
|
||||
)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return driverTimeoutMessages.some((message) =>
|
||||
exception.message.includes(message),
|
||||
);
|
||||
}
|
||||
|
||||
@Catch()
|
||||
export class AllExceptionsFilter implements ExceptionFilter {
|
||||
catch(exception: unknown, host: ArgumentsHost): void {
|
||||
if (databaseUnavailable(exception)) {
|
||||
exception = new ApiException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE,
|
||||
'unavailable',
|
||||
'База данных недоступна',
|
||||
);
|
||||
}
|
||||
const context = host.switchToHttp();
|
||||
const request = context.getRequest<FastifyRequest>();
|
||||
const response = context.getResponse<FastifyReply>();
|
||||
if (exception instanceof AuthException) {
|
||||
void response.header('WWW-Authenticate', exception.challenge);
|
||||
}
|
||||
|
||||
const status =
|
||||
exception instanceof HttpException
|
||||
? exception.getStatus()
|
||||
: HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
const payload =
|
||||
exception instanceof HttpException ? exception.getResponse() : undefined;
|
||||
const { code, message } = this.extractError(status, payload);
|
||||
const requestId = request.id;
|
||||
|
||||
if (status >= HttpStatus.INTERNAL_SERVER_ERROR) {
|
||||
request.raw.log?.error(
|
||||
{ err: exception, status, requestId },
|
||||
'Request failed',
|
||||
);
|
||||
}
|
||||
|
||||
void response.status(status).send({
|
||||
error: { code, message, request_id: requestId },
|
||||
});
|
||||
}
|
||||
|
||||
private extractError(
|
||||
status: HttpStatus,
|
||||
payload: string | object | undefined,
|
||||
): { code: string; message: string } {
|
||||
if (status === HttpStatus.INTERNAL_SERVER_ERROR) {
|
||||
return {
|
||||
code: 'internal_error',
|
||||
message: defaultMessages[status] ?? 'Ошибка сервера',
|
||||
};
|
||||
}
|
||||
|
||||
const body = (
|
||||
payload !== null && typeof payload === 'object' ? payload : {}
|
||||
) as ExceptionPayload;
|
||||
|
||||
return {
|
||||
code:
|
||||
typeof body.code === 'string'
|
||||
? body.code
|
||||
: (defaultCodes[status] ?? 'http_error'),
|
||||
message:
|
||||
defaultMessages[status] ??
|
||||
(status >= HttpStatus.INTERNAL_SERVER_ERROR
|
||||
? 'Ошибка сервера'
|
||||
: 'Ошибка запроса'),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
FastifyAdapter,
|
||||
NestFastifyApplication,
|
||||
} from '@nestjs/platform-fastify';
|
||||
|
||||
const requestIdHeader = 'x-request-id';
|
||||
const instanceIdHeader = 'x-instance-id';
|
||||
const instanceId = process.env.INSTANCE_ID?.trim() || randomUUID();
|
||||
|
||||
// Two parallel readiness rounds: 2 * (1s request timeout + 2s interval),
|
||||
// plus one second for the upstream reload. Keep below the public hook's 10s limit.
|
||||
export const DRAIN_DELAY_MS = 7000;
|
||||
|
||||
export class DrainingFastifyAdapter extends FastifyAdapter {
|
||||
private draining = false;
|
||||
private closing?: Promise<void>;
|
||||
private httpClosing?: Promise<undefined>;
|
||||
|
||||
constructor(private readonly drainDelayMs: number) {
|
||||
super({ requestIdHeader, genReqId: () => randomUUID() });
|
||||
this.getInstance().addHook('onSend', (_request, reply, payload, done) => {
|
||||
// An already active request may finish after Fastify starts closing.
|
||||
// Finish its response, then close its keep-alive socket as well.
|
||||
if (this.httpClosing) void reply.header('connection', 'close');
|
||||
done(null, payload);
|
||||
});
|
||||
}
|
||||
|
||||
get isDraining(): boolean {
|
||||
return this.draining;
|
||||
}
|
||||
|
||||
override beforeClose(): Promise<void> {
|
||||
return (this.closing ??= this.drainAndClose());
|
||||
}
|
||||
|
||||
override close(): Promise<undefined> {
|
||||
return (this.httpClosing ??= super.close());
|
||||
}
|
||||
|
||||
private async drainAndClose(): Promise<void> {
|
||||
this.draining = true;
|
||||
if (this.drainDelayMs > 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, this.drainDelayMs));
|
||||
}
|
||||
// Nest awaits beforeClose before destroying Prisma/Redis providers.
|
||||
// Fastify first stops new connections and finishes active HTTP requests.
|
||||
await this.close();
|
||||
}
|
||||
}
|
||||
|
||||
export function createFastifyAdapter(drainDelayMs = 0): DrainingFastifyAdapter {
|
||||
return new DrainingFastifyAdapter(drainDelayMs);
|
||||
}
|
||||
|
||||
export function exposeRequestIdHeader(app: NestFastifyApplication): void {
|
||||
app
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.addHook('onRequest', (request, reply, done) => {
|
||||
void reply.header(requestIdHeader, request.id);
|
||||
void reply.header(instanceIdHeader, instanceId);
|
||||
done();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { ApiException } from './exceptions/api.exception';
|
||||
import { HttpStatus } from '@nestjs/common';
|
||||
|
||||
export const uuidPattern =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function invalidRequest(message: string): never {
|
||||
throw new ApiException(HttpStatus.BAD_REQUEST, 'invalid_request', message);
|
||||
}
|
||||
|
||||
export function objectBody(
|
||||
value: unknown,
|
||||
keys: string[],
|
||||
): Record<string, unknown> {
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
return invalidRequest('Ожидается JSON-объект');
|
||||
}
|
||||
const body = value as Record<string, unknown>;
|
||||
if (Object.keys(body).some((key) => !keys.includes(key))) {
|
||||
return invalidRequest('Неизвестное поле запроса');
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
export function requiredText(
|
||||
value: unknown,
|
||||
field: string,
|
||||
max: number,
|
||||
): string {
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
return invalidRequest(`Некорректное поле ${field}`);
|
||||
}
|
||||
let codePoints = 0;
|
||||
for (const _character of value) codePoints++;
|
||||
if (codePoints > max) return invalidRequest(`Некорректное поле ${field}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function uuid(value: unknown, field: string): string {
|
||||
if (typeof value !== 'string' || !uuidPattern.test(value)) {
|
||||
return invalidRequest(`Некорректное поле ${field}`);
|
||||
}
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
export function pageLimit(value: unknown): number {
|
||||
if (value === undefined) return 50;
|
||||
if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value)) {
|
||||
return invalidRequest('Некорректный limit');
|
||||
}
|
||||
const limit = Number(value);
|
||||
if (limit > 100) return invalidRequest('Некорректный limit');
|
||||
return limit;
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||
import { apiReference } from '@scalar/nestjs-api-reference';
|
||||
import { AppModule } from './app.module';
|
||||
import { Logger } from 'nestjs-pino';
|
||||
import {
|
||||
createFastifyAdapter,
|
||||
DRAIN_DELAY_MS,
|
||||
exposeRequestIdHeader,
|
||||
} from './common/http/fastify-adapter';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
createFastifyAdapter(DRAIN_DELAY_MS),
|
||||
{
|
||||
bufferLogs: true,
|
||||
},
|
||||
);
|
||||
|
||||
exposeRequestIdHeader(app);
|
||||
app.useLogger(app.get(Logger));
|
||||
app.enableShutdownHooks();
|
||||
|
||||
const swaggerConfig = new DocumentBuilder()
|
||||
.setTitle('Messenger API')
|
||||
.setDescription('Документация API')
|
||||
.setVersion('1.3.0')
|
||||
.addBasicAuth({ type: 'http', scheme: 'basic' }, 'basicAuth')
|
||||
.addBearerAuth(
|
||||
{ type: 'http', scheme: 'bearer', bearerFormat: 'opaque' },
|
||||
'bearerAuth',
|
||||
)
|
||||
.build();
|
||||
|
||||
const document = SwaggerModule.createDocument(app, swaggerConfig);
|
||||
|
||||
SwaggerModule.setup('swagger', app, document);
|
||||
|
||||
app.use(
|
||||
'/docs',
|
||||
apiReference({
|
||||
content: document,
|
||||
theme: 'laserwave',
|
||||
withFastify: true,
|
||||
}),
|
||||
);
|
||||
|
||||
await app.listen(
|
||||
Number(process.env.APP_PORT ?? process.env.PORT ?? 8080),
|
||||
process.env.APP_HOST ?? '0.0.0.0',
|
||||
);
|
||||
}
|
||||
|
||||
void bootstrap();
|
||||
@@ -0,0 +1,6 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { PrismaService } from './prisma.service';
|
||||
|
||||
@Global()
|
||||
@Module({ providers: [PrismaService], exports: [PrismaService] })
|
||||
export class PrismaModule {}
|
||||
@@ -0,0 +1,42 @@
|
||||
import 'dotenv/config';
|
||||
import {
|
||||
Injectable,
|
||||
type OnModuleDestroy,
|
||||
type OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { PrismaPg } from '@prisma/adapter-pg';
|
||||
import { PrismaClient } from '../generated/prisma/client';
|
||||
|
||||
@Injectable()
|
||||
export class PrismaService
|
||||
extends PrismaClient
|
||||
implements OnModuleInit, OnModuleDestroy
|
||||
{
|
||||
constructor() {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
if (!connectionString) throw new Error('DATABASE_URL is required');
|
||||
const poolMax = Number(process.env.DB_POOL_MAX ?? 15);
|
||||
if (!Number.isSafeInteger(poolMax) || poolMax < 1) {
|
||||
throw new Error('DB_POOL_MAX must be a positive integer');
|
||||
}
|
||||
super({
|
||||
adapter: new PrismaPg({
|
||||
connectionString,
|
||||
max: poolMax,
|
||||
connectionTimeoutMillis: 5000,
|
||||
query_timeout: 5000,
|
||||
idleTimeoutMillis: 30000,
|
||||
options: '-c statement_timeout=5000',
|
||||
}),
|
||||
transactionOptions: { maxWait: 5000, timeout: 10000 },
|
||||
});
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.$connect();
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
await this.$disconnect();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { createRequire } from 'node:module';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
const require = createRequire(`${process.cwd()}/package.json`);
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
|
||||
const { PrismaService } = require(`${process.cwd()}/dist/prisma/prisma.service`);
|
||||
const { UserService } = require(`${process.cwd()}/dist/User/user.service`);
|
||||
const { PasswordService } = require(`${process.cwd()}/dist/Auth/password.service`);
|
||||
const { createFastifyAdapter } = require(`${process.cwd()}/dist/common/http/fastify-adapter`);
|
||||
process.env.CACHE_URL = '';
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false });
|
||||
await app.init();
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
const prisma = app.get(PrismaService);
|
||||
const passwords = app.get(PasswordService);
|
||||
const originalHash = passwords.hash.bind(passwords);
|
||||
const spans = [];
|
||||
passwords.hash = async (password) => {
|
||||
const start = performance.now();
|
||||
const value = await originalHash(password);
|
||||
spans.push({ phase: 'kdf', ms: Math.round(performance.now() - start) });
|
||||
return value;
|
||||
};
|
||||
app.get(UserService).prisma = prisma.$extends({ query: { user: { async create({ args, query }) {
|
||||
const start = performance.now();
|
||||
const value = await query(args);
|
||||
spans.push({ phase: 'sql_user_create', ms: Math.round(performance.now() - start) });
|
||||
return value;
|
||||
} } } });
|
||||
const ids = [];
|
||||
try {
|
||||
const health = await server.inject({ method: 'GET', url: '/health/ready' });
|
||||
console.log(JSON.stringify({ phase: 'health', status: health.statusCode }));
|
||||
for (let sample = 1; sample <= 3; sample++) {
|
||||
spans.length = 0;
|
||||
const start = performance.now();
|
||||
const response = await server.inject({ method: 'POST', url: '/api/v1/users', payload: {
|
||||
username: `warm_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
display_name: 'Synthetic startup diagnostic', password: 'Synthetic-diagnostic-password-123',
|
||||
} });
|
||||
if (response.statusCode === 201) ids.push(response.json().id);
|
||||
console.log(JSON.stringify({ sample, status: response.statusCode, total_ms: Math.round(performance.now() - start), spans }));
|
||||
}
|
||||
} finally {
|
||||
await prisma.user.deleteMany({ where: { id: { in: ids } } });
|
||||
await app.close();
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createServer } from 'node:http';
|
||||
import test from 'node:test';
|
||||
|
||||
test('scaling benchmark schedules reads and writes and reports distinct instances', async () => {
|
||||
let sequence = 0;
|
||||
const server = createServer(async (request, response) => {
|
||||
for await (const _ of request) {
|
||||
// Consume the request body before responding.
|
||||
}
|
||||
response.setHeader('Content-Type', 'application/json');
|
||||
response.setHeader('X-Instance-Id', 'mock1');
|
||||
let status = 200;
|
||||
let body = {};
|
||||
if (request.url === '/api/v1/users' && request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: `user-${++sequence}` };
|
||||
} else if (request.url === '/api/v1/auth/sessions') {
|
||||
status = 201;
|
||||
body = { token: 'synthetic-token' };
|
||||
} else if (request.url === '/api/v1/chats' && request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: 'chat-1' };
|
||||
} else if (request.url.startsWith('/api/v1/chats/chat-1/messages')) {
|
||||
if (request.method === 'POST') {
|
||||
status = 201;
|
||||
body = { id: `message-${++sequence}` };
|
||||
} else {
|
||||
body = { items: [] };
|
||||
}
|
||||
} else {
|
||||
status = 404;
|
||||
}
|
||||
response.writeHead(status);
|
||||
response.end(JSON.stringify(body));
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
try {
|
||||
const port = server.address().port;
|
||||
const args = [
|
||||
'scripts/benchmark-scaling.mjs', '--base-url', `http://127.0.0.1:${port}`,
|
||||
'--label', 'mock', '--rate', '20', '--duration-seconds', '2',
|
||||
'--warmup-seconds', '1', '--seed-messages', '2',
|
||||
'--read-percent', '50', '--expected-instances', '1',
|
||||
];
|
||||
const child = spawn(process.execPath, args, { cwd: process.cwd() });
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => { stdout += chunk; });
|
||||
child.stderr.on('data', (chunk) => { stderr += chunk; });
|
||||
const code = await new Promise((resolve) => child.on('close', resolve));
|
||||
assert.equal(code, 0, stderr);
|
||||
const result = JSON.parse(stdout);
|
||||
assert.equal(result.measurement.scheduled_slots, 40);
|
||||
assert.equal(result.measurement.client_dropped_slots, 0);
|
||||
assert.equal(result.measurement.all.successful, 40);
|
||||
assert.equal(result.measurement.all.instance_counts.mock1, 40);
|
||||
assert.ok(result.measurement.read.successful > 0);
|
||||
assert.ok(result.measurement.write.successful > 0);
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,541 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(`${process.cwd()}/package.json`);
|
||||
const pg = require('pg');
|
||||
let counts = {};
|
||||
const originalQuery = pg.Client.prototype.query;
|
||||
pg.Client.prototype.query = function (query, ...args) {
|
||||
const sql = typeof query === 'string' ? query : query.text;
|
||||
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
|
||||
const table =
|
||||
['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) =>
|
||||
new RegExp(`\\b${name}\\b`).test(sql),
|
||||
) ?? 'other';
|
||||
const label = `${verb}:${table}`;
|
||||
counts[label] = (counts[label] ?? 0) + 1;
|
||||
return originalQuery.call(this, query, ...args);
|
||||
};
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
|
||||
const { PrismaService } = require(
|
||||
`${process.cwd()}/dist/prisma/prisma.service`,
|
||||
);
|
||||
const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`);
|
||||
const { createFastifyAdapter, exposeRequestIdHeader } = require(
|
||||
`${process.cwd()}/dist/common/http/fastify-adapter`,
|
||||
);
|
||||
const stage = process.argv[2];
|
||||
const compareRedis = process.argv.includes('--compare-redis');
|
||||
const historyControl = !process.argv.includes('--skip-history-control');
|
||||
function integerOption(name, fallback) {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
const value = index === -1 ? fallback : Number(process.argv[index + 1]);
|
||||
assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`);
|
||||
return value;
|
||||
}
|
||||
const concurrency = integerOption('concurrency', 16);
|
||||
const duration = integerOption('duration-seconds', 10);
|
||||
const warmup = integerOption('warmup-seconds', 3);
|
||||
const repetitions = integerOption('repetitions', 3);
|
||||
const seedMessages = integerOption('seed-messages', 10000);
|
||||
const coldBurst = integerOption('cold-burst', 32);
|
||||
const writesPerRepeat = integerOption('writes-per-repeat', 512);
|
||||
assert.ok(['before', 'after'].includes(stage));
|
||||
const prefix = `experiment:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
|
||||
const observer = new Redis(process.env.CACHE_URL);
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
const apps = [];
|
||||
async function start() {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
return { app, url: await app.getUrl() };
|
||||
}
|
||||
const first = await start();
|
||||
const prisma = first.app.get(PrismaService);
|
||||
let userId;
|
||||
const chatIds = [];
|
||||
const report = {
|
||||
stage,
|
||||
recorded_at: new Date().toISOString(),
|
||||
node: process.version,
|
||||
methodology: {
|
||||
requests:
|
||||
'real loopback HTTP; instrumented API process in measured backend image',
|
||||
sql: 'pg.Client.query calls; transaction control counted separately',
|
||||
concurrency,
|
||||
duration_seconds: duration,
|
||||
warmup_seconds: warmup,
|
||||
repetitions,
|
||||
seed_messages: seedMessages,
|
||||
cold_burst_requests: coldBurst,
|
||||
writes_per_repeat: writesPerRepeat,
|
||||
history_control: historyControl,
|
||||
resources:
|
||||
'process CPU/RSS includes HTTP generator and instrumentation; not container CPU',
|
||||
limitations:
|
||||
'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF',
|
||||
},
|
||||
dist_sha256: {},
|
||||
dependencies: {},
|
||||
samples: [],
|
||||
correctness: {},
|
||||
};
|
||||
for (const name of [
|
||||
'@nestjs/core',
|
||||
'@prisma/adapter-pg',
|
||||
'pg',
|
||||
'ioredis',
|
||||
'fastify',
|
||||
]) {
|
||||
try {
|
||||
report.dependencies[name] = require(`${name}/package.json`).version;
|
||||
} catch {
|
||||
report.dependencies[name] = 'package metadata not exported';
|
||||
}
|
||||
}
|
||||
for (const name of [
|
||||
'Chat/chat.service.js',
|
||||
'cache/cache.service.js',
|
||||
'Auth/auth.service.js',
|
||||
'User/user.guard.js',
|
||||
'Chat/chat.controller.js',
|
||||
]) {
|
||||
report.dist_sha256[name] = createHash('sha256')
|
||||
.update(await readFile(`${process.cwd()}/dist/${name}`))
|
||||
.digest('hex');
|
||||
}
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
async function clearCache() {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
}
|
||||
let token;
|
||||
async function request(method, path, body, target = first) {
|
||||
const response = await fetch(`${target.url}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(body ? { 'content-type': 'application/json' } : {}),
|
||||
...(method === 'POST' && /\/messages$/.test(path)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
const json = await response.json();
|
||||
assert.equal(
|
||||
response.status,
|
||||
method === 'POST' ? 201 : 200,
|
||||
`HTTP ${response.status}`,
|
||||
);
|
||||
return json;
|
||||
}
|
||||
async function newChat() {
|
||||
const chat = await prisma.chat.create({
|
||||
data: {
|
||||
title: 'Synthetic epoch benchmark',
|
||||
members: { create: { userId, position: 0 } },
|
||||
},
|
||||
});
|
||||
chatIds.push(chat.id);
|
||||
return chat.id;
|
||||
}
|
||||
async function measure(
|
||||
workload,
|
||||
repeat,
|
||||
operation,
|
||||
{
|
||||
duration: secondsToRun = duration,
|
||||
total,
|
||||
concurrency: clients = concurrency,
|
||||
} = {},
|
||||
) {
|
||||
counts = {};
|
||||
const cpu = process.cpuUsage();
|
||||
let peakRss = process.memoryUsage().rss;
|
||||
const timer = setInterval(() => {
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
}, 100);
|
||||
const latencies = [];
|
||||
let errors = 0,
|
||||
index = 0;
|
||||
const start = performance.now();
|
||||
await Promise.all(
|
||||
Array.from({ length: clients }, async () => {
|
||||
while (
|
||||
total === undefined
|
||||
? performance.now() - start < secondsToRun * 1000
|
||||
: index < total
|
||||
) {
|
||||
const item = index++;
|
||||
const began = performance.now();
|
||||
try {
|
||||
await operation(item);
|
||||
latencies.push(performance.now() - began);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
const seconds = (performance.now() - start) / 1000;
|
||||
clearInterval(timer);
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
const used = process.cpuUsage(cpu);
|
||||
latencies.sort((a, b) => a - b);
|
||||
const percentile = (p) =>
|
||||
latencies.length
|
||||
? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3)
|
||||
: null;
|
||||
const sql = Object.entries(counts)
|
||||
.filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key))
|
||||
.reduce((sum, [, n]) => sum + n, 0);
|
||||
const sample = {
|
||||
workload,
|
||||
repeat,
|
||||
concurrency: clients,
|
||||
elapsed_seconds: +seconds.toFixed(3),
|
||||
successes: latencies.length,
|
||||
errors,
|
||||
error_ratio: errors / (latencies.length + errors),
|
||||
successful_rps: +(latencies.length / seconds).toFixed(3),
|
||||
latency_ms: {
|
||||
mean: latencies.length
|
||||
? +(
|
||||
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
|
||||
).toFixed(3)
|
||||
: null,
|
||||
p50: percentile(0.5),
|
||||
p95: percentile(0.95),
|
||||
p99: percentile(0.99),
|
||||
},
|
||||
sql_queries: sql,
|
||||
sql_per_request: +(sql / (latencies.length + errors)).toFixed(4),
|
||||
sql_calls: { ...counts },
|
||||
process_cpu_percent_one_core: +(
|
||||
(used.user + used.system) /
|
||||
(seconds * 10000)
|
||||
).toFixed(2),
|
||||
peak_process_rss_mib: +(peakRss / 1048576).toFixed(2),
|
||||
};
|
||||
report.samples.push(sample);
|
||||
console.log(JSON.stringify(sample));
|
||||
assert.equal(errors, 0, 'load errors recorded in report');
|
||||
}
|
||||
try {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic benchmark',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 3600000),
|
||||
},
|
||||
});
|
||||
const readChat = await newChat();
|
||||
for (let offset = 0; offset < seedMessages; offset += 1000) {
|
||||
await prisma.message.createMany({
|
||||
data: Array.from(
|
||||
{ length: Math.min(1000, seedMessages - offset) },
|
||||
(_, i) => ({
|
||||
chatId: readChat,
|
||||
senderId: userId,
|
||||
text: `Synthetic message ${offset + i}`,
|
||||
createdAt: new Date(1700000000000 + offset + i),
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
await prisma.chatMessageEpoch.createMany({
|
||||
data: Array.from({ length: 16 }, (_, bucket) => ({
|
||||
chatId: readChat,
|
||||
bucket,
|
||||
epoch: BigInt(
|
||||
Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0),
|
||||
),
|
||||
})),
|
||||
});
|
||||
const path = `/api/v1/chats/${readChat}/messages?limit=50`;
|
||||
if (compareRedis) {
|
||||
const cacheUrl = process.env.CACHE_URL;
|
||||
delete process.env.CACHE_URL;
|
||||
let withoutRedis;
|
||||
try {
|
||||
withoutRedis = await start();
|
||||
} finally {
|
||||
process.env.CACHE_URL = cacheUrl;
|
||||
}
|
||||
assert.equal(withoutRedis.app.get(CacheService).redis, undefined);
|
||||
report.methodology.comparison = {
|
||||
modes: ['redis', 'no-redis'],
|
||||
no_redis:
|
||||
'API started without CACHE_URL; all application Redis caches disabled',
|
||||
epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes',
|
||||
order: 'redis first in repeats 1/3, no-redis first in repeat 2',
|
||||
processes:
|
||||
'both API instances and HTTP generator share one measured Node process',
|
||||
};
|
||||
report.correctness.read_after_write = {};
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
const variants = [
|
||||
{ name: 'redis', target: first },
|
||||
{ name: 'no-redis', target: withoutRedis },
|
||||
];
|
||||
if (repeat % 2 === 0) variants.reverse();
|
||||
for (const { name, target } of variants) {
|
||||
await clearCache();
|
||||
await measure(
|
||||
`cold-history-burst:${name}`,
|
||||
repeat,
|
||||
() => request('GET', path, undefined, target),
|
||||
{ total: coldBurst, concurrency: coldBurst },
|
||||
);
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil)
|
||||
await request('GET', path, undefined, target);
|
||||
await measure(`warm-history:${name}`, repeat, () =>
|
||||
request('GET', path, undefined, target),
|
||||
);
|
||||
const writeChat = await newChat();
|
||||
const writePath = `/api/v1/chats/${writeChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', writePath, undefined, target)).items.length,
|
||||
0,
|
||||
);
|
||||
await measure(
|
||||
`writes-single-chat:${name}`,
|
||||
repeat,
|
||||
(i) =>
|
||||
request(
|
||||
'POST',
|
||||
writePath,
|
||||
{ text: `Concurrent write ${i}` },
|
||||
target,
|
||||
),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
assert.equal(
|
||||
await messageEpoch(prisma, writeChat),
|
||||
BigInt(writesPerRepeat),
|
||||
);
|
||||
const page = await request('GET', writePath, undefined, target);
|
||||
assert.equal(page.items.length, 50);
|
||||
assert.equal(new Set(page.items.map((message) => message.id)).size, 50);
|
||||
report.correctness.read_after_write[`${name}:${repeat}`] = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const cache = first.app.get(CacheService);
|
||||
const remember = cache.remember.bind(cache);
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
await clearCache();
|
||||
await request('GET', `/api/v1/chats/${readChat}`);
|
||||
await measure('cold-history-burst', repeat, () => request('GET', path), {
|
||||
total: coldBurst,
|
||||
concurrency: coldBurst,
|
||||
});
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil) await request('GET', path);
|
||||
await measure('warm-history', repeat, () => request('GET', path));
|
||||
if (historyControl) {
|
||||
cache.remember = (namespace, parameters, ttl, load) =>
|
||||
namespace.startsWith('messages:')
|
||||
? load()
|
||||
: remember(namespace, parameters, ttl, load);
|
||||
const directUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < directUntil) await request('GET', path);
|
||||
await measure('history-cache-bypassed', repeat, () =>
|
||||
request('GET', path),
|
||||
);
|
||||
cache.remember = remember;
|
||||
}
|
||||
const writeChat = await newChat();
|
||||
await request('GET', `/api/v1/chats/${writeChat}`);
|
||||
await measure(
|
||||
'writes-single-chat',
|
||||
repeat,
|
||||
(i) =>
|
||||
request('POST', `/api/v1/chats/${writeChat}/messages`, {
|
||||
text: `Concurrent write ${i}`,
|
||||
}),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
}
|
||||
const second = await start();
|
||||
const faultChat = await newChat();
|
||||
const faultPath = `/api/v1/chats/${faultChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', faultPath, undefined, second)).items.length,
|
||||
0,
|
||||
);
|
||||
const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`);
|
||||
assert.ok(keysBefore.length);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
const written = await request('POST', faultPath, {
|
||||
text: 'Committed while Redis is unavailable',
|
||||
});
|
||||
const during = await request('GET', faultPath, undefined, second);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis?.status === 'ready'))
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
const recovered = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.redis_reconnect = {
|
||||
old_keys_retained:
|
||||
(await observer.exists(...keysBefore)) === keysBefore.length,
|
||||
read_after_write_during_outage: during.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
read_after_write_after_reconnect_other_replica: recovered.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
};
|
||||
await clearCache();
|
||||
const cold = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.empty_cache_recovery = cold.items.some(
|
||||
(x) => x.id === written.id,
|
||||
);
|
||||
const raceChat = await newChat();
|
||||
const racePath = `/api/v1/chats/${raceChat}/messages`;
|
||||
const service = first.app.get(
|
||||
require(`${process.cwd()}/dist/Chat/chat.service`).ChatService,
|
||||
);
|
||||
const actualPrisma = service.prisma;
|
||||
let release, began;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
service.prisma = actualPrisma.$extends({
|
||||
query: {
|
||||
message: {
|
||||
async findMany({ args, query }) {
|
||||
const rows = await query(args);
|
||||
if (args.where?.chatId === raceChat) {
|
||||
began();
|
||||
await held;
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const pending = request('GET', racePath);
|
||||
let fresh;
|
||||
try {
|
||||
await Promise.race([
|
||||
started,
|
||||
sleep(10000).then(() => {
|
||||
throw new Error('race load did not start');
|
||||
}),
|
||||
]);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
fresh = await request(
|
||||
'POST',
|
||||
racePath,
|
||||
{ text: 'New message after old SELECT' },
|
||||
second,
|
||||
);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready')
|
||||
)
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
} finally {
|
||||
release();
|
||||
service.prisma = actualPrisma;
|
||||
}
|
||||
await pending;
|
||||
report.correctness.delayed_old_fill = (
|
||||
await request('GET', racePath, undefined, second)
|
||||
).items.some((x) => x.id === fresh.id);
|
||||
if (stage === 'after') {
|
||||
assert.equal(
|
||||
report.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
true,
|
||||
);
|
||||
assert.equal(report.correctness.delayed_old_fill, true);
|
||||
assert.equal(report.correctness.empty_cache_recovery, true);
|
||||
}
|
||||
}
|
||||
report.completed = true;
|
||||
} finally {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
await clearCache();
|
||||
report.cleanup = 'only experiment chats/user/cache prefix removed';
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
await writeFile(
|
||||
process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`,
|
||||
JSON.stringify(report, null, 2) + '\n',
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
stage,
|
||||
correctness: report.correctness,
|
||||
completed: report.completed,
|
||||
}),
|
||||
);
|
||||
@@ -0,0 +1,25 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$dockerInfo = docker info --format '{{json .}}' | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) { throw 'docker info failed' }
|
||||
$processor = Get-CimInstance Win32_Processor | Select-Object -First 1
|
||||
$computer = Get-CimInstance Win32_ComputerSystem
|
||||
$containers = docker inspect mtusi2026lab2-fork-backend-1 mtusi2026lab2-fork-postgres-1 mtusi2026lab2-fork-redis-1 | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) { throw 'docker inspect failed' }
|
||||
$metadata = [ordered]@{
|
||||
recorded_at = [DateTime]::UtcNow.ToString('o')
|
||||
host_cpu = $processor.Name
|
||||
host_physical_cores = $processor.NumberOfCores
|
||||
host_logical_processors = $processor.NumberOfLogicalProcessors
|
||||
host_ram_bytes = $computer.TotalPhysicalMemory
|
||||
docker_version = $dockerInfo.ServerVersion
|
||||
docker_cpu_count = $dockerInfo.NCPU
|
||||
docker_ram_bytes = $dockerInfo.MemTotal
|
||||
docker_kernel = $dockerInfo.KernelVersion
|
||||
containers = @($containers | ForEach-Object {
|
||||
[ordered]@{ name = $_.Name; image = $_.Image; memory_limit_bytes = $_.HostConfig.Memory; nano_cpus_limit = $_.HostConfig.NanoCpus; cpu_quota = $_.HostConfig.CpuQuota }
|
||||
})
|
||||
before_image = 'sha256:075170aed76a42aa98b435cfb918d83879259b7e17177275d5274e328198e119'
|
||||
before_image_local_tag = 'lab2-cache-epochs:before'
|
||||
}
|
||||
$metadataPath = Join-Path (Get-Location).Path 'evidence/cache-epochs-environment.json'
|
||||
[System.IO.File]::WriteAllText($metadataPath, ($metadata | ConvertTo-Json -Depth 6) + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false))
|
||||
@@ -0,0 +1,91 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const reports = await Promise.all(
|
||||
['after', 'optimized'].map(async (name) =>
|
||||
JSON.parse(await readFile(`evidence/cache-epochs-${name}.json`, 'utf8')),
|
||||
),
|
||||
);
|
||||
assert.ok(reports.every((report) => report.completed));
|
||||
assert.deepEqual(reports[0].methodology, reports[1].methodology);
|
||||
const median = (values) =>
|
||||
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const labels = ['Отдельная транзакция', 'Один CTE'];
|
||||
const lines = [
|
||||
'# Оптимизация эпохи кэша в PostgreSQL',
|
||||
'',
|
||||
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
|
||||
'',
|
||||
'Сравнение сохранённого замера прежней реализации и нового замера после оптимизации. В обоих вариантах эпоха хранится в PostgreSQL; новый вариант объединяет проверку членства, UPDATE эпохи и INSERT сообщения одним запросом. При чтении истории убрано получение неиспользуемых метаданных чата.',
|
||||
'',
|
||||
'Медианы трёх повторов. Перцентили — медианы перцентилей повторов, без объединения выборок.',
|
||||
'',
|
||||
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | BEGIN/COMMIT на HTTP | CPU, % ядра | RSS, MiB | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const workload of new Set(
|
||||
reports[0].samples.map((sample) => sample.workload),
|
||||
)) {
|
||||
for (const [index, report] of reports.entries()) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.workload === workload,
|
||||
);
|
||||
assert.equal(samples.length, 3);
|
||||
const get = (select) => median(samples.map(select)).toFixed(3);
|
||||
const controls = (sample) =>
|
||||
Object.entries(sample.sql_calls)
|
||||
.filter(([key]) => /^(BEGIN|COMMIT):/.test(key))
|
||||
.reduce((total, [, count]) => total + count, 0) /
|
||||
(sample.successes + sample.errors);
|
||||
lines.push(
|
||||
`| ${workload} | ${labels[index]} | ${get((sample) => sample.successful_rps)} | ${get((sample) => sample.latency_ms.p50)} | ${get((sample) => sample.latency_ms.p95)} | ${get((sample) => sample.latency_ms.p99)} | ${get((sample) => sample.sql_per_request)} | ${get(controls)} | ${get((sample) => sample.process_cpu_percent_one_core)} | ${get((sample) => sample.peak_process_rss_mib)} | ${samples.reduce((total, sample) => total + sample.errors, 0)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'| Проверка | Отдельная транзакция | Один CTE |',
|
||||
'| --- | --- | --- |',
|
||||
);
|
||||
for (const [label, select] of [
|
||||
[
|
||||
'Чтение после записи при отказе Redis',
|
||||
(report) =>
|
||||
report.correctness.redis_reconnect.read_after_write_during_outage,
|
||||
],
|
||||
[
|
||||
'Чтение другой репликой после восстановления Redis',
|
||||
(report) =>
|
||||
report.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
],
|
||||
[
|
||||
'Старые ключи сохранились в Redis',
|
||||
(report) => report.correctness.redis_reconnect.old_keys_retained,
|
||||
],
|
||||
[
|
||||
'Восстановление с пустым кэшем',
|
||||
(report) => report.correctness.empty_cache_recovery,
|
||||
],
|
||||
[
|
||||
'Задержанная загрузка старой истории',
|
||||
(report) => report.correctness.delayed_old_fill,
|
||||
],
|
||||
]) {
|
||||
lines.push(
|
||||
`| ${label} | ${reports.map((report) => (select(report) ? 'PASS' : 'FAIL')).join(' | ')} |`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'Методика: тот же `test/cache-benchmark.mjs`, 10 000 сообщений, limit=50, конкурентность 16, 3 с прогрева и 10 с чтения, 512 записей в новый чат, три повтора. Реальный HTTP по loopback в отдельных benchmark-контейнерах; общий PostgreSQL/Redis. Токены и пользователи создаются напрямую, хеширование паролей не измеряется. BEGIN/COMMIT исключены из SQL/HTTP; запрос WITH учитывается как один вызов pg.Client.query.',
|
||||
'',
|
||||
'CPU/RSS включают API, генератор HTTP и инструментацию. Docker-host разделяется с другими работающими сервисами. Старый и новый замеры сделаны в разное время; изменение фоновой нагрузки и состояния host ограничивает сравнение RPS/latency. Замеры не доказывают максимальную ёмкость или SLA. UPDATE общей строки эпохи по-прежнему сериализует записи одного чата.',
|
||||
'',
|
||||
'Сырые результаты: [до оптимизации](cache-epochs-after.json), [после оптимизации](cache-epochs-optimized.json). Хеши измеренных модулей сохранены в JSON. Исторические замеры до введения эпох остаются в `cache-epochs-summary.md`.',
|
||||
);
|
||||
await writeFile(
|
||||
'evidence/cache-epochs-optimization-summary.md',
|
||||
lines.join('\n') + '\n',
|
||||
);
|
||||
console.log(lines.join('\n'));
|
||||
@@ -0,0 +1,200 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { CacheService } = require('../dist/cache/cache.service');
|
||||
const { ChatService } = require('../dist/Chat/chat.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
test(
|
||||
'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback',
|
||||
{ timeout: 90000 },
|
||||
async (t) => {
|
||||
assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL);
|
||||
const oldUrl = process.env.CACHE_URL;
|
||||
const oldPrefix = process.env.CACHE_PREFIX;
|
||||
const prefix = `test:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(oldUrl, 6379);
|
||||
const observer = new Redis(oldUrl);
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
const apps = [];
|
||||
let prisma, userId;
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
if (prisma) {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
}
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
process.env.CACHE_URL = oldUrl;
|
||||
if (oldPrefix === undefined) delete process.env.CACHE_PREFIX;
|
||||
else process.env.CACHE_PREFIX = oldPrefix;
|
||||
}
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
await app.getHttpAdapter().getInstance().ready();
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
}
|
||||
prisma = apps[0].get(PrismaService);
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic epochs',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
const call = async (replica, method, url, payload) => {
|
||||
const response = await apps[replica]
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(payload ? { payload } : {}),
|
||||
});
|
||||
return { status: response.statusCode, body: response.json() };
|
||||
};
|
||||
const createChat = async () => {
|
||||
const response = await call(0, 'POST', '/api/v1/chats', {
|
||||
title: 'Epoch test',
|
||||
member_ids: [userId],
|
||||
});
|
||||
assert.equal(response.status, 201);
|
||||
chatIds.push(response.body.id);
|
||||
return response.body.id;
|
||||
};
|
||||
assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const chatId = await createChat();
|
||||
proxy.setMode('online');
|
||||
const reconnect = async () => {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis.status === 'ready'))
|
||||
return;
|
||||
await pause(100);
|
||||
}
|
||||
assert.fail('Redis did not reconnect');
|
||||
};
|
||||
await reconnect();
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id),
|
||||
[chatId],
|
||||
);
|
||||
assert.equal(
|
||||
(await prisma.user.findUniqueOrThrow({ where: { id: userId } }))
|
||||
.chatsEpoch,
|
||||
1n,
|
||||
);
|
||||
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal((await call(1, 'GET', path)).body.items.length, 0);
|
||||
const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`);
|
||||
assert.ok(oldKeys.length);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const posted = await call(0, 'POST', path, {
|
||||
text: 'Write during Redis outage',
|
||||
});
|
||||
assert.equal(posted.status, 201);
|
||||
proxy.setMode('online');
|
||||
await reconnect();
|
||||
assert.equal(
|
||||
await observer.exists(...oldKeys),
|
||||
oldKeys.length,
|
||||
'old cache keys must still exist',
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', path)).body.items.map((x) => x.id),
|
||||
[posted.body.id],
|
||||
);
|
||||
|
||||
const service = apps[0].get(ChatService);
|
||||
const original = service.prisma;
|
||||
service.prisma = original.$extends({
|
||||
query: {
|
||||
$queryRaw({ args, query }) {
|
||||
const index = args.values.indexOf('Must roll back');
|
||||
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
assert.equal(
|
||||
(await call(0, 'POST', path, { text: 'Must roll back' })).status,
|
||||
500,
|
||||
);
|
||||
} finally {
|
||||
service.prisma = original;
|
||||
}
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 1n);
|
||||
|
||||
const writes = await Promise.all(
|
||||
Array.from({ length: 20 }, (_, i) =>
|
||||
call(i % 2, 'POST', path, { text: `Concurrent ${i}` }),
|
||||
),
|
||||
);
|
||||
assert.ok(writes.every((x) => x.status === 201));
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 21);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 21n);
|
||||
const history = (await call(1, 'GET', path)).body.items;
|
||||
assert.equal(new Set(history.map((x) => x.id)).size, 21);
|
||||
assert.ok(
|
||||
writes.every((x) => history.some((message) => message.id === x.body.id)),
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
const before = JSON.parse(
|
||||
await readFile('evidence/cache-epochs-before.json', 'utf8'),
|
||||
);
|
||||
const after = JSON.parse(
|
||||
await readFile('evidence/cache-epochs-after.json', 'utf8'),
|
||||
);
|
||||
assert.ok(before.completed && after.completed);
|
||||
assert.deepEqual(before.methodology, after.methodology);
|
||||
const median = (values) =>
|
||||
values.sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const lines = [
|
||||
'# Эпохи кэша — сравнение 2026-10-13',
|
||||
'',
|
||||
'Медианы трёх повторов. p50/p95/p99 — медианы перцентилей отдельных повторов, не перцентили объединённой выборки.',
|
||||
'',
|
||||
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | CPU процесса, % ядра | Пиковый RSS, MiB | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const workload of [...new Set(before.samples.map((x) => x.workload))]) {
|
||||
for (const report of [before, after]) {
|
||||
const samples = report.samples.filter((x) => x.workload === workload);
|
||||
assert.equal(samples.length, 3);
|
||||
const get = (fn) => median(samples.map(fn)).toFixed(3);
|
||||
lines.push(
|
||||
`| ${workload} | ${report.stage} | ${get((x) => x.successful_rps)} | ${get((x) => x.latency_ms.p50)} | ${get((x) => x.latency_ms.p95)} | ${get((x) => x.latency_ms.p99)} | ${get((x) => x.sql_per_request)} | ${get((x) => x.process_cpu_percent_one_core)} | ${get((x) => x.peak_process_rss_mib)} | ${samples.reduce((sum, x) => sum + x.errors, 0)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push('', '| Проверка | До | После |', '| --- | --- | --- |');
|
||||
for (const [label, select] of [
|
||||
[
|
||||
'Чтение после успешной записи при недоступном Redis',
|
||||
(x) => x.correctness.redis_reconnect.read_after_write_during_outage,
|
||||
],
|
||||
[
|
||||
'Чтение другой репликой после восстановления Redis со старыми ключами',
|
||||
(x) =>
|
||||
x.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
],
|
||||
[
|
||||
'Старые ключи действительно сохранились',
|
||||
(x) => x.correctness.redis_reconnect.old_keys_retained,
|
||||
],
|
||||
['Восстановление с пустым кэшем', (x) => x.correctness.empty_cache_recovery],
|
||||
[
|
||||
'Задержанная загрузка старой истории не отравляет следующий GET',
|
||||
(x) => x.correctness.delayed_old_fill,
|
||||
],
|
||||
])
|
||||
lines.push(
|
||||
`| ${label} | ${select(before) ? 'PASS' : 'FAIL'} | ${select(after) ? 'PASS' : 'FAIL'} |`,
|
||||
);
|
||||
lines.push(
|
||||
'',
|
||||
'Методика: `test/cache-benchmark.mjs`, сырой результат: `cache-epochs-before.json` и `cache-epochs-after.json`. Конфигурация: `cache-epochs-environment.json`.',
|
||||
'',
|
||||
'Реальный HTTP по loopback, 10 000 синтетических сообщений, первая страница limit=50. На повтор: холодная волна 32 запросов; прогрев 3 с; чтение с конкурентностью 16 в течение 10 с; прогрев 3 с и такой же контроль с отключённым кэшем истории; 512 записей в новый чат с конкурентностью 16. Три повтора. SQL считается на уровне pg.Client.query; BEGIN/COMMIT/ROLLBACK/SET вынесены в raw sql_calls и исключены из SQL/HTTP. Пароли не измеряются: пользователь и сессия создаются напрямую, токен остаётся в памяти.',
|
||||
'',
|
||||
'CPU/RSS относятся к процессу эксперимента, включающему API, генератор HTTP и инструментацию. Docker Desktop: 4 CPU, около 2.80 GiB RAM, отдельные лимиты сервисов не заданы. Замеры closed-loop, короткие и последовательные before → after; разброс и прогрев ограничивают выводы о latency/RPS. Это не тест максимальной ёмкости или SLA. Первый холодный повтор также включает прогрев runtime.',
|
||||
'',
|
||||
'Первичные короткие замеры (1 с прогрева, 5 с чтений, 128 записей) сохранены отдельно в cache-epochs-before-short.json и cache-epochs-after-short.json. Из-за большого разброса сделано одно повторное сравнение с увеличенным прогревом и нагрузкой; основные результаты получены в отдельных контейнерах benchmark из соответствующих образов. PostgreSQL/Redis и остальные контейнеры разделяют ресурсы Docker-host. Другие проекты на этом host не останавливались. Формулировки provenance в основных JSON уточнены после перехода от docker exec к отдельному контейнеру; численные измерения не менялись.',
|
||||
'',
|
||||
'До изменений использован фактически запущенный образ (локальный тег lab2-cache-epochs:before), после — пересобранный проект. Старый образ ещё использовал OR для курсорной пагинации; измеряемая первая страница не содержит курсора, поэтому этот путь в нагрузочном сравнении не выполняется. Хеши измеренных модулей записаны в JSON.',
|
||||
'',
|
||||
'Эпоха истории читается JOIN-ом вместе с проверкой членства; эпоха списка чатов — JOIN-ом с проверкой сессии. Для отдельного запроса дополнительного чтения эпохи нет. Однако raw JOIN проверки членства не использует прежний batching findUnique в Prisma: под конкуренцией суммарный SQL/HTTP может вырасти. Запись добавляет UPDATE чата и транзакционные команды; обновления одного чата сериализуются. Старые ключи недоступны новым читателям и удаляются по TTL.',
|
||||
'',
|
||||
'Корректность зависит от основного PostgreSQL и обновления эпох во всех путях изменения данных. Прямые SQL-изменения в обход приложения требуют увеличения соответствующей эпохи. После восстановления PostgreSQL из старого backup следует сменить CACHE_PREFIX или очистить кэш, чтобы исключить повторное использование эпох. Сохранность Docker-host и высокая доступность PostgreSQL не исследовались. Cache stampede не устраняется этим изменением.',
|
||||
);
|
||||
await writeFile('evidence/cache-epochs-summary.md', lines.join('\n') + '\n');
|
||||
console.log(lines.slice(0, 14).join('\n'));
|
||||
@@ -0,0 +1,117 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
|
||||
const resultPrefix = process.argv[2] ?? 'cache-strategy';
|
||||
assert.match(resultPrefix, /^[a-z0-9-]+$/);
|
||||
|
||||
const reports = await Promise.all(
|
||||
['full', 'no-redis-no-epochs'].map(async (name) =>
|
||||
JSON.parse(await readFile(`evidence/${resultPrefix}-${name}.json`, 'utf8')),
|
||||
),
|
||||
);
|
||||
const provenance = JSON.parse(
|
||||
await readFile(`evidence/${resultPrefix}-experiment.json`, 'utf8'),
|
||||
);
|
||||
assert.ok(reports.every((report) => report.completed));
|
||||
assert.deepEqual(reports[0].methodology, reports[1].methodology);
|
||||
assert.equal(provenance.exit_code, 0);
|
||||
assert.ok(provenance.sources_unchanged);
|
||||
assert.ok(reports[1].correctness.epochs_unchanged);
|
||||
assert.ok(reports[1].correctness.no_redis_client);
|
||||
const median = (values) =>
|
||||
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
|
||||
const labels = ['Redis и эпохи', 'Без Redis и эпох'];
|
||||
const aggregates = [];
|
||||
const settings = reports[0].methodology;
|
||||
const lines = [
|
||||
'# Полная стратегия кэша и вариант без Redis и эпох',
|
||||
'',
|
||||
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
|
||||
'',
|
||||
'Сначала выполнен свежий прогон текущего образа с Redis, PostgreSQL-эпохами и записью сообщения через CTE. Затем шесть модулей приложения заменены временными версиями без поддержания эпох: убраны изменения счётчиков и чтения эпох в проверках сессии/членства. Объединённые проверки сессии, публичного профиля и доступа к истории сохранены. Сообщение вставляется одним INSERT ... SELECT с проверкой членства, без обновления счётчиков. Второй API запущен без CACHE_URL: все кэши приложения отключены. Исходники рабочего проекта и запущенный backend не изменены.',
|
||||
'',
|
||||
`Параметры обоих прогонов совпадают: ${settings.repetitions} повтора, конкурентность ${settings.concurrency}, ${settings.seed_messages} сообщений, limit=50, прогрев ${settings.warmup_seconds} с и чтение ${settings.duration_seconds} с, ${settings.writes_per_repeat} записей в новый чат. RPS = общее число успешных запросов / общее время повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее отдельных p95, не перцентиль объединённой выборки. SQL/HTTP исключает команды транзакций.`,
|
||||
'',
|
||||
'| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | Медиана RPS | Ошибки |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
||||
];
|
||||
for (const [workload, label] of [
|
||||
['cold-history-burst', 'Холодная волна истории'],
|
||||
['warm-history', 'Прогретая история'],
|
||||
['history-cache-bypassed', 'Контроль без кэша истории'],
|
||||
['writes-single-chat', 'Запись в один чат'],
|
||||
]) {
|
||||
if (
|
||||
reports.every(
|
||||
(report) =>
|
||||
!report.samples.some((sample) => sample.workload === workload),
|
||||
)
|
||||
)
|
||||
continue;
|
||||
for (const [index, report] of reports.entries()) {
|
||||
const samples = report.samples.filter(
|
||||
(sample) => sample.workload === workload,
|
||||
);
|
||||
assert.equal(samples.length, settings.repetitions);
|
||||
const sum = (select) =>
|
||||
samples.reduce((total, sample) => total + select(sample), 0);
|
||||
const successes = sum((sample) => sample.successes);
|
||||
const errors = sum((sample) => sample.errors);
|
||||
const aggregate = {
|
||||
workload,
|
||||
variant: labels[index],
|
||||
successes,
|
||||
errors,
|
||||
mean_rps: successes / sum((sample) => sample.elapsed_seconds),
|
||||
mean_latency_ms:
|
||||
sum((sample) => sample.latency_ms.mean * sample.successes) / successes,
|
||||
mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length,
|
||||
sql_per_request:
|
||||
sum((sample) => sample.sql_queries) / (successes + errors),
|
||||
median_rps: median(samples.map((sample) => sample.successful_rps)),
|
||||
repeat_rps: samples.map((sample) => sample.successful_rps),
|
||||
};
|
||||
aggregates.push(aggregate);
|
||||
lines.push(
|
||||
`| ${label} | ${labels[index]} | ${aggregate.mean_rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.median_rps.toFixed(3)} | ${errors} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
`Учтено успешных HTTP-запросов: ${reports.map((report, index) => `${labels[index]} — ${report.samples.reduce((sum, sample) => sum + sample.successes, 0)}`).join('; ')}.`,
|
||||
'',
|
||||
...['warm-history', 'writes-single-chat'].map((workload) => {
|
||||
const pair = aggregates.filter(
|
||||
(aggregate) => aggregate.workload === workload,
|
||||
);
|
||||
return `${workload}: отношение среднего RPS полной стратегии к варианту без стратегии — ${(pair[0].mean_rps / pair[1].mean_rps).toFixed(3)}. RPS по повторам: ${pair.map((aggregate) => `${aggregate.variant}: ${aggregate.repeat_rps.join(' / ')}`).join('; ')}.`;
|
||||
}),
|
||||
'',
|
||||
settings.history_control === false
|
||||
? 'Дополнительный контроль с частично отключённым кэшем в этом прогоне пропущен. Сравниваются только полная стратегия и отсутствие Redis/поддержания эпох.'
|
||||
: 'В контрольном сценарии у первого API отключён только кэш истории; кэш профиля продолжает работать. У второго API отключены все кэши. Поэтому этот контроль не измеряет отдельно чистую стоимость эпох.',
|
||||
'',
|
||||
`Проверки полной стратегии: чтение после записи при отказе Redis, чтение другой репликой после восстановления со старыми ключами, восстановление с пустым кэшем, задержанная загрузка старой истории. Проверки варианта без стратегии: приложение не создало Redis-клиент, эпохи синтетических пользователей/чатов остались равны 0 после всех записей, ${settings.writes_per_repeat} сообщений записаны в каждом новом чате, история читается после записи без повторяющихся ID.`,
|
||||
'',
|
||||
`Оба прогона используют один базовый образ; изменённые модули временной версии скомпилированы TypeScript без полной пересборки контейнера и подключены отдельными bind mounts. Хеши исходников и JS сохранены в [метаданных эксперимента](${resultPrefix}-experiment.json). Колонки эпох остаются в схеме; это исключает миграцию и позволяет сохранить рабочую БД. SQL-проверки профиля и членства во втором варианте не используют эпохи. Временные файлы удалены, хеши рабочих исходников проверены.`,
|
||||
'',
|
||||
'Ограничения: прогоны последовательные, Docker-host разделяется с другими сервисами; большой разброс ограничивает выводы о скорости. CPU/RSS процесса включают API, HTTP-генератор и инструментацию; CPU PostgreSQL отдельно не измерен. Синтетические сессии обходят KDF. Холодная волна означает пустой кэш приложения, а не холодный PostgreSQL. Отдельные цены Redis и эпох этим сравнением не разделяются: меняется вся стратегия. Данные рабочей БД и volumes сохранены, удалены только фикстуры эксперимента.',
|
||||
'',
|
||||
'Повторение:',
|
||||
'',
|
||||
'```powershell',
|
||||
`docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/${resultPrefix}-full.json --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
|
||||
`$env:EXPERIMENT_PREFIX='${resultPrefix}'`,
|
||||
`node test/no-epochs-experiment.cjs --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
|
||||
`node test/cache-strategy-summary.mjs ${resultPrefix}`,
|
||||
'```',
|
||||
'',
|
||||
`Сырые результаты: [полная стратегия](${resultPrefix}-full.json), [без Redis и эпох](${resultPrefix}-no-redis-no-epochs.json).`,
|
||||
);
|
||||
await writeFile(`evidence/${resultPrefix}-summary.md`, lines.join('\n') + '\n');
|
||||
await writeFile(
|
||||
`evidence/${resultPrefix}-averages.json`,
|
||||
JSON.stringify(aggregates, null, 2) + '\n',
|
||||
);
|
||||
console.log(lines.slice(0, 18).join('\n'));
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { CacheService } = require('../dist/cache/cache.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
|
||||
async function isolatedCache(t) {
|
||||
assert.ok(process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance');
|
||||
const previous = process.env.CACHE_PREFIX;
|
||||
const prefix = `test:cache:${randomUUID()}`;
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
const observer = new Redis(process.env.CACHE_URL, {
|
||||
maxRetriesPerRequest: 0,
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 1000,
|
||||
});
|
||||
observer.on('error', () => {});
|
||||
await observer.ping();
|
||||
t.after(async () => {
|
||||
try {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
observer.disconnect();
|
||||
if (previous === undefined) delete process.env.CACHE_PREFIX;
|
||||
else process.env.CACHE_PREFIX = previous;
|
||||
}
|
||||
});
|
||||
return { prefix, observer };
|
||||
}
|
||||
|
||||
test('Redis cache: hits, expiry, invalidation race and unavailable-server fallback', async (t) => {
|
||||
await isolatedCache(t);
|
||||
const cache = new CacheService();
|
||||
t.after(() => cache.onModuleDestroy());
|
||||
await cache.onModuleInit();
|
||||
let reads = 0;
|
||||
const load = async () => ({ value: ++reads });
|
||||
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
|
||||
value: 1,
|
||||
});
|
||||
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
|
||||
value: 1,
|
||||
});
|
||||
assert.equal(reads, 1);
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 2,
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 1100));
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 3,
|
||||
});
|
||||
await cache.invalidate('expiry');
|
||||
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
|
||||
value: 4,
|
||||
});
|
||||
|
||||
let resolveOld;
|
||||
let began;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const pending = cache.remember('race', 'page', 30, () => {
|
||||
began();
|
||||
return new Promise((resolve) => {
|
||||
resolveOld = resolve;
|
||||
});
|
||||
});
|
||||
await started;
|
||||
await cache.invalidate('race');
|
||||
resolveOld({ value: 'old' });
|
||||
await pending;
|
||||
assert.deepEqual(
|
||||
await cache.remember('race', 'page', 30, async () => ({ value: 'new' })),
|
||||
{ value: 'new' },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await cache.remember('race', 'page', 30, async () => {
|
||||
throw new Error('cache miss');
|
||||
}),
|
||||
{ value: 'new' },
|
||||
);
|
||||
await assert.rejects(
|
||||
cache.remember('db-error', 'page', 30, async () => {
|
||||
throw new Error('database failure');
|
||||
}),
|
||||
/database failure/,
|
||||
);
|
||||
|
||||
const previous = process.env.CACHE_URL;
|
||||
process.env.CACHE_URL = 'redis://127.0.0.1:1/0';
|
||||
const offline = new CacheService();
|
||||
process.env.CACHE_URL = previous;
|
||||
t.after(() => offline.onModuleDestroy());
|
||||
await offline.onModuleInit();
|
||||
assert.deepEqual(
|
||||
await offline.remember('offline', 'page', 30, async () => ({
|
||||
value: 'database',
|
||||
})),
|
||||
{ value: 'database' },
|
||||
);
|
||||
await offline.invalidate('offline');
|
||||
});
|
||||
|
||||
test('API cache: fewer SQL reads, write invalidation, permissions, logout and Redis fallback', async (t) => {
|
||||
assert.ok(process.env.DATABASE_URL);
|
||||
const { prefix, observer } = await isolatedCache(t);
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
const prisma = app.get(PrismaService);
|
||||
const userIds = [];
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
});
|
||||
const actors = [];
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `cache_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Cache test',
|
||||
},
|
||||
});
|
||||
userIds.push(user.id);
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const session = await prisma.session.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
actors.push({ id: user.id, token, sessionId: session.id });
|
||||
}
|
||||
const [alice, outsider] = actors;
|
||||
const request = (method, url, payload, actor = alice) =>
|
||||
server.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
authorization: `Bearer ${actor.token}`,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
});
|
||||
const counts = new Map();
|
||||
const tracked = prisma.$extends({
|
||||
query: {
|
||||
$allModels: {
|
||||
$allOperations({ model, operation, args, query }) {
|
||||
const key = `${model}:${operation}`;
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
for (const name of [
|
||||
'User/user.service',
|
||||
'Chat/chat.service',
|
||||
'Auth/auth.service',
|
||||
]) {
|
||||
const exported = require(`../dist/${name}`);
|
||||
const Service = Object.values(exported)[0];
|
||||
app.get(Service).prisma = tracked;
|
||||
}
|
||||
const spy = (model, operation) => ({
|
||||
mock: { callCount: () => counts.get(`${model}:${operation}`) ?? 0 },
|
||||
});
|
||||
const profileReads = spy('User', 'findUnique');
|
||||
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
|
||||
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
|
||||
assert.equal(profileReads.mock.callCount(), 0);
|
||||
|
||||
const chatListReads = spy('Chat', 'findMany');
|
||||
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
|
||||
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
|
||||
assert.equal(chatListReads.mock.callCount(), 1);
|
||||
const created = await request('POST', '/api/v1/chats', {
|
||||
title: 'Cached chat',
|
||||
member_ids: [alice.id],
|
||||
});
|
||||
assert.equal(created.statusCode, 201);
|
||||
const chatId = created.json().id;
|
||||
chatIds.push(chatId);
|
||||
assert.deepEqual(
|
||||
(await request('GET', '/api/v1/chats')).json().items.map((x) => x.id),
|
||||
[chatId],
|
||||
);
|
||||
assert.equal(chatListReads.mock.callCount(), 2);
|
||||
|
||||
const metadataReads = spy('Chat', 'findUnique');
|
||||
const messageReads = spy('Message', 'findMany');
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal((await request('GET', path)).json().items.length, 0);
|
||||
assert.equal((await request('GET', path)).json().items.length, 0);
|
||||
assert.equal(metadataReads.mock.callCount(), 0);
|
||||
assert.equal(messageReads.mock.callCount(), 1);
|
||||
const message = await request('POST', path, {
|
||||
text: 'Invalidate cached empty history',
|
||||
});
|
||||
assert.equal(message.statusCode, 201);
|
||||
assert.deepEqual(
|
||||
(await request('GET', path)).json().items.map((x) => x.id),
|
||||
[message.json().id],
|
||||
);
|
||||
assert.equal(messageReads.mock.callCount(), 2);
|
||||
assert.equal(
|
||||
(await request('GET', path, undefined, outsider)).statusCode,
|
||||
404,
|
||||
);
|
||||
await prisma.chatMember.delete({
|
||||
where: { chatId_userId: { chatId, userId: alice.id } },
|
||||
});
|
||||
assert.equal((await request('GET', path)).statusCode, 404);
|
||||
await prisma.chatMember.create({
|
||||
data: { chatId, userId: alice.id, position: 0 },
|
||||
});
|
||||
await prisma.session.update({
|
||||
where: { id: outsider.sessionId },
|
||||
data: { revokedAt: new Date() },
|
||||
});
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me', undefined, outsider)).statusCode,
|
||||
401,
|
||||
);
|
||||
|
||||
const keys = [];
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, batch] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
keys.push(...batch);
|
||||
} while (cursor !== '0');
|
||||
assert.ok(keys.length >= 4, 'Redis must contain the cached responses');
|
||||
for (const key of keys) {
|
||||
const value = await observer.get(key);
|
||||
assert.ok(
|
||||
!value.includes('password_hash') && !value.includes('passwordHash'),
|
||||
);
|
||||
for (const actor of actors) assert.ok(!value.includes(actor.token));
|
||||
}
|
||||
app.get(CacheService).onModuleDestroy();
|
||||
assert.equal((await request('GET', path)).statusCode, 200);
|
||||
assert.equal(messageReads.mock.callCount(), 3);
|
||||
assert.equal(
|
||||
(await request('POST', path, { text: 'Write while Redis is unavailable' }))
|
||||
.statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal((await request('GET', path)).json().items.length, 2);
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the lab3 migration while a lab2 API uses an isolated temporary database."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
from urllib.parse import quote
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NETWORK = 'mtusi2026lab3-fork_default'
|
||||
OLD_API = 'mtusi2026lab2-fork-backend:latest'
|
||||
OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest'
|
||||
NEW_API = 'mtusi2026lab3-fork-api:latest'
|
||||
NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def request(base, method, path, body=None, token=None, basic=None, key=None):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
if basic:
|
||||
headers['Authorization'] = 'Basic ' + base64.b64encode(
|
||||
(basic[0] + ':' + basic[1]).encode()).decode()
|
||||
if key:
|
||||
headers['Idempotency-Key'] = key
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
headers['Content-Type'] = 'application/json'
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers,
|
||||
method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=3) as response:
|
||||
raw = response.read()
|
||||
return response.status, json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None
|
||||
|
||||
|
||||
def wait_http(base):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if request(base, 'GET', '/health/ready')[0] == 200:
|
||||
return
|
||||
except RuntimeError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError('temporary API did not become ready')
|
||||
|
||||
|
||||
def publish_port(name):
|
||||
value = docker('port', name, '8080/tcp')
|
||||
return 'http://127.0.0.1:' + value.rsplit(':', 1)[1]
|
||||
|
||||
|
||||
def make_user(base, label):
|
||||
username = 'compat_' + label + '_' + uuid.uuid4().hex[:8]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, user = request(base, 'POST', '/api/v1/users',
|
||||
{'username': username, 'display_name': label,
|
||||
'password': password})
|
||||
assert status == 201
|
||||
status, session = request(base, 'POST', '/api/v1/auth/sessions',
|
||||
basic=(username, password))
|
||||
assert status == 201
|
||||
return user, session['token']
|
||||
|
||||
|
||||
def message(base, token, chat_id, text, key=None):
|
||||
status, item = request(base, 'POST',
|
||||
f'/api/v1/chats/{chat_id}/messages',
|
||||
{'text': text}, token=token, key=key)
|
||||
assert status == 201
|
||||
return item
|
||||
|
||||
|
||||
def history(base, token, chat_id):
|
||||
items = []
|
||||
cursor = None
|
||||
for _ in range(100):
|
||||
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
|
||||
if cursor:
|
||||
path += '&cursor=' + quote(cursor, safe='')
|
||||
status, page = request(base, 'GET', path, token=token)
|
||||
assert status == 200
|
||||
items.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return items
|
||||
raise RuntimeError('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def run():
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
names = {label: 'lab3-compat-' + suffix + '-' + label
|
||||
for label in ('pg', 'redis', 'old', 'new')}
|
||||
password = secrets.token_urlsafe(18)
|
||||
db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat'
|
||||
cache_url = f'redis://{names["redis"]}:6379/0'
|
||||
created = []
|
||||
try:
|
||||
print('Starting isolated PostgreSQL and Redis...', flush=True)
|
||||
docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK,
|
||||
'--tmpfs', '/var/lib/postgresql/data:rw,mode=0700',
|
||||
'-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app',
|
||||
'-e', 'POSTGRES_PASSWORD=' + password,
|
||||
'postgres:17.6-bookworm')
|
||||
created.append(names['pg'])
|
||||
docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK,
|
||||
'redis:7.4.5-alpine', 'redis-server', '--save', '',
|
||||
'--appendonly', 'no')
|
||||
created.append(names['redis'])
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
docker('exec', names['pg'], 'pg_isready', '-U', 'app',
|
||||
'-d', 'compat', timeout=5)
|
||||
break
|
||||
except RuntimeError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('temporary PostgreSQL did not start')
|
||||
time.sleep(0.3)
|
||||
docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url,
|
||||
OLD_MIGRATE, timeout=90)
|
||||
print('Lab2 schema applied.', flush=True)
|
||||
before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert before_table == 'f', 'new table exists before migration'
|
||||
base_env = ['-e', 'DATABASE_URL=' + db_url,
|
||||
'-e', 'CACHE_URL=' + cache_url,
|
||||
'-e', 'CACHE_PREFIX=compat:' + suffix,
|
||||
'-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080',
|
||||
'-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres']
|
||||
docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env, OLD_API)
|
||||
created.append(names['old'])
|
||||
old_base = publish_port(names['old'])
|
||||
wait_http(old_base)
|
||||
print('Lab2 API ready; preparing synthetic chat.', flush=True)
|
||||
alice, token = make_user(old_base, 'alice')
|
||||
bob, _ = make_user(old_base, 'bob')
|
||||
status, chat = request(old_base, 'POST', '/api/v1/chats',
|
||||
{'title': 'Migration compatibility',
|
||||
'member_ids': [alice['id'], bob['id']]},
|
||||
token=token)
|
||||
assert status == 201
|
||||
pre = message(old_base, token, chat['id'], 'before migration')
|
||||
|
||||
stop = threading.Event()
|
||||
events = []
|
||||
|
||||
def old_writer():
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
at = time.monotonic()
|
||||
try:
|
||||
item = message(old_base, token, chat['id'],
|
||||
f'during migration {index}')
|
||||
events.append((at, True, item['id']))
|
||||
except (RuntimeError, AssertionError):
|
||||
events.append((at, False, None))
|
||||
index += 1
|
||||
stop.wait(0.3)
|
||||
|
||||
thread = threading.Thread(target=old_writer, daemon=True)
|
||||
thread.start()
|
||||
time.sleep(0.5)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
print('Applying lab3 migration while lab2 API writes...', flush=True)
|
||||
docker('run', '--rm', '--network', NETWORK,
|
||||
'-e', 'DATABASE_URL=' + db_url,
|
||||
NEW_MIGRATE, timeout=90)
|
||||
finally:
|
||||
migrated = time.monotonic()
|
||||
time.sleep(0.7)
|
||||
stop.set()
|
||||
thread.join(timeout=5)
|
||||
after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert after_table == 't', 'new table was not created'
|
||||
assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration'
|
||||
assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration'
|
||||
assert all(ok for _, ok, _ in events), 'old API write failed near migration'
|
||||
after = message(old_base, token, chat['id'], 'after migration')
|
||||
print('Lab2 API still reads and writes; starting lab3 API.', flush=True)
|
||||
|
||||
docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env,
|
||||
'-e', 'INSTANCE_ID=compat-new', NEW_API)
|
||||
created.append(names['new'])
|
||||
new_base = publish_port(names['new'])
|
||||
wait_http(new_base)
|
||||
print('Lab3 API ready; checking mixed-version reads and writes.', flush=True)
|
||||
status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token)
|
||||
assert status == 200 and profile['id'] == alice['id'], 'session not shared'
|
||||
key = uuid.uuid4().hex
|
||||
new_message = message(new_base, token, chat['id'], 'from new version', key)
|
||||
replay = message(new_base, token, chat['id'], 'from new version', key)
|
||||
assert replay == new_message, 'new API idempotent replay changed response'
|
||||
old_after_new = message(old_base, token, chat['id'], 'from old version')
|
||||
for version, base in (('old', old_base), ('new', new_base)):
|
||||
items = history(base, token, chat['id'])
|
||||
ids = [item['id'] for item in items]
|
||||
for label, item in (('pre', pre), ('after', after),
|
||||
('new', new_message), ('old-after-new', old_after_new)):
|
||||
assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message'
|
||||
result = {
|
||||
'old_image': OLD_API, 'new_image': NEW_API,
|
||||
'migration_image': NEW_MIGRATE,
|
||||
'migration_seconds': round(migrated - started, 3),
|
||||
'old_writes_near_migration': len(events),
|
||||
'old_write_errors': sum(not ok for _, ok, _ in events),
|
||||
'old_and_new_read_each_others_messages': True,
|
||||
'existing_session_accepted_by_new_api': True,
|
||||
'new_api_idempotent_replay': True,
|
||||
'temporary_database': True,
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
print('PASS: additive migration compatible with running lab2 API')
|
||||
finally:
|
||||
print('Stopping isolated test containers...', flush=True)
|
||||
for name in reversed(created):
|
||||
try:
|
||||
docker('stop', name, timeout=30)
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
print(f'WARNING: stop temporary container {name} manually',
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
run()
|
||||
except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROBE = r'''
|
||||
import json, sys, threading, time, urllib.error, urllib.request
|
||||
url = sys.argv[1]
|
||||
live_url = url.replace('/health/ready', '/health/live')
|
||||
phase = 'before'
|
||||
events = []
|
||||
def commands():
|
||||
global phase
|
||||
for line in sys.stdin:
|
||||
phase = line.strip()
|
||||
if phase == 'end': break
|
||||
def status(address):
|
||||
try:
|
||||
with urllib.request.urlopen(address, timeout=1) as response:
|
||||
return str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
return str(error.code)
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return 'connection_error'
|
||||
threading.Thread(target=commands, daemon=True).start()
|
||||
print('READY', flush=True)
|
||||
while phase != 'end':
|
||||
current = phase
|
||||
ready = status(url)
|
||||
live = status(live_url)
|
||||
events.append({'phase': current, 'ready': ready, 'live': live})
|
||||
time.sleep(0.05)
|
||||
print(json.dumps(events), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def command(*args, timeout=45):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def both_ready(base_url, expected):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response:
|
||||
seen.add(response.headers['X-Instance-Id'])
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if seen == expected:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError(f'both replicas did not recover: {seen}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
args = parser.parse_args()
|
||||
ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines()
|
||||
if len(ids) != 2:
|
||||
raise RuntimeError('shutdown probe requires two API replicas')
|
||||
expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids}
|
||||
target = ids[0]
|
||||
networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target))
|
||||
network, settings = next(iter(networks.items()))
|
||||
address = settings['IPAddress']
|
||||
nginx = command('docker', 'compose', 'ps', '-q', 'nginx')
|
||||
both_ready(args.base_url, expected)
|
||||
probe = subprocess.Popen(
|
||||
['docker', 'run', '--rm', '-i', '--network', network,
|
||||
'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE,
|
||||
f'http://{address}:8080/health/ready'],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
stop = None
|
||||
|
||||
def phase(value):
|
||||
probe.stdin.write(value + '\n')
|
||||
probe.stdin.flush()
|
||||
|
||||
try:
|
||||
if probe.stdout.readline().strip() != 'READY':
|
||||
raise RuntimeError('direct readiness probe did not start')
|
||||
time.sleep(1)
|
||||
phase('stopping')
|
||||
started = time.monotonic()
|
||||
stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
withdrawn_at = None
|
||||
withdrawn_while_running = False
|
||||
while stop.poll() is None and time.monotonic() - started < 35:
|
||||
upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10)
|
||||
if f'server {address}:8080 ' not in upstreams:
|
||||
withdrawn_at = time.monotonic() - started
|
||||
withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true'
|
||||
if withdrawn_while_running:
|
||||
phase('withdrawn')
|
||||
break
|
||||
time.sleep(0.1)
|
||||
stop.communicate(timeout=40)
|
||||
if stop.returncode:
|
||||
raise RuntimeError('docker stop failed')
|
||||
stop_seconds = time.monotonic() - started
|
||||
exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target))
|
||||
phase('stopped')
|
||||
time.sleep(1)
|
||||
phase('end')
|
||||
output, _ = probe.communicate(timeout=15)
|
||||
if probe.returncode:
|
||||
raise RuntimeError('direct probe failed')
|
||||
events = json.loads(output.strip())
|
||||
summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events)
|
||||
for value in sorted({e['ready'] for e in events if e['phase'] == name})}
|
||||
for name in ('before', 'stopping', 'withdrawn', 'stopped')}
|
||||
draining = [e for e in events if e['ready'] == '503']
|
||||
first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None)
|
||||
if summary['before'].get('200', 0) < 5:
|
||||
raise RuntimeError('readiness was not healthy before SIGTERM')
|
||||
if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]):
|
||||
raise RuntimeError('expected readiness 200 -> 503 with no late 200')
|
||||
# These are separate requests; transport failures can straddle HTTP
|
||||
# close or the client's deadline. Do not confuse them with unhealthy
|
||||
# HTTP responses. Record them separately rather than hiding them.
|
||||
if not draining or not any(e['live'] == '200' for e in draining):
|
||||
raise RuntimeError('liveness must remain 200 while readiness is 503')
|
||||
if any(e['live'] not in {'200', 'connection_error'} for e in draining):
|
||||
raise RuntimeError('liveness returned an unhealthy HTTP status during draining')
|
||||
if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events):
|
||||
raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive')
|
||||
if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0):
|
||||
raise RuntimeError('replica did not close HTTP after draining')
|
||||
if exit_code == 137:
|
||||
raise RuntimeError('container required SIGKILL instead of graceful shutdown')
|
||||
print(json.dumps({'stop_seconds': round(stop_seconds, 3),
|
||||
'lb_withdrawn_seconds': round(withdrawn_at, 3),
|
||||
'lb_withdrawn_before_http_close': True,
|
||||
'readiness_sequence': [200, 503, 'connection_error'],
|
||||
'readiness_503_samples': len(draining),
|
||||
'liveness_200_while_draining': sum(e['live'] == '200' for e in draining),
|
||||
'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining),
|
||||
'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2))
|
||||
print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close')
|
||||
finally:
|
||||
if probe.poll() is None:
|
||||
probe.kill()
|
||||
probe.communicate()
|
||||
if stop is not None and stop.poll() is None:
|
||||
stop.communicate(timeout=40)
|
||||
command('docker', 'start', target)
|
||||
both_ready(args.base_url, expected)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,12 @@
|
||||
services:
|
||||
benchmark:
|
||||
image: ${BENCHMARK_IMAGE:-mtusi2026lab2-fork-backend}
|
||||
working_dir: /app
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
volumes:
|
||||
- ./test:/app/test:ro
|
||||
- ./evidence:/results
|
||||
command: ['node', 'test/cache-benchmark.mjs', '${BENCHMARK_STAGE:-after}', '/results/cache-epochs-${BENCHMARK_STAGE:-after}.json']
|
||||
@@ -0,0 +1,5 @@
|
||||
services:
|
||||
api:
|
||||
command: ['sh', '-c', 'INSTANCE_ID=$$(hostname) exec node --require /app/diagnostics/pool.cjs dist/main.js']
|
||||
volumes:
|
||||
- ./scripts/diagnostics/pool.cjs:/app/diagnostics/pool.cjs:ro
|
||||
@@ -0,0 +1,35 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
import tempfile
|
||||
|
||||
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).resolve().parents[1] / "scripts/run-scaling-diagnostics.py")
|
||||
diagnostics = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diagnostics)
|
||||
|
||||
|
||||
class SummaryTest(unittest.TestCase):
|
||||
def test_cumulative_counters_become_interval_rates(self):
|
||||
summary = diagnostics.resource_summary([
|
||||
{"time_ms": 1000, "cpu_usage_usec": 100, "cpu_nr_periods": 10,
|
||||
"cpu_nr_throttled": 1, "cpu_throttled_usec": 100, "memory_bytes": 1024**2},
|
||||
{"time_ms": 3000, "cpu_usage_usec": 1000100, "cpu_nr_periods": 30,
|
||||
"cpu_nr_throttled": 6, "cpu_throttled_usec": 500100, "memory_bytes": 2*1024**2},
|
||||
])
|
||||
self.assertEqual(summary["mean_cpu_cores"], 0.5)
|
||||
self.assertEqual(summary["throttled_period_fraction"], 0.25)
|
||||
self.assertEqual(summary["throttled_seconds"], 0.5)
|
||||
self.assertEqual(summary["peak_memory_mib"], 2)
|
||||
|
||||
def test_short_sample_does_not_invent_rates(self):
|
||||
self.assertEqual(diagnostics.resource_summary([]), {"samples": 0})
|
||||
|
||||
def test_postgres_multiline_json_and_incomplete_tail(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "postgres.jsonl"
|
||||
path.write_text('{"kind":"postgres",\n"activity":[\n{"state":"active"}\n]}\n{"kind":"cgroup"}\n{"unfinished":', encoding="utf-8")
|
||||
self.assertEqual([r["kind"] for r in diagnostics.rows(path)], ["postgres", "cgroup"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user