diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..68445fc --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,87 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab3-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting + run: test -z "$(gofmt -l cmd internal migrations)" + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab3.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 120 + network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)") + docker network connect "$network" "$HOSTNAME" + printf '%s\n' "$network" > .ci-network + + - name: Integration checks + run: | + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + docker compose up -d --scale api=2 --wait --wait-timeout 90 + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi + if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..55344f4 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,117 @@ +name: Deploy production + +on: + workflow_dispatch: + +concurrency: + group: production + cancel-in-progress: false + +jobs: + verify: + name: Verify selected commit + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab3-deploy-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting + run: test -z "$(gofmt -l cmd internal migrations)" + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab3.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 120 + network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)") + docker network connect "$network" "$HOSTNAME" + printf '%s\n' "$network" > .ci-network + + - name: Integration checks + run: | + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi + if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi + + deploy: + name: Deploy selected commit + needs: verify + runs-on: ubuntu-latest + environment: production + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + DEPLOY_SHA: ${{ github.sha }} + steps: + - name: Configure SSH + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + + - name: Deploy and verify + run: | + ssh "$DEPLOY_USER@$DEPLOY_HOST" " + set -eu + cd ~/mtusi2026lab3 + git fetch origin main + git cat-file -e '$DEPLOY_SHA^{commit}' + git reset --hard '$DEPLOY_SHA' + git clean -fd + scripts/rolling-update.sh + test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA' + docker compose ps + " diff --git a/Dockerfile b/Dockerfile index 361ff16..39cf6e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,14 @@ FROM golang:1.26.5-alpine3.24 AS build WORKDIR /src COPY . . -RUN go build ./cmd/messenger +RUN go build -o /out/messenger ./cmd/messenger +RUN go build -o /out/lab3experiment ./cmd/lab3experiment +RUN go build -o /out/lab3summary ./cmd/lab3summary FROM build AS test CMD ["go", "test", "./internal/postgres", "./internal/session"] -FROM build AS experiment - -CMD ["go", "run", "./cmd/lab2experiment"] - FROM swaggerapi/swagger-ui:v5.31.2 AS swagger COPY contracts/openapi.json /spec/openapi.json @@ -26,10 +24,16 @@ FROM grafana/grafana:13.1.6 AS grafana COPY monitoring/grafana/provisioning /etc/grafana/provisioning COPY monitoring/grafana/dashboards /etc/grafana/dashboards +FROM nginx:1.30.5-alpine AS loadbalancer + +COPY nginx/nginx.conf /etc/nginx/nginx.conf + FROM alpine:3.24.2 RUN addgroup -S app && adduser -S -G app -u 10001 app -COPY --from=build --chown=app:app /src/messenger /app/messenger +COPY --from=build --chown=app:app /out/messenger /app/messenger +COPY --from=build --chown=app:app /out/lab3experiment /app/lab3experiment +COPY --from=build --chown=app:app /out/lab3summary /app/lab3summary USER app WORKDIR /app diff --git a/REPORT.md b/REPORT.md index c267900..66e6902 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,197 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Пластинин Денис, БВТ2303 +- Целевая оценка: 5 +- Выбранный трек: 1 — цена масштабирования +- Ветка/PR: `solution/bvt2303-plastinin`, PR #1 +- Среда эксперимента: Docker, `linux/arm64`, Go 1.26.5 ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Git, Docker Engine с Docker Compose v2 и свободные локальные порты 8080, 8081, 9090 и 3000. Все внешние образы закреплены по версиям. Локальные пароли из примера предназначены только для изолированного стенда. Compose передаёт каждому контейнеру только нужные ему переменные: API получает строки подключения PostgreSQL/Valkey без отдельных исходных паролей и пароля Grafana, а `migrate` — только настройки PostgreSQL и миграций. + +```sh +cp lab3.env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120 +``` + +Compose сначала ожидает PostgreSQL, затем однократно запускает контейнер `migrate`. API-реплики стартуют только после успешного завершения миграций. Единственная публичная точка API — Nginx на `http://localhost:8080`; порты отдельных API-контейнеров на хост не публикуются. + +Доступные интерфейсы: + +- Swagger UI: `http://localhost:8081`; +- Prometheus: `http://localhost:9090`; +- Grafana: `http://localhost:3000`, логин `admin`, локальный пароль берётся из `.env`; +- Nginx API: `http://localhost:8080`. + +Проверки запущенного стенда: + +```sh +make check +make test +docker compose --profile tests run --rm --build tests +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Безопасная остановка сохраняет данные в named volumes: + +```sh +docker compose stop +``` + +Повторный запуск выполняется командой `docker compose up -d --scale api=2 --wait`. Намеренное удаление всех учебных данных: + +```sh +docker compose down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```mermaid +flowchart LR + C[Клиент] -->|HTTP :8080| N[Nginx] + N --> A1[API replica 1] + N --> A2[API replica 2] + N -. scale .-> AN[API replica N] + A1 --> P[(PostgreSQL)] + A2 --> P + AN --> P + A1 --> V[(Valkey)] + A2 --> V + AN --> V + M[Однократный migrate] --> P + PR[Prometheus] --> A1 + PR --> A2 + PR --> NE[Nginx exporter] + PR --> PE[PostgreSQL exporter] + PR --> VE[Valkey exporter] + G[Grafana] --> PR +``` + +PostgreSQL является источником истины для пользователей, чатов, сообщений и записей идемпотентности. Valkey хранит общие сессии, поэтому запросы с одним bearer token могут обслуживаться разными API-репликами. В Valkey ключ имеет вид `session:`, а значением служит JSON с `user_id` и сроком действия. Исходный случайный 256-битный токен возвращается клиенту один раз и не записывается в хранилище. + +API остаётся stateless относительно пользовательских данных. Локально в процессе находятся только конфигурация, подключения, метрики конкретной реплики и флаг readiness. Каждый HTTP-ответ содержит `X-Instance-Id`, равный hostname контейнера. + +### Балансировка и отказ реплики + +Nginx OSS 1.30.5 использует round-robin, Docker DNS `127.0.0.11`, повторное разрешение имени `api` каждые 2 секунды и общий upstream zone. Keep-alive соединения ограничены 32. Таймаут подключения равен 1 секунде, таймауты отправки и чтения — 10 секунд. Ошибка соединения или ответ 502/503/504 временно исключают backend на 3 секунды; небезопасные POST автоматически не повторяются. + +Docker проверяет `/health/ready` каждой API-реплики раз в 2 секунды с таймаутом 1 секунду. Readiness обращается к PostgreSQL и Valkey. При SIGTERM приложение сначала снимает readiness, затем вызывает `http.Server.Shutdown` с лимитом 5 секунд; Compose даёт контейнеру 8 секунд до принудительной остановки. + +Nginx OSS в этом стенде использует пассивное исключение backend. Docker healthcheck не передаёт своё состояние напрямую в Nginx. Поэтому верхняя практическая граница обнаружения отказа складывается из connect timeout и fail timeout; в проверках восстановление произошло примерно за 1.4 секунды, что ниже требуемых 10 секунд. + +### Общая идемпотентность + +Для `POST /api/v1/chats/{chat_id}/messages` обязателен один заголовок `Idempotency-Key` длиной до 128 символов. Область ключа — `(user_id, chat_id, key)`, поэтому одинаковая строка независима для другого пользователя или чата. Минимальный TTL равен 24 часам. + +Запись сообщения и запись идемпотентности создаются в одной транзакции PostgreSQL: + +1. Проверяется участие пользователя в чате. +2. Из текста считается SHA-256 для выявления повторного использования ключа с другим запросом. +3. Строка идемпотентности блокируется через `SELECT ... FOR UPDATE`; конкурентные вставки разрешаются первичным ключом и `ON CONFLICT DO NOTHING`. +4. Победившая транзакция резервирует UUID сообщения и вставляет сообщение. Отложенный внешний ключ проверяется при commit. +5. Повтор с тем же телом читает исходное сообщение и возвращает тот же объект. Повтор с другим телом получает `409 Conflict`. + +Это позволяет безопасно повторить POST после timeout: подтверждённая ранее запись не дублируется, даже если повтор попал на другую реплику. Каждый успешный запрос также удаляет до 100 истёкших записей по индексу `expires_at`; `SKIP LOCKED` не даёт параллельным репликам ждать одну и ту же пачку. + +### Миграции, обновление и развёртывание + +API не выполняет DDL при старте. Отдельная команда `/app/messenger migrate` берёт advisory lock и применяет versioned SQL один раз. Миграция идемпотентности только добавляет таблицу и индекс, поэтому совместима с уже работающей предыдущей версией. + +`scripts/rolling-update.sh` собирает образы, запускает миграции, временно увеличивает число API с двух до трёх и поочерёдно заменяет старые контейнеры. Скрипт ждёт health каждого состава и затем возвращает две реплики. Workflow deploy запускается только вручную из `main` через `workflow_dispatch`; перед подключением к серверу он повторяет check, formatting, Go tests и build для SHA запуска. На сервере выполняется reset именно на этот SHA, даже если `main` успел измениться. Для SSH используются environment secrets `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY` и заранее проверенный `DEPLOY_KNOWN_HOSTS`; изменяемых actions для SSH нет. + +## Наблюдаемость + +HTTP middleware каждой реплики публикует число запросов по route/status class, histogram latency, in-flight gauge, число replay идемпотентности, Go runtime и process metrics. Prometheus использует DNS service discovery имени `api` с периодом обновления 2 секунды и отдельно опрашивает Nginx, PostgreSQL и Valkey exporters каждые 5 секунд. Данные хранятся в Prometheus TSDB с retention 7 суток в named volume. + +Provisioned dashboard Grafana показывает общий RPS, RPS и p95 по репликам, классы HTTP-ответов, число healthy replicas, in-flight, heap, replay rate, активные соединения и запросы Nginx, соединения PostgreSQL и состояние Valkey. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make test` через Nginx | PASS: регистрация, сессия, чат, пагинация, доступ и идемпотентность | `evidence/lab3-verification.txt` | +| Общие сессии и ≥2 реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Одна сессия обслужена двумя instance ID | `evidence/lab3-verification.txt` | +| Конкурентная идемпотентность | Go black-box через Nginx и database integration suite | 16 одновременных HTTP-запросов обработаны минимум двумя instance ID и вернули один message ID; история содержит одну запись | `evidence/lab3-verification.txt` | +| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | API, PostgreSQL и Valkey перезапущены; старые данные и token доступны | `evidence/lab3-verification.txt` | +| Отказ API | Два запуска failover с остановкой разных реплик | 125–131 успешная запись; 0–1 переходная ошибка; восстановление 1.35–1.44 с; дублей нет | `evidence/lab3-verification.txt` | +| Rolling update | 60 секунд Go-нагрузки и `scripts/rolling-update.sh` | 134894 подтверждения, 2248.16 RPS, 0 ошибок, p95 13.12 ms | `evidence/lab3-rolling-update.json` | +| CPU/RAM-лимиты | `docker compose config` и scale experiment | Лимиты заданы всем сервисам; при 128 MiB API не хватает памяти для Argon2id, поэтому рабочий минимум эксперимента — 256 MiB | `evidence/lab3-verification.txt` | +| Трек 1 | `scripts/run-scale-experiment.sh` | 1/2/4 реплики, два бюджета, concurrency 8/32/64, три повтора | `evidence/lab3-scale-results.json`, `evidence/lab3-scale-rps.svg` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Методика исследования масштабирования + +Гипотеза: при фиксированном суммарном бюджете дробление ресурсов между большим числом процессов сначала улучшит распределение конкурентных запросов, но затем проиграет из-за меньших CPU-квот и пулов каждой реплики. При добавлении одинаковых ресурсов на реплику RPS должен расти до насыщения PostgreSQL или Nginx. + +Используется Go-клиент `cmd/lab3experiment`. Он создаёт отдельного пользователя, сессию и чат, прогревает стенд 2 секунды, затем 6 секунд выполняет closed-loop POST сообщений через Nginx. Для каждого запроса генерируется уникальный idempotency key. Считаются только HTTP-подтверждения; автоматических retry измеряемого трафика нет. Для каждой точки используются concurrency 8, 32 и 64 и три изолированных повтора с новой базой. + +CPU и Nginx RPS считаются по разности монотонных `process_cpu_seconds_total` и `nginx_http_requests_total`, прочитанных непосредственно до и после измеряемого интервала. Поэтому bootstrap, warmup и простой между точками в них не входят. Resident memory и число healthy replicas снимаются после нагрузки с `/metrics` каждой реплики, соединения PostgreSQL — instant query к Prometheus. Короткая контрольная точка дала 2355.97 client RPS и 2359.64 RPS по Nginx-счётчику, расхождение 0.16%. + +Фиксированная серия имеет общий бюджет API 2 CPU и 1 GiB RAM: `1×(2 CPU, 1 GiB)`, `2×(1 CPU, 512 MiB)`, `4×(0.5 CPU, 256 MiB)`. Пулы PostgreSQL соответственно равны 16, 8 и 4 соединениям на реплику. В растущей серии каждая реплика получает 1 CPU, 256 MiB и пул 8, поэтому суммарный бюджет растёт вместе с числом процессов. + +Предварительный прогон с общим лимитом 512 MiB показал, что вариант `4×128 MiB` не выполняет регистрацию: базовое потребление процесса около 71 MiB, а Argon2id временно требует ещё 64 MiB. Cgroup завершает реплику, Nginx наблюдает premature upstream close. Этот результат определил минимальный рабочий лимит 256 MiB и не включён в сравнение throughput. + +Медианные результаты фиксированного суммарного бюджета: + +| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms | +| ---: | ---: | ---: | ---: | +| 8 | 2412 / 5.86 | 2484 / 5.59 | 2242 / 6.35 | +| 32 | 2731 / 18.63 | 2365 / 24.93 | 2013 / 34.58 | +| 64 | 2733 / 32.99 | 2198 / 55.06 | 2103 / 67.89 | + +При фиксированных 2 CPU и 1 GiB увеличение числа процессов не даёт линейного выигрыша. Две реплики дают небольшой выигрыш только при concurrency 8, одна быстрее при 32 и 64, а четыре проигрывают во всех профилях из-за квоты 0.5 CPU на процесс и расходов четырёх runtime. Для рабочего стенда две реплики остаются осознанным компромиссом ради отказоустойчивости, дальнейшее дробление бюджета не оправдано. + +Медианные результаты серии с 1 CPU и 256 MiB на каждую реплику: + +| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms | +| ---: | ---: | ---: | ---: | +| 8 | 2712 / 4.95 | 2554 / 5.28 | 2185 / 7.05 | +| 32 | 2818 / 15.34 | 2471 / 21.94 | 2130 / 24.14 | +| 64 | 2044 / 61.44 | 2239 / 57.76 | 1859 / 67.42 | + +Даже при росте суммарного CPU практический предел не превышает двух реплик. При concurrency 64 переход `1→2` повышает медианный RPS на 9.6%, а `2→4` снижает его на 17.0%. Одновременно число соединений PostgreSQL растёт `9→17→33`, суммарная resident memory — примерно `36→65→397 MiB`, а p95 у четырёх реплик достигает 67.42 ms. Точная средняя загрузка API составляет примерно `0.43→0.52→0.54 core` при доступных `1→2→4 core`. API не упирается в CPU-квоту. Вместе с ростом DB connections и tail latency это указывает на насыщение общего пути PostgreSQL/Nginx/network и расходы дополнительных процессов, но данный эксперимент не разделяет вклад этих компонентов. + +Во всех 54 основных точках было 0 HTTP-ошибок; каждая конфигурация наблюдала все заявленные реплики. Диапазоны трёх повторов заметно пересекаются и доступны в raw JSON, поэтому небольшие отличия нельзя считать точной оценкой абсолютной производительности. + +Для настройки пула конфигурация `4×0.5 CPU, 256 MiB` при concurrency 64 дополнительно сравнивалась с 10 и 4 соединениями PostgreSQL на реплику: + +| Пул на реплику | Median RPS | p50 / p95 / p99, ms | Соединения PostgreSQL | Ошибки | +| ---: | ---: | ---: | ---: | ---: | +| 10 | 1918 | 30.00 / 59.52 / 101.70 | 41 | 0 | +| 4 | 2179 | 23.61 / 66.76 / 80.91 | 17 | 0 | + +Уменьшение пула `10→4` сократило число соединений на 58.5%, повысило медианный RPS на 13.6%, уменьшило p50 на 21.3% и p99 на 20.4%. p95 вырос на 12.2%, то есть распределение задержек изменилось неравномерно. Для данного стенда выбран пул 4: он соответствует CPU-квоте 0.5 core и заметно снижает давление на общую БД. При строгом SLO на tail latency значение следует подбирать повторно на более длинном open-loop тесте. + +Итоговый график находится в `evidence/lab3-scale-rps.svg`, обезличенные исходные точки, медианы и диапазоны — в `evidence/lab3-scale-results.json`. + +### Отказ и обновление | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| SIGTERM одной API-реплики | Nginx исключает backend, другая реплика продолжает работу | 0–1 переходная ошибка, старый instance ID исчез из последующих ответов | 1.35–1.44 с; подтверждённые сообщения сохранены, дублей нет | +| Полный restart API, PostgreSQL и Valkey без удаления volumes | Данные и сессия сохраняются | Старый bearer token и созданные сущности доступны после запуска | В пределах 90-секундного окна теста; потерь не обнаружено | +| Rolling update API 2→3→2 | Во время замены остаются healthy backend | 134894 успешных POST, 0 ошибок; наблюдались старые и новые instance ID | Перерыва по подтверждённым ответам не зафиксировано | +| Остановка PostgreSQL или Valkey | Readiness становится отрицательной | Все API теряют готовность, потому что хранилища общие | Автоматический failover хранилищ не реализован | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Все компоненты работают на одном Docker-хосте. Отказ хоста, Nginx, PostgreSQL или Valkey останавливает сервис; выполнен отказ процесса API, а не отказ узла. +- Nginx является одной публичной точкой и SPOF. Для эксплуатации потребуются несколько LB с внешним virtual IP или управляемый балансировщик. +- PostgreSQL и Valkey не реплицированы. AOF Valkey с `appendfsync everysec` ограничивает возможную потерю примерно последней секундой при аварии хранилища, но этот сценарий в работе не заявляется как защищённый. +- Нагрузочный клиент closed-loop и подвержен coordinated omission. Результаты сравнивают конфигурации на одной машине и не являются абсолютной производительностью production-системы. +- Все реплики и клиент делят ресурсы одного хоста; дельта process CPU показывает потребление API, а не полную стоимость Docker/network/storage. +- Для эксплуатации нужны TLS, внешнее управление secrets, репликация хранилищ, алерты и длительный soak test; при большом потоке следует отдельно измерить стоимость пакетной очистки idempotency records. + +## Источники и инструменты + +- Nginx upstream и динамическое DNS: +- Docker Compose services и healthcheck: +- PostgreSQL explicit locking: +- Valkey persistence: +- Argon2: RFC 9106, +- Go packages: `pgx/v5`, `valkey-go`, `prometheus/client_golang`, `google/uuid`, `viper`, `x/crypto/argon2`. +- OpenAI Codex использовался для подготовки кода, конфигурации и текста; реализация и результаты проверены запуском тестов и экспериментов. diff --git a/cmd/dependencycheck/main.go b/cmd/dependencycheck/main.go index d22455f..eaa3d64 100644 --- a/cmd/dependencycheck/main.go +++ b/cmd/dependencycheck/main.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" "github.com/google/uuid" @@ -19,6 +20,7 @@ type apiClient struct { baseURL string client *http.Client token string + userID string } func main() { @@ -26,7 +28,7 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - fmt.Println("PASS: PostgreSQL and Valkey outage recovery") + fmt.Println("PASS: replica idempotency and dependency recovery") } func run() error { @@ -39,6 +41,9 @@ func run() error { if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil { return err } + if err := client.verifyConcurrentIdempotency(); err != nil { + return err + } if err := verifyOutage(client, "valkey"); err != nil { return err } @@ -62,9 +67,21 @@ func (c *apiClient) registerAndLogin() error { if err != nil { return fmt.Errorf("register: %w", err) } - if err := closeResponse(response, http.StatusCreated); err != nil { - return fmt.Errorf("register: %w", err) + registrationBody := response.Body + defer func() { _ = registrationBody.Close() }() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("register status = %d", response.StatusCode) } + var user struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil { + return fmt.Errorf("decode user: %w", err) + } + if user.ID == "" { + return fmt.Errorf("register returned an empty user id") + } + c.userID = user.ID request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil) if err != nil { @@ -92,6 +109,154 @@ func (c *apiClient) registerAndLogin() error { return nil } +type messageResult struct { + messageID string + instanceID string + err error +} + +func (c *apiClient) verifyConcurrentIdempotency() error { + chatID, err := c.createChat() + if err != nil { + return err + } + + const workers = 16 + var ( + key = uuid.NewString() + start = make(chan struct{}) + results = make(chan messageResult, workers) + group sync.WaitGroup + instances = make(map[string]struct{}) + messageID string + ) + transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers} + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + defer transport.CloseIdleConnections() + + for range workers { + group.Add(1) + go func() { + defer group.Done() + <-start + id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message") + results <- messageResult{messageID: id, instanceID: instance, err: requestErr} + }() + } + close(start) + group.Wait() + close(results) + + for result := range results { + if result.err != nil { + return result.err + } + if result.instanceID == "" { + return fmt.Errorf("message response has an empty X-Instance-Id") + } + instances[result.instanceID] = struct{}{} + if messageID == "" { + messageID = result.messageID + } + if result.messageID != messageID { + return fmt.Errorf("idempotent responses returned different message ids") + } + } + if len(instances) < 2 { + return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances)) + } + return c.verifyMessageHistory(chatID, messageID) +} + +func (c *apiClient) createChat() (string, error) { + payload, err := json.Marshal(map[string]any{ + "title": "Replica check", "member_ids": []string{c.userID}, + }) + if err != nil { + return "", err + } + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload)) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + response, err := c.client.Do(request) + if err != nil { + return "", fmt.Errorf("create chat: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create chat status = %d", response.StatusCode) + } + var chat struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil { + return "", fmt.Errorf("decode chat: %w", err) + } + return chat.ID, nil +} + +func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) { + payload, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", "", err + } + request, err := http.NewRequest( + http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload), + ) + if err != nil { + return "", "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", key) + response, err := client.Do(request) + if err != nil { + return "", "", fmt.Errorf("create message: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode) + } + var message struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err) + } + return message.ID, response.Header.Get("X-Instance-Id"), nil +} + +func (c *apiClient) verifyMessageHistory(chatID, messageID string) error { + request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+c.token) + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("list messages: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("list messages status = %d", response.StatusCode) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil { + return fmt.Errorf("decode messages: %w", err) + } + if len(page.Items) != 1 || page.Items[0].ID != messageID { + return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items)) + } + return nil +} + func verifyOutage(client *apiClient, service string) (returnErr error) { if err := compose("stop", service); err != nil { return err diff --git a/cmd/lab3experiment/main.go b/cmd/lab3experiment/main.go new file mode 100644 index 0000000..efc40d0 --- /dev/null +++ b/cmd/lab3experiment/main.go @@ -0,0 +1,522 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" +) + +type configuration struct { + baseURL string + prometheusURL string + label string + duration time.Duration + warmup time.Duration + concurrency int + replicas int + apiCPUs string + apiMemory string + dbMaxConns int + output string +} + +type apiClient struct { + baseURL string + client *http.Client + token string + chatID string +} + +type collector struct { + mu sync.Mutex + latencies []time.Duration + errors map[string]int + instances map[string]struct{} +} + +type result struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latencySummary `json:"latency_ms"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type latencySummary struct { + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + P99 float64 `json:"p99"` +} + +type runtimeSnapshot struct { + apiCPUSeconds float64 + apiResident float64 + healthyReplicas float64 + nginxRequests float64 +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + settings := parseFlags() + client := &apiClient{ + baseURL: strings.TrimRight(settings.baseURL, "/"), + client: &http.Client{Transport: &http.Transport{ + MaxIdleConns: settings.concurrency * 2, + MaxIdleConnsPerHost: settings.concurrency, + MaxConnsPerHost: settings.concurrency, + IdleConnTimeout: 30 * time.Second, + }, Timeout: 10 * time.Second}, + } + if err := client.waitForReplicas(settings.replicas); err != nil { + return err + } + if err := client.bootstrap(); err != nil { + return err + } + if settings.warmup > 0 { + warmup := newCollector() + if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil { + return fmt.Errorf("warmup: %w", err) + } + } + + var before runtimeSnapshot + if settings.prometheusURL != "" { + snapshot, err := collectRuntimeSnapshot() + if err != nil { + return fmt.Errorf("collect metrics before load: %w", err) + } + before = snapshot + } + measurements := newCollector() + started := time.Now() + if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil { + return err + } + elapsed := time.Since(started) + summary := measurements.result(settings, elapsed) + if settings.prometheusURL != "" { + after, err := collectRuntimeSnapshot() + if err != nil { + return fmt.Errorf("collect metrics after load: %w", err) + } + summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed) + if err != nil { + return err + } + } + return writeResult(settings.output, summary) +} + +func (c *apiClient) waitForReplicas(expected int) error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + instances := make(map[string]struct{}, expected) + for ctx.Err() == nil { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody) + if err != nil { + return err + } + response, err := c.client.Do(request) + if err == nil { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + _ = response.Body.Close() + if response.StatusCode == http.StatusOK { + if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" { + instances[instanceID] = struct{}{} + } + if len(instances) == expected { + return nil + } + } + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err()) +} + +func parseFlags() configuration { + var settings configuration + flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL") + flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL") + flag.StringVar(&settings.label, "label", "manual", "experiment point label") + flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration") + flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration") + flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients") + flag.IntVar(&settings.replicas, "replicas", 2, "API replica count") + flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica") + flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica") + flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica") + flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty") + flag.Parse() + if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 { + fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive") + os.Exit(2) + } + return settings +} + +func (c *apiClient) bootstrap() error { + username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] + password := "Synthetic-" + uuid.NewString() + var user struct { + ID string `json:"id"` + } + if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{ + "username": username, "display_name": "Scale experiment", "password": password, + }, &user, http.StatusCreated); err != nil { + return fmt.Errorf("register: %w", err) + } + + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody) + if err != nil { + return err + } + request.SetBasicAuth(username, password) + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("login: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return responseError("login", response) + } + var session struct { + Token string `json:"token"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return fmt.Errorf("decode session: %w", err) + } + c.token = session.Token + + var chat struct { + ID string `json:"id"` + } + if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{ + "title": "Scale experiment", "member_ids": []string{user.ID}, + }, &chat, http.StatusCreated); err != nil { + return fmt.Errorf("create chat: %w", err) + } + c.chatID = chat.ID + return nil +} + +func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error { + ctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + var sequence atomic.Uint64 + var group sync.WaitGroup + for range concurrency { + group.Add(1) + go func() { + defer group.Done() + for ctx.Err() == nil { + number := sequence.Add(1) + started := time.Now() + instanceID, status, err := client.createMessage(ctx, number) + if err != nil && ctx.Err() != nil { + return + } + measurements.observe(time.Since(started), instanceID, status, err) + } + }() + } + group.Wait() + if len(measurements.latencies) == 0 { + return errors.New("no successful requests") + } + return nil +} + +func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) { + payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)}) + if err != nil { + return "", 0, err + } + request, err := http.NewRequestWithContext( + ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload), + ) + if err != nil { + return "", 0, err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + response, err := c.client.Do(request) + if err != nil { + return "", 0, err + } + defer func() { _ = response.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + if response.StatusCode != http.StatusCreated { + return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status") + } + return response.Header.Get("X-Instance-Id"), response.StatusCode, nil +} + +func newCollector() *collector { + return &collector{errors: make(map[string]int), instances: make(map[string]struct{})} +} + +func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) { + c.mu.Lock() + defer c.mu.Unlock() + if err != nil { + key := "network" + if status != 0 { + key = strconv.Itoa(status) + } + c.errors[key]++ + return + } + c.latencies = append(c.latencies, latency) + if instanceID != "" { + c.instances[instanceID] = struct{}{} + } +} + +func (c *collector) result(settings configuration, elapsed time.Duration) result { + sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] }) + return result{ + Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339), + GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, + DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(), + Concurrency: settings.concurrency, Replicas: settings.replicas, + APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns, + SuccessfulRequests: len(c.latencies), + SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors, + LatencyMS: latencySummary{ + P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99), + }, + ObservedInstances: len(c.instances), + } +} + +func percentile(values []time.Duration, percentile float64) float64 { + if len(values) == 0 { + return 0 + } + index := int(float64(len(values)-1) * percentile) + return float64(values[index].Microseconds()) / 1000 +} + +func collectRuntimeSnapshot() (runtimeSnapshot, error) { + hosts, err := net.LookupHost("api") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err) + } + uniqueHosts := make(map[string]struct{}, len(hosts)) + var snapshot runtimeSnapshot + for _, host := range hosts { + if _, duplicate := uniqueHosts[host]; duplicate { + continue + } + uniqueHosts[host] = struct{}{} + metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err) + } + cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total") + if err != nil { + return runtimeSnapshot{}, err + } + resident, err := requiredMetric(metrics, "process_resident_memory_bytes") + if err != nil { + return runtimeSnapshot{}, err + } + snapshot.apiCPUSeconds += cpuSeconds + snapshot.apiResident += resident + snapshot.healthyReplicas++ + } + nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err) + } + snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total") + if err != nil { + return runtimeSnapshot{}, err + } + return snapshot, nil +} + +func scrapeMetrics(endpoint string) (map[string]float64, error) { + client := &http.Client{Timeout: 5 * time.Second} + response, err := client.Get(endpoint) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status %d", response.StatusCode) + } + metrics := make(map[string]float64) + scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + name, _, _ := strings.Cut(fields[0], "{") + value, err := strconv.ParseFloat(fields[len(fields)-1], 64) + if err != nil { + return nil, fmt.Errorf("parse metric %s: %w", name, err) + } + metrics[name] += value + } + if err := scanner.Err(); err != nil { + return nil, err + } + return metrics, nil +} + +func requiredMetric(metrics map[string]float64, name string) (float64, error) { + value, found := metrics[name] + if !found { + return 0, fmt.Errorf("metric %s not found", name) + } + return value, nil +} + +func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) { + postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`) + if err != nil { + return nil, fmt.Errorf("query PostgreSQL connections: %w", err) + } + seconds := elapsed.Seconds() + return map[string]float64{ + "api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds, + "api_resident_bytes": after.apiResident, + "healthy_replicas": after.healthyReplicas, + "nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds, + "postgres_connections": postgresConnections, + }, nil +} + +func prometheusQuery(baseURL, query string) (float64, error) { + endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query) + client := &http.Client{Timeout: 5 * time.Second} + response, err := client.Get(endpoint) + if err != nil { + return 0, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return 0, fmt.Errorf("prometheus status %d", response.StatusCode) + } + var payload struct { + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil { + return 0, err + } + var total float64 + for _, item := range payload.Data.Result { + if len(item.Value) != 2 { + continue + } + var encoded string + if err := json.Unmarshal(item.Value[1], &encoded); err != nil { + return 0, err + } + value, err := strconv.ParseFloat(encoded, 64) + if err != nil { + return 0, err + } + total += value + } + return total, nil +} + +func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error { + payload, err := json.Marshal(body) + if err != nil { + return err + } + request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if idempotencyKey != "" { + request.Header.Set("Idempotency-Key", idempotencyKey) + } + response, err := c.client.Do(request) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != expected { + return responseError(method+" "+path, response) + } + return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target) +} + +func responseError(action string, response *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body)) +} + +func writeResult(path string, value result) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if path == "" { + _, err = os.Stdout.Write(data) + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} diff --git a/cmd/lab3summary/main.go b/cmd/lab3summary/main.go new file mode 100644 index 0000000..8a3b6c4 --- /dev/null +++ b/cmd/lab3summary/main.go @@ -0,0 +1,366 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "html" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type latency struct { + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + P99 float64 `json:"p99"` +} + +type inputResult struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latency `json:"latency_ms"` + Instances []string `json:"instances"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type rawResult struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latency `json:"latency_ms"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type aggregate struct { + Label string `json:"label"` + Repeats int `json:"repeats"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRPSMedian float64 `json:"successful_rps_median"` + SuccessfulRPSMin float64 `json:"successful_rps_min"` + SuccessfulRPSMax float64 `json:"successful_rps_max"` + LatencyP50MedianMS float64 `json:"latency_p50_median_ms"` + LatencyP95MedianMS float64 `json:"latency_p95_median_ms"` + LatencyP99MedianMS float64 `json:"latency_p99_median_ms"` + TotalErrors int `json:"total_errors"` + ObservedInstancesMin int `json:"observed_instances_min"` + APICPUCoresMedian float64 `json:"api_cpu_cores_median"` + APIResidentBytesMedian float64 `json:"api_resident_bytes_median"` + PostgresConnectionsMedian float64 `json:"postgres_connections_median"` + PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"` +} + +type report struct { + GeneratedAt string `json:"generated_at"` + Method string `json:"method"` + Raw []rawResult `json:"raw"` + Summary []aggregate `json:"summary"` +} + +func main() { + inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files") + outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output") + chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output") + flag.Parse() + + results, err := readResults(*inputDirectory) + if err != nil { + fatal(err) + } + aggregates := summarize(results) + if err := writeJSON(*outputPath, report{ + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Method: "closed-loop HTTP through Nginx; median of three isolated repeats", + Raw: sanitize(results), + Summary: aggregates, + }); err != nil { + fatal(err) + } + if err := writeChart(*chartPath, aggregates); err != nil { + fatal(err) + } +} + +func readResults(directory string) ([]inputResult, error) { + var paths []string + for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} { + matches, err := filepath.Glob(filepath.Join(directory, pattern)) + if err != nil { + return nil, err + } + paths = append(paths, matches...) + } + if len(paths) == 0 { + return nil, fmt.Errorf("no experiment results in %s", directory) + } + + results := make([]inputResult, 0, len(paths)) + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var result inputResult + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + results = append(results, result) + } + sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label }) + return results, nil +} + +func sanitize(results []inputResult) []rawResult { + raw := make([]rawResult, 0, len(results)) + for _, value := range results { + raw = append(raw, rawResult{ + Label: value.Label, Timestamp: value.Timestamp, + DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds, + Concurrency: value.Concurrency, Replicas: value.Replicas, + APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns, + SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS, + Errors: value.Errors, LatencyMS: value.LatencyMS, + ObservedInstances: instanceCount(value), Prometheus: value.Prometheus, + }) + } + return raw +} + +func summarize(results []inputResult) []aggregate { + groups := make(map[string][]inputResult) + for _, value := range results { + groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value) + } + + labels := make([]string, 0, len(groups)) + for label := range groups { + labels = append(labels, label) + } + sort.Strings(labels) + + aggregates := make([]aggregate, 0, len(labels)) + for _, label := range labels { + values := groups[label] + first := values[0] + rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS }) + p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 }) + p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 }) + p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 }) + cpu := metricNumbers(values, "api_cpu_cores") + memory := metricNumbers(values, "api_resident_bytes") + connections := metricNumbers(values, "postgres_connections") + healthy := metricNumbers(values, "healthy_replicas") + aggregate := aggregate{ + Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas, + APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns, + SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps), + LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99), + ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu), + APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections), + PrometheusHealthyMin: minimum(healthy), + } + for _, value := range values { + if count := instanceCount(value); count < aggregate.ObservedInstancesMin { + aggregate.ObservedInstancesMin = count + } + for _, count := range value.Errors { + aggregate.TotalErrors += count + } + } + aggregates = append(aggregates, aggregate) + } + return aggregates +} + +func instanceCount(value inputResult) int { + if value.ObservedInstances > 0 { + return value.ObservedInstances + } + return len(value.Instances) +} + +func withoutRepeat(label string) string { + index := strings.LastIndex(label, "-n") + if index == -1 { + return label + } + if _, err := strconv.Atoi(label[index+2:]); err != nil { + return label + } + return label[:index] +} + +func numbers(values []inputResult, value func(inputResult) float64) []float64 { + result := make([]float64, 0, len(values)) + for _, item := range values { + result = append(result, value(item)) + } + return result +} + +func metricNumbers(values []inputResult, name string) []float64 { + result := make([]float64, 0, len(values)) + for _, item := range values { + if value, ok := item.Prometheus[name]; ok { + result = append(result, value) + } + } + return result +} + +func median(values []float64) float64 { + if len(values) == 0 { + return 0 + } + values = append([]float64(nil), values...) + sort.Float64s(values) + middle := len(values) / 2 + if len(values)%2 == 1 { + return values[middle] + } + return (values[middle-1] + values[middle]) / 2 +} + +func minimum(values []float64) float64 { + if len(values) == 0 { + return 0 + } + minimum := values[0] + for _, value := range values[1:] { + if value < minimum { + minimum = value + } + } + return minimum +} + +func maximum(values []float64) float64 { + if len(values) == 0 { + return 0 + } + maximum := values[0] + for _, value := range values[1:] { + if value > maximum { + maximum = value + } + } + return maximum +} + +func writeJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o644) +} + +func writeChart(path string, values []aggregate) error { + const width, height = 960, 560 + const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0 + plotWidth := width - left - right + plotHeight := height - top - bottom + + var selected []aggregate + var maxRPS float64 + for _, value := range values { + if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") { + selected = append(selected, value) + if value.SuccessfulRPSMedian > maxRPS { + maxRPS = value.SuccessfulRPSMedian + } + } + } + if maxRPS == 0 { + return fmt.Errorf("no scale results for chart") + } + maxRPS *= 1.1 + + var output strings.Builder + fmt.Fprintf(&output, ``, width, height, width, height) + output.WriteString(``) + fmt.Fprintf(&output, `Median successful RPS by replica count`, left) + + for step := 0; step <= 5; step++ { + y := top + plotHeight*float64(5-step)/5 + value := maxRPS * float64(step) / 5 + fmt.Fprintf(&output, `%.0f`, left, y, left+plotWidth, y, left-10, y+4, value) + } + fmt.Fprintf(&output, ``, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight) + for index, replicas := range []int{1, 2, 4} { + x := left + plotWidth*float64(index)/2 + fmt.Fprintf(&output, `%d`, x, top+plotHeight+25, replicas) + } + fmt.Fprintf(&output, `API replicas`, left+plotWidth/2, height-15) + + colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"} + type series struct { + name string + values []aggregate + } + groups := make(map[string][]aggregate) + for _, value := range selected { + name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency) + groups[name] = append(groups[name], value) + } + var names []string + for name := range groups { + names = append(names, name) + } + sort.Strings(names) + allSeries := make([]series, 0, len(names)) + for _, name := range names { + points := groups[name] + sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas }) + allSeries = append(allSeries, series{name: name, values: points}) + } + for index, current := range allSeries { + color := colors[index%len(colors)] + var coordinates []string + for _, point := range current.values { + xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas] + x := left + plotWidth*xIndex/2 + y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS) + coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y)) + fmt.Fprintf(&output, ``, x, y, color) + } + fmt.Fprintf(&output, ``, strings.Join(coordinates, " "), color) + legendY := top + 20 + float64(index*28) + fmt.Fprintf(&output, `%s`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name)) + } + output.WriteString(``) + return os.WriteFile(path, []byte(output.String()), 0o644) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 8341bce..aefd2bc 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "log/slog" "net" "net/http" @@ -9,33 +10,46 @@ import ( "os/signal" "syscall" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/logger" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/messenger" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/password" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/logger" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/messenger" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session" ) func main() { appLogger := logger.New() - if err := run(appLogger); err != nil { + if err := run(appLogger, os.Args[1:]); err != nil { appLogger.Error("server stopped", "error", err) os.Exit(1) } } -func run(logger *slog.Logger) error { +func run(logger *slog.Logger, arguments []string) error { + if len(arguments) == 0 { + return runServer(logger) + } + if len(arguments) == 1 && arguments[0] == "migrate" { + return runMigrations() + } + return fmt.Errorf("usage: messenger [migrate]") +} + +func runServer(logger *slog.Logger) error { settings, err := config.Load() if err != nil { return err } + instanceID, err := os.Hostname() + if err != nil { + return err + } pool, err := database.Open(context.Background(), database.Config{ URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - MigrationTimeout: settings.MigrationTimeout, }) if err != nil { return err @@ -60,7 +74,9 @@ func run(logger *slog.Logger) error { sessions, passwords, settings.SessionTTL, + settings.IdempotencyTTL, settings.RequestTimeout, + instanceID, settings.SwaggerPort, ) server := &http.Server{ @@ -97,3 +113,20 @@ func run(logger *slog.Logger) error { return nil } } + +func runMigrations() error { + settings, err := config.LoadMigration() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + return database.Migrate(context.Background(), pool, settings.MigrationTimeout) +} diff --git a/compose.yaml b/compose.yaml index e5b65f9..e8288df 100644 --- a/compose.yaml +++ b/compose.yaml @@ -32,31 +32,107 @@ services: mem_limit: ${VALKEY_MEMORY} restart: unless-stopped - api: - build: . - image: ${API_IMAGE:-mtusi-messenger:lab2} - env_file: .env - ports: - - "${API_BIND_ADDRESS}:${APP_PORT}" + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab3} + command: ["/app/messenger", "migrate"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} depends_on: postgres: condition: service_healthy + healthcheck: + disable: true + cpus: ${MIGRATE_CPUS} + mem_limit: ${MIGRATE_MEMORY} + restart: "no" + + api: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab3} + environment: + APP_HOST: ${APP_HOST} + APP_PORT: ${APP_PORT} + DATABASE_URL: ${DATABASE_URL} + CACHE_URL: ${CACHE_URL} + SESSION_STORE: ${SESSION_STORE} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} + HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} + HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} + HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} + HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} + HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} + PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} + PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} + PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} + PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} + PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} + SWAGGER_PORT: ${SWAGGER_PORT} + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully valkey: condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + interval: 2s + timeout: 1s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} cpus: ${API_CPUS} mem_limit: ${API_MEMORY} - stop_grace_period: 10s + stop_grace_period: 8s + restart: unless-stopped + + nginx: + build: + context: . + target: loadbalancer + image: ${NGINX_IMAGE:-mtusi-nginx:lab3} + ports: + - "${API_BIND_ADDRESS}:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"] + interval: 2s + timeout: 1s + retries: 10 + cpus: ${NGINX_CPUS} + mem_limit: ${NGINX_MEMORY} restart: unless-stopped swagger: build: context: . target: swagger + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab3} ports: - "${SWAGGER_BIND_ADDRESS}:8080" depends_on: - api: + nginx: condition: service_healthy + cpus: ${SWAGGER_CPUS} + mem_limit: ${SWAGGER_MEMORY} restart: unless-stopped tests: @@ -68,23 +144,30 @@ services: TEST_DATABASE_URL: ${DATABASE_URL} TEST_CACHE_URL: ${CACHE_URL} depends_on: - postgres: - condition: service_healthy + migrate: + condition: service_completed_successfully valkey: condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} experiment: profiles: ["research"] - build: - context: . - target: experiment - env_file: .env - command: ["go", "run", "./cmd/lab2experiment", "-output=/output/lab2-track2.json"] + image: ${API_IMAGE:-mtusi-messenger:lab3} + command: + - /app/lab3experiment + - -base-url=http://nginx:8080 + - -prometheus-url=http://prometheus:9090 + - -output=/output/lab3-scale.json volumes: - ./evidence:/output depends_on: - postgres: + nginx: condition: service_healthy + prometheus: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 @@ -96,6 +179,8 @@ services: depends_on: postgres: condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} restart: unless-stopped valkey-exporter: @@ -107,12 +192,25 @@ services: depends_on: valkey: condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] + depends_on: + nginx: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} restart: unless-stopped prometheus: build: context: . target: prometheus + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab3} command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus @@ -121,16 +219,25 @@ services: - prometheus_data:/prometheus ports: - "${PROMETHEUS_BIND_ADDRESS}:9090" + healthcheck: + test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"] + interval: 5s + timeout: 3s + retries: 20 depends_on: - api + - nginx-exporter - postgres-exporter - valkey-exporter + cpus: ${PROMETHEUS_CPUS} + mem_limit: ${PROMETHEUS_MEMORY} restart: unless-stopped grafana: build: context: . target: grafana + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab3} environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} @@ -141,6 +248,8 @@ services: - "${GRAFANA_BIND_ADDRESS}:3000" depends_on: - prometheus + cpus: ${GRAFANA_CPUS} + mem_limit: ${GRAFANA_MEMORY} restart: unless-stopped volumes: diff --git a/evidence/lab3-rolling-update.json b/evidence/lab3-rolling-update.json new file mode 100644 index 0000000..b85d4df --- /dev/null +++ b/evidence/lab3-rolling-update.json @@ -0,0 +1,29 @@ +{ + "label": "rolling-update", + "timestamp": "2026-10-02T09:54:15Z", + "goos": "linux", + "goarch": "arm64", + "duration_seconds": 60.001996238, + "warmup_seconds": 3, + "concurrency": 16, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 10, + "successful_requests": 134894, + "successful_rps": 2248.158535675018, + "errors": {}, + "latency_ms": { + "p50": 6.41, + "p95": 13.12, + "p99": 22.006 + }, + "observed_instances": 5, + "prometheus": { + "api_cpu_cores": 0.4491430158747135, + "api_resident_bytes": 58523648, + "healthy_replicas": 2, + "nginx_rps": 2253.547396138092, + "postgres_connections": 21 + } +} diff --git a/evidence/lab3-scale-results.json b/evidence/lab3-scale-results.json new file mode 100644 index 0000000..79054b9 --- /dev/null +++ b/evidence/lab3-scale-results.json @@ -0,0 +1,2048 @@ +{ + "generated_at": "2026-10-02T11:34:39Z", + "method": "closed-loop HTTP through Nginx; median of three isolated repeats", + "raw": [ + { + "label": "fixed-r1-c32-n1", + "timestamp": "2026-10-02T11:13:52Z", + "duration_seconds": 6.001228475, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 16389, + "successful_rps": 2730.940851239629, + "errors": {}, + "latency_ms": { + "p50": 10.785, + "p95": 18.628, + "p99": 34.675 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42657932632701506, + "api_resident_bytes": 36622336, + "healthy_replicas": 1, + "nginx_rps": 2736.939623016103, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c32-n2", + "timestamp": "2026-10-02T11:20:24Z", + "duration_seconds": 6.001659422, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 15981, + "successful_rps": 2662.7635585950115, + "errors": {}, + "latency_ms": { + "p50": 11.032, + "p95": 19.337, + "p99": 29.03 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4198838725773999, + "api_resident_bytes": 33845248, + "healthy_replicas": 1, + "nginx_rps": 2668.9285202161873, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c32-n3", + "timestamp": "2026-10-02T11:28:48Z", + "duration_seconds": 6.000657423, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 18345, + "successful_rps": 3057.1650248996393, + "errors": {}, + "latency_ms": { + "p50": 10.149, + "p95": 14.379, + "p99": 20.03 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.426619921708535, + "api_resident_bytes": 34050048, + "healthy_replicas": 1, + "nginx_rps": 3063.331015955583, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c64-n1", + "timestamp": "2026-10-02T11:14:03Z", + "duration_seconds": 6.004250916, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 16412, + "successful_rps": 2733.3967599964303, + "errors": {}, + "latency_ms": { + "p50": 21.54, + "p95": 32.991, + "p99": 66.967 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.41970264655075573, + "api_resident_bytes": 35110912, + "healthy_replicas": 1, + "nginx_rps": 2744.555520837264, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c64-n2", + "timestamp": "2026-10-02T11:20:35Z", + "duration_seconds": 6.002174937, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 17519, + "successful_rps": 2918.775307931349, + "errors": {}, + "latency_ms": { + "p50": 20.995, + "p95": 29.421, + "p99": 36.314 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4281781232595221, + "api_resident_bytes": 35639296, + "healthy_replicas": 1, + "nginx_rps": 2930.2711408126356, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c64-n3", + "timestamp": "2026-10-02T11:28:59Z", + "duration_seconds": 6.005813394, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 13572, + "successful_rps": 2259.810471893593, + "errors": {}, + "latency_ms": { + "p50": 25.128, + "p95": 47.38, + "p99": 80.951 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.422923563115954, + "api_resident_bytes": 35971072, + "healthy_replicas": 1, + "nginx_rps": 2271.2993403404435, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r1-c8-n1", + "timestamp": "2026-10-02T11:13:40Z", + "duration_seconds": 6.000388895, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 14474, + "successful_rps": 2412.1769860718337, + "errors": {}, + "latency_ms": { + "p50": 2.981, + "p95": 5.859, + "p99": 9.624 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42497245505618175, + "api_resident_bytes": 30003200, + "healthy_replicas": 1, + "nginx_rps": 2414.1768564485683, + "postgres_connections": 10 + } + }, + { + "label": "fixed-r1-c8-n2", + "timestamp": "2026-10-02T11:20:12Z", + "duration_seconds": 6.000665371, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 9954, + "successful_rps": 1658.816045318185, + "errors": {}, + "latency_ms": { + "p50": 3.644, + "p95": 12.672, + "p99": 20.831 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4366182478133057, + "api_resident_bytes": 30031872, + "healthy_replicas": 1, + "nginx_rps": 1660.8158235524445, + "postgres_connections": 10 + } + }, + { + "label": "fixed-r1-c8-n3", + "timestamp": "2026-10-02T11:28:37Z", + "duration_seconds": 6.000879206, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_requests": 17507, + "successful_rps": 2917.4058332145005, + "errors": {}, + "latency_ms": { + "p50": 2.655, + "p95": 4.172, + "p99": 5.742 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42660415451128814, + "api_resident_bytes": 29147136, + "healthy_replicas": 1, + "nginx_rps": 2919.5721824366283, + "postgres_connections": 10 + } + }, + { + "label": "fixed-r2-c32-n1", + "timestamp": "2026-10-02T11:14:45Z", + "duration_seconds": 6.002851707, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 14197, + "successful_rps": 2365.0425985777233, + "errors": {}, + "latency_ms": { + "p50": 12.159, + "p95": 24.925, + "p99": 44.989 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.514755344763342, + "api_resident_bytes": 62197760, + "healthy_replicas": 2, + "nginx_rps": 2371.039748225451, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c32-n2", + "timestamp": "2026-10-02T11:21:17Z", + "duration_seconds": 6.002198827, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 15653, + "successful_rps": 2607.877621378903, + "errors": {}, + "latency_ms": { + "p50": 11.861, + "p95": 20.571, + "p99": 43.978 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.49981683154262496, + "api_resident_bytes": 62091264, + "healthy_replicas": 2, + "nginx_rps": 2613.7088177469, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c32-n3", + "timestamp": "2026-10-02T11:29:42Z", + "duration_seconds": 6.001806643, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 13823, + "successful_rps": 2303.139841421246, + "errors": {}, + "latency_ms": { + "p50": 12.392, + "p95": 26.935, + "p99": 49.468 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5165111414603097, + "api_resident_bytes": 62054400, + "healthy_replicas": 2, + "nginx_rps": 2309.3046518193205, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c64-n1", + "timestamp": "2026-10-02T11:14:57Z", + "duration_seconds": 6.00093491, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 13188, + "successful_rps": 2197.657564661037, + "errors": {}, + "latency_ms": { + "p50": 28.424, + "p95": 55.062, + "p99": 78.884 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5149197660602521, + "api_resident_bytes": 65277952, + "healthy_replicas": 2, + "nginx_rps": 2209.155773029373, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c64-n2", + "timestamp": "2026-10-02T11:21:29Z", + "duration_seconds": 6.001721202, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 15112, + "successful_rps": 2517.944351524378, + "errors": {}, + "latency_ms": { + "p50": 24.691, + "p95": 49.451, + "p99": 72.134 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5165184945556889, + "api_resident_bytes": 64376832, + "healthy_replicas": 2, + "nginx_rps": 2529.607672369184, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c64-n3", + "timestamp": "2026-10-02T11:29:54Z", + "duration_seconds": 6.004428561, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 11522, + "successful_rps": 1918.9169931736324, + "errors": {}, + "latency_ms": { + "p50": 30.021, + "p95": 67.901, + "p99": 111.126 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5179510370395763, + "api_resident_bytes": 64458752, + "healthy_replicas": 2, + "nginx_rps": 1930.4085113587548, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r2-c8-n1", + "timestamp": "2026-10-02T11:14:34Z", + "duration_seconds": 6.001018858, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 14010, + "successful_rps": 2334.603561747381, + "errors": {}, + "latency_ms": { + "p50": 3.026, + "p95": 6.321, + "p99": 11.643 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.4949159584860614, + "api_resident_bytes": 59621376, + "healthy_replicas": 2, + "nginx_rps": 2336.6032221857085, + "postgres_connections": 16 + } + }, + { + "label": "fixed-r2-c8-n2", + "timestamp": "2026-10-02T11:21:06Z", + "duration_seconds": 6.000636999, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 15770, + "successful_rps": 2628.0543220041563, + "errors": {}, + "latency_ms": { + "p50": 2.814, + "p95": 5.05, + "p99": 7.998 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5032799018676317, + "api_resident_bytes": 58634240, + "healthy_replicas": 2, + "nginx_rps": 2630.2207586678246, + "postgres_connections": 14 + } + }, + { + "label": "fixed-r2-c8-n3", + "timestamp": "2026-10-02T11:29:31Z", + "duration_seconds": 6.000844108, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_requests": 14908, + "successful_rps": 2484.31716133493, + "errors": {}, + "latency_ms": { + "p50": 2.934, + "p95": 5.586, + "p99": 8.643 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.4949303708857487, + "api_resident_bytes": 58679296, + "healthy_replicas": 2, + "nginx_rps": 2486.3168800051753, + "postgres_connections": 15 + } + }, + { + "label": "fixed-r4-c32-n1", + "timestamp": "2026-10-02T11:15:41Z", + "duration_seconds": 6.0029428, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 12085, + "successful_rps": 2013.1792693410307, + "errors": {}, + "latency_ms": { + "p50": 12.487, + "p95": 34.58, + "p99": 68.218 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5613913229358108, + "api_resident_bytes": 369168384, + "healthy_replicas": 4, + "nginx_rps": 2019.1763279836684, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c32-n2", + "timestamp": "2026-10-02T11:22:13Z", + "duration_seconds": 6.003303354, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 11491, + "successful_rps": 1914.112834618552, + "errors": {}, + "latency_ms": { + "p50": 13.572, + "p95": 34.597, + "p99": 69.614 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5763493523435897, + "api_resident_bytes": 474034176, + "healthy_replicas": 4, + "nginx_rps": 1920.2761080395671, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c32-n3", + "timestamp": "2026-10-02T11:30:38Z", + "duration_seconds": 6.00495133, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13473, + "successful_rps": 2243.64849265148, + "errors": {}, + "latency_ms": { + "p50": 12.734, + "p95": 27.775, + "p99": 41.779 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5795217660823239, + "api_resident_bytes": 374923264, + "healthy_replicas": 4, + "nginx_rps": 2249.6435454040557, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c64-n1", + "timestamp": "2026-10-02T11:15:54Z", + "duration_seconds": 6.003245223, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13699, + "successful_rps": 2281.9324367286467, + "errors": {}, + "latency_ms": { + "p50": 24.118, + "p95": 60.359, + "p99": 87.286 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5680261047700327, + "api_resident_bytes": 437239808, + "healthy_replicas": 4, + "nginx_rps": 2293.259643509985, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c64-n2", + "timestamp": "2026-10-02T11:22:25Z", + "duration_seconds": 6.001440594, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 12619, + "successful_rps": 2102.6618196664267, + "errors": {}, + "latency_ms": { + "p50": 26.308, + "p95": 67.889, + "p99": 121.945 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5698631764212044, + "api_resident_bytes": 438214656, + "healthy_replicas": 4, + "nginx_rps": 2114.159059190714, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c64-n3", + "timestamp": "2026-10-02T11:30:50Z", + "duration_seconds": 6.0011934, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 11940, + "successful_rps": 1989.6042677111523, + "errors": {}, + "latency_ms": { + "p50": 26.312, + "p95": 77.357, + "p99": 93.368 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5665539790802274, + "api_resident_bytes": 365211648, + "healthy_replicas": 4, + "nginx_rps": 2001.2686143392746, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c8-n1", + "timestamp": "2026-10-02T11:15:29Z", + "duration_seconds": 6.000357282, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13452, + "successful_rps": 2241.8665035753115, + "errors": {}, + "latency_ms": { + "p50": 3.164, + "p95": 6.348, + "p99": 9.942 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5682994928034352, + "api_resident_bytes": 615251968, + "healthy_replicas": 4, + "nginx_rps": 2244.0330412311605, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c8-n2", + "timestamp": "2026-10-02T11:22:01Z", + "duration_seconds": 6.001054705, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 9964, + "successful_rps": 1660.3747990662587, + "errors": {}, + "latency_ms": { + "p50": 4.097, + "p95": 9.858, + "p99": 18.19 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5632343256567598, + "api_resident_bytes": 587333632, + "healthy_replicas": 4, + "nginx_rps": 1662.2078101852599, + "postgres_connections": 17 + } + }, + { + "label": "fixed-r4-c8-n3", + "timestamp": "2026-10-02T11:30:26Z", + "duration_seconds": 6.00084231, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 15560, + "successful_rps": 2592.9693193354383, + "errors": {}, + "latency_ms": { + "p50": 2.877, + "p95": 5.636, + "p99": 7.487 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5615878281594106, + "api_resident_bytes": 127115264, + "healthy_replicas": 4, + "nginx_rps": 2595.1356818772997, + "postgres_connections": 17 + } + }, + { + "label": "growing-r1-c32-n1", + "timestamp": "2026-10-02T11:16:36Z", + "duration_seconds": 6.003394308, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 17867, + "successful_rps": 2976.1496718932494, + "errors": {}, + "latency_ms": { + "p50": 10.134, + "p95": 14.455, + "p99": 21.076 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42142825711590765, + "api_resident_bytes": 33411072, + "healthy_replicas": 1, + "nginx_rps": 2982.3128519380275, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c32-n2", + "timestamp": "2026-10-02T11:23:08Z", + "duration_seconds": 6.001408915, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 13653, + "successful_rps": 2274.9657944279575, + "errors": {}, + "latency_ms": { + "p50": 11.761, + "p95": 23.312, + "p99": 55.308 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4382304284293216, + "api_resident_bytes": 32964608, + "healthy_replicas": 1, + "nginx_rps": 2281.1310133830466, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c32-n3", + "timestamp": "2026-10-02T11:31:33Z", + "duration_seconds": 6.001539678, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 16912, + "successful_rps": 2817.9435457195686, + "errors": {}, + "latency_ms": { + "p50": 10.857, + "p95": 15.335, + "p99": 21.528 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4265572065422242, + "api_resident_bytes": 31309824, + "healthy_replicas": 1, + "nginx_rps": 2823.775382527763, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c64-n1", + "timestamp": "2026-10-02T11:16:47Z", + "duration_seconds": 6.003172059, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 12270, + "successful_rps": 2043.9194278306127, + "errors": {}, + "latency_ms": { + "p50": 26.551, + "p95": 61.444, + "p99": 102.98 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42644121721649303, + "api_resident_bytes": 34877440, + "healthy_replicas": 1, + "nginx_rps": 2055.5799298638763, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c64-n2", + "timestamp": "2026-10-02T11:23:19Z", + "duration_seconds": 6.002412878, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 9446, + "successful_rps": 1573.7004754573632, + "errors": {}, + "latency_ms": { + "p50": 28.822, + "p95": 96.357, + "p99": 222.405 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.43315913997344313, + "api_resident_bytes": 36052992, + "healthy_replicas": 1, + "nginx_rps": 1585.362452302802, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c64-n3", + "timestamp": "2026-10-02T11:31:45Z", + "duration_seconds": 6.003872011, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15085, + "successful_rps": 2512.545232870055, + "errors": {}, + "latency_ms": { + "p50": 23.32, + "p95": 36.688, + "p99": 58.614 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.4263915012361512, + "api_resident_bytes": 35610624, + "healthy_replicas": 1, + "nginx_rps": 2523.87125712164, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c8-n1", + "timestamp": "2026-10-02T11:16:25Z", + "duration_seconds": 6.000515988, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15415, + "successful_rps": 2568.9457424707057, + "errors": {}, + "latency_ms": { + "p50": 2.94, + "p95": 5.009, + "p99": 7.398 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.42496345399288354, + "api_resident_bytes": 29818880, + "healthy_replicas": 1, + "nginx_rps": 2571.112222824395, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c8-n2", + "timestamp": "2026-10-02T11:22:56Z", + "duration_seconds": 6.000646429, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 16486, + "successful_rps": 2747.370669987529, + "errors": {}, + "latency_ms": { + "p50": 2.63, + "p95": 4.946, + "p99": 9.012 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.41995475484463013, + "api_resident_bytes": 30105600, + "healthy_replicas": 1, + "nginx_rps": 2749.537103246648, + "postgres_connections": 9 + } + }, + { + "label": "growing-r1-c8-n3", + "timestamp": "2026-10-02T11:31:22Z", + "duration_seconds": 6.000635097, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 16276, + "successful_rps": 2712.3795626461506, + "errors": {}, + "latency_ms": { + "p50": 2.735, + "p95": 4.824, + "p99": 8.061 + }, + "observed_instances": 1, + "prometheus": { + "api_cpu_cores": 0.41662256737622116, + "api_resident_bytes": 30113792, + "healthy_replicas": 1, + "nginx_rps": 2714.545999996507, + "postgres_connections": 9 + } + }, + { + "label": "growing-r2-c32-n1", + "timestamp": "2026-10-02T11:17:29Z", + "duration_seconds": 6.00069083, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15746, + "successful_rps": 2624.0312067535733, + "errors": {}, + "latency_ms": { + "p50": 11.67, + "p95": 21.115, + "p99": 31.725 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5066083366271347, + "api_resident_bytes": 62099456, + "healthy_replicas": 2, + "nginx_rps": 2630.030516003105, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c32-n2", + "timestamp": "2026-10-02T11:24:02Z", + "duration_seconds": 6.002365388, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 14830, + "successful_rps": 2470.6926422120705, + "errors": {}, + "latency_ms": { + "p50": 11.857, + "p95": 21.935, + "p99": 34.466 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5031349817586277, + "api_resident_bytes": 62484480, + "healthy_replicas": 2, + "nginx_rps": 2476.85687874355, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c32-n3", + "timestamp": "2026-10-02T11:32:27Z", + "duration_seconds": 6.003222172, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 14108, + "successful_rps": 2350.071277688505, + "errors": {}, + "latency_ms": { + "p50": 12.949, + "p95": 24.988, + "p99": 42.291 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5080604902856491, + "api_resident_bytes": 62377984, + "healthy_replicas": 2, + "nginx_rps": 2356.234634455904, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c64-n1", + "timestamp": "2026-10-02T11:17:41Z", + "duration_seconds": 6.006687453, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 14149, + "successful_rps": 2355.541238113426, + "errors": {}, + "latency_ms": { + "p50": 24.56, + "p95": 57.759, + "p99": 83.657 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5194210660056459, + "api_resident_bytes": 64593920, + "healthy_replicas": 2, + "nginx_rps": 2366.861953654575, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c64-n2", + "timestamp": "2026-10-02T11:24:13Z", + "duration_seconds": 6.014530011, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 12614, + "successful_rps": 2097.2544782269274, + "errors": {}, + "latency_ms": { + "p50": 28.563, + "p95": 57.99, + "p99": 89.912 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5104305730265314, + "api_resident_bytes": 65654784, + "healthy_replicas": 2, + "nginx_rps": 2108.5604322874497, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c64-n3", + "timestamp": "2026-10-02T11:32:39Z", + "duration_seconds": 6.001685646, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 13439, + "successful_rps": 2239.2042490524, + "errors": {}, + "latency_ms": { + "p50": 28.242, + "p95": 55.31, + "p99": 82.937 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5198539517109522, + "api_resident_bytes": 64917504, + "healthy_replicas": 2, + "nginx_rps": 2250.7010191383156, + "postgres_connections": 17 + } + }, + { + "label": "growing-r2-c8-n1", + "timestamp": "2026-10-02T11:17:18Z", + "duration_seconds": 6.000737232, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15448, + "successful_rps": 2574.3503510903274, + "errors": {}, + "latency_ms": { + "p50": 2.815, + "p95": 5.282, + "p99": 10.559 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5049379572633153, + "api_resident_bytes": 58855424, + "healthy_replicas": 2, + "nginx_rps": 2576.516751567035, + "postgres_connections": 15 + } + }, + { + "label": "growing-r2-c8-n2", + "timestamp": "2026-10-02T11:23:50Z", + "duration_seconds": 6.001116768, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15263, + "successful_rps": 2543.3599428338935, + "errors": {}, + "latency_ms": { + "p50": 2.932, + "p95": 5.415, + "p99": 7.726 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5132378054070886, + "api_resident_bytes": 59572224, + "healthy_replicas": 2, + "nginx_rps": 2545.69284194938, + "postgres_connections": 14 + } + }, + { + "label": "growing-r2-c8-n3", + "timestamp": "2026-10-02T11:32:16Z", + "duration_seconds": 6.000150243, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 15324, + "successful_rps": 2553.9360481643857, + "errors": {}, + "latency_ms": { + "p50": 2.891, + "p95": 5.208, + "p99": 8.66 + }, + "observed_instances": 2, + "prometheus": { + "api_cpu_cores": 0.5149871044654106, + "api_resident_bytes": 58920960, + "healthy_replicas": 2, + "nginx_rps": 2555.93599808464, + "postgres_connections": 15 + } + }, + { + "label": "growing-r4-c32-n1", + "timestamp": "2026-10-02T11:18:24Z", + "duration_seconds": 6.001886675, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 14778, + "successful_rps": 2462.2257633679833, + "errors": {}, + "latency_ms": { + "p50": 12.519, + "p95": 20.369, + "p99": 27.493 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5364979671163151, + "api_resident_bytes": 423374848, + "healthy_replicas": 4, + "nginx_rps": 2468.390491561556, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c32-n2", + "timestamp": "2026-10-02T11:24:57Z", + "duration_seconds": 6.003686343, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 10176, + "successful_rps": 1694.9586335176737, + "errors": {}, + "latency_ms": { + "p50": 16.525, + "p95": 33.496, + "p99": 66.101 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5496622927091513, + "api_resident_bytes": 492273664, + "healthy_replicas": 4, + "nginx_rps": 1701.2880781003853, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c32-n3", + "timestamp": "2026-10-02T11:33:22Z", + "duration_seconds": 6.001351862, + "warmup_seconds": 2, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 12781, + "successful_rps": 2129.686826217956, + "errors": {}, + "latency_ms": { + "p50": 13.923, + "p95": 24.14, + "p99": 39.802 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5415446510608212, + "api_resident_bytes": 369414144, + "healthy_replicas": 4, + "nginx_rps": 2135.3522164136693, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c64-n1", + "timestamp": "2026-10-02T11:18:36Z", + "duration_seconds": 6.006332231, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 11165, + "successful_rps": 1858.871532675962, + "errors": {}, + "latency_ms": { + "p50": 30.388, + "p95": 67.422, + "p99": 98.596 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5410956096011533, + "api_resident_bytes": 397496320, + "healthy_replicas": 4, + "nginx_rps": 1870.3594086951864, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c64-n2", + "timestamp": "2026-10-02T11:25:09Z", + "duration_seconds": 6.040332097, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 10524, + "successful_rps": 1742.288309814433, + "errors": {}, + "latency_ms": { + "p50": 26.708, + "p95": 81.814, + "p99": 125.924 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5512941915319329, + "api_resident_bytes": 414949376, + "healthy_replicas": 4, + "nginx_rps": 1753.7115227921217, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c64-n3", + "timestamp": "2026-10-02T11:33:34Z", + "duration_seconds": 6.006393765, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 13269, + "successful_rps": 2209.1458734057874, + "errors": {}, + "latency_ms": { + "p50": 24.93, + "p95": 59.356, + "p99": 84.385 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5410900662121342, + "api_resident_bytes": 376426496, + "healthy_replicas": 4, + "nginx_rps": 2220.6336317345986, + "postgres_connections": 33 + } + }, + { + "label": "growing-r4-c8-n1", + "timestamp": "2026-10-02T11:18:13Z", + "duration_seconds": 6.000820517, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 13215, + "successful_rps": 2202.198843068647, + "errors": {}, + "latency_ms": { + "p50": 3.125, + "p95": 7.049, + "p99": 13.356 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5615898676610925, + "api_resident_bytes": 614596608, + "healthy_replicas": 4, + "nginx_rps": 2204.365213478022, + "postgres_connections": 19 + } + }, + { + "label": "growing-r4-c8-n2", + "timestamp": "2026-10-02T11:24:45Z", + "duration_seconds": 6.000744805, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 12122, + "successful_rps": 2020.082572066652, + "errors": {}, + "latency_ms": { + "p50": 3.487, + "p95": 7.328, + "p99": 13.364 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5749286317134096, + "api_resident_bytes": 605945856, + "healthy_replicas": 4, + "nginx_rps": 2022.2489698093402, + "postgres_connections": 17 + } + }, + { + "label": "growing-r4-c8-n3", + "timestamp": "2026-10-02T11:33:10Z", + "duration_seconds": 6.000546553, + "warmup_seconds": 2, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_requests": 13110, + "successful_rps": 2184.8009817448374, + "errors": {}, + "latency_ms": { + "p50": 3.294, + "p95": 6.916, + "p99": 10.948 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.561615507893219, + "api_resident_bytes": 614023168, + "healthy_replicas": 4, + "nginx_rps": 2186.96745106312, + "postgres_connections": 19 + } + }, + { + "label": "pool-baseline-r4-c64-n1", + "timestamp": "2026-10-02T11:19:09Z", + "duration_seconds": 6.003512334, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 10, + "successful_requests": 12010, + "successful_rps": 2000.495598548728, + "errors": {}, + "latency_ms": { + "p50": 27.976, + "p95": 56.907, + "p99": 100.328 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5463468412356225, + "api_resident_bytes": 612368384, + "healthy_replicas": 4, + "nginx_rps": 2011.9888705137455, + "postgres_connections": 41 + } + }, + { + "label": "pool-baseline-r4-c64-n2", + "timestamp": "2026-10-02T11:25:42Z", + "duration_seconds": 6.006350196, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 10, + "successful_requests": 9128, + "successful_rps": 1519.7249081611826, + "errors": {}, + "latency_ms": { + "p50": 33.678, + "p95": 92.261, + "p99": 165.149 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5460887049483655, + "api_resident_bytes": 586547200, + "healthy_replicas": 4, + "nginx_rps": 1531.0462593613315, + "postgres_connections": 41 + } + }, + { + "label": "pool-baseline-r4-c64-n3", + "timestamp": "2026-10-02T11:34:06Z", + "duration_seconds": 6.002334631, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 10, + "successful_requests": 11511, + "successful_rps": 1917.7537920911027, + "errors": {}, + "latency_ms": { + "p50": 29.998, + "p95": 59.521, + "p99": 101.701 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5247957992430696, + "api_resident_bytes": 610664448, + "healthy_replicas": 4, + "nginx_rps": 1929.415920963171, + "postgres_connections": 41 + } + }, + { + "label": "pool-tuned-r4-c64-n1", + "timestamp": "2026-10-02T11:19:41Z", + "duration_seconds": 6.002401553, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13059, + "successful_rps": 2175.6291852005656, + "errors": {}, + "latency_ms": { + "p50": 22.884, + "p95": 71.837, + "p99": 93.913 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.5681059439110139, + "api_resident_bytes": 441999360, + "healthy_replicas": 4, + "nginx_rps": 2187.124584065627, + "postgres_connections": 17 + } + }, + { + "label": "pool-tuned-r4-c64-n2", + "timestamp": "2026-10-02T11:28:06Z", + "duration_seconds": 6.002283087, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13783, + "successful_rps": 2296.2928939242815, + "errors": {}, + "latency_ms": { + "p50": 23.605, + "p95": 66.761, + "p99": 80.909 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.564785091083459, + "api_resident_bytes": 566091776, + "healthy_replicas": 4, + "nginx_rps": 2307.955122943704, + "postgres_connections": 17 + } + }, + { + "label": "pool-tuned-r4-c64-n3", + "timestamp": "2026-10-02T11:34:38Z", + "duration_seconds": 6.00190551, + "warmup_seconds": 2, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_requests": 13078, + "successful_rps": 2178.974657000223, + "errors": {}, + "latency_ms": { + "p50": 26.267, + "p95": 63.481, + "p99": 75.128 + }, + "observed_instances": 4, + "prometheus": { + "api_cpu_cores": 0.56981903402208, + "api_resident_bytes": 536719360, + "healthy_replicas": 4, + "nginx_rps": 2190.3043921796097, + "postgres_connections": 17 + } + } + ], + "summary": [ + { + "label": "fixed-r1-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_rps_median": 2730.940851239629, + "successful_rps_min": 2662.7635585950115, + "successful_rps_max": 3057.1650248996393, + "latency_p50_median_ms": 10.785, + "latency_p95_median_ms": 18.628, + "latency_p99_median_ms": 29.03, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.42657932632701506, + "api_resident_bytes_median": 34050048, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "fixed-r1-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_rps_median": 2733.3967599964303, + "successful_rps_min": 2259.810471893593, + "successful_rps_max": 2918.775307931349, + "latency_p50_median_ms": 21.54, + "latency_p95_median_ms": 32.991, + "latency_p99_median_ms": 66.967, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.422923563115954, + "api_resident_bytes_median": 35639296, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "fixed-r1-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "2", + "api_memory_per_replica": "1g", + "db_max_connections_per_replica": 16, + "successful_rps_median": 2412.1769860718337, + "successful_rps_min": 1658.816045318185, + "successful_rps_max": 2917.4058332145005, + "latency_p50_median_ms": 2.981, + "latency_p95_median_ms": 5.859, + "latency_p99_median_ms": 9.624, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.42660415451128814, + "api_resident_bytes_median": 30003200, + "postgres_connections_median": 10, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "fixed-r2-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2365.0425985777233, + "successful_rps_min": 2303.139841421246, + "successful_rps_max": 2607.877621378903, + "latency_p50_median_ms": 12.159, + "latency_p95_median_ms": 24.925, + "latency_p99_median_ms": 44.989, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.514755344763342, + "api_resident_bytes_median": 62091264, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "fixed-r2-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2197.657564661037, + "successful_rps_min": 1918.9169931736324, + "successful_rps_max": 2517.944351524378, + "latency_p50_median_ms": 28.424, + "latency_p95_median_ms": 55.062, + "latency_p99_median_ms": 78.884, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.5165184945556889, + "api_resident_bytes_median": 64458752, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "fixed-r2-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "512m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2484.31716133493, + "successful_rps_min": 2334.603561747381, + "successful_rps_max": 2628.0543220041563, + "latency_p50_median_ms": 2.934, + "latency_p95_median_ms": 5.586, + "latency_p99_median_ms": 8.643, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.4949303708857487, + "api_resident_bytes_median": 58679296, + "postgres_connections_median": 15, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "fixed-r4-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_rps_median": 2013.1792693410307, + "successful_rps_min": 1914.112834618552, + "successful_rps_max": 2243.64849265148, + "latency_p50_median_ms": 12.734, + "latency_p95_median_ms": 34.58, + "latency_p99_median_ms": 68.218, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5763493523435897, + "api_resident_bytes_median": 374923264, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "fixed-r4-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_rps_median": 2102.6618196664267, + "successful_rps_min": 1989.6042677111523, + "successful_rps_max": 2281.9324367286467, + "latency_p50_median_ms": 26.308, + "latency_p95_median_ms": 67.889, + "latency_p99_median_ms": 93.368, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5680261047700327, + "api_resident_bytes_median": 437239808, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "fixed-r4-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_rps_median": 2241.8665035753115, + "successful_rps_min": 1660.3747990662587, + "successful_rps_max": 2592.9693193354383, + "latency_p50_median_ms": 3.164, + "latency_p95_median_ms": 6.348, + "latency_p99_median_ms": 9.942, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5632343256567598, + "api_resident_bytes_median": 587333632, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "growing-r1-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2817.9435457195686, + "successful_rps_min": 2274.9657944279575, + "successful_rps_max": 2976.1496718932494, + "latency_p50_median_ms": 10.857, + "latency_p95_median_ms": 15.335, + "latency_p99_median_ms": 21.528, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.4265572065422242, + "api_resident_bytes_median": 32964608, + "postgres_connections_median": 9, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "growing-r1-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2043.9194278306127, + "successful_rps_min": 1573.7004754573632, + "successful_rps_max": 2512.545232870055, + "latency_p50_median_ms": 26.551, + "latency_p95_median_ms": 61.444, + "latency_p99_median_ms": 102.98, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.42644121721649303, + "api_resident_bytes_median": 35610624, + "postgres_connections_median": 9, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "growing-r1-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 1, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2712.3795626461506, + "successful_rps_min": 2568.9457424707057, + "successful_rps_max": 2747.370669987529, + "latency_p50_median_ms": 2.735, + "latency_p95_median_ms": 4.946, + "latency_p99_median_ms": 8.061, + "total_errors": 0, + "observed_instances_min": 1, + "api_cpu_cores_median": 0.41995475484463013, + "api_resident_bytes_median": 30105600, + "postgres_connections_median": 9, + "prometheus_healthy_replicas_min": 1 + }, + { + "label": "growing-r2-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2470.6926422120705, + "successful_rps_min": 2350.071277688505, + "successful_rps_max": 2624.0312067535733, + "latency_p50_median_ms": 11.857, + "latency_p95_median_ms": 21.935, + "latency_p99_median_ms": 34.466, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.5066083366271347, + "api_resident_bytes_median": 62377984, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "growing-r2-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2239.2042490524, + "successful_rps_min": 2097.2544782269274, + "successful_rps_max": 2355.541238113426, + "latency_p50_median_ms": 28.242, + "latency_p95_median_ms": 57.759, + "latency_p99_median_ms": 83.657, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.5194210660056459, + "api_resident_bytes_median": 64917504, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "growing-r2-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 2, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2553.9360481643857, + "successful_rps_min": 2543.3599428338935, + "successful_rps_max": 2574.3503510903274, + "latency_p50_median_ms": 2.891, + "latency_p95_median_ms": 5.282, + "latency_p99_median_ms": 8.66, + "total_errors": 0, + "observed_instances_min": 2, + "api_cpu_cores_median": 0.5132378054070886, + "api_resident_bytes_median": 58920960, + "postgres_connections_median": 15, + "prometheus_healthy_replicas_min": 2 + }, + { + "label": "growing-r4-c32", + "repeats": 3, + "concurrency": 32, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2129.686826217956, + "successful_rps_min": 1694.9586335176737, + "successful_rps_max": 2462.2257633679833, + "latency_p50_median_ms": 13.923, + "latency_p95_median_ms": 24.14, + "latency_p99_median_ms": 39.802, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5415446510608212, + "api_resident_bytes_median": 423374848, + "postgres_connections_median": 33, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "growing-r4-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 1858.871532675962, + "successful_rps_min": 1742.288309814433, + "successful_rps_max": 2209.1458734057874, + "latency_p50_median_ms": 26.708, + "latency_p95_median_ms": 67.422, + "latency_p99_median_ms": 98.596, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5410956096011533, + "api_resident_bytes_median": 397496320, + "postgres_connections_median": 33, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "growing-r4-c8", + "repeats": 3, + "concurrency": 8, + "replicas": 4, + "api_cpus_per_replica": "1", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 8, + "successful_rps_median": 2184.8009817448374, + "successful_rps_min": 2020.082572066652, + "successful_rps_max": 2202.198843068647, + "latency_p50_median_ms": 3.294, + "latency_p95_median_ms": 7.049, + "latency_p99_median_ms": 13.356, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.561615507893219, + "api_resident_bytes_median": 614023168, + "postgres_connections_median": 19, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "pool-baseline-r4-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 10, + "successful_rps_median": 1917.7537920911027, + "successful_rps_min": 1519.7249081611826, + "successful_rps_max": 2000.495598548728, + "latency_p50_median_ms": 29.998, + "latency_p95_median_ms": 59.521, + "latency_p99_median_ms": 101.701, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5460887049483655, + "api_resident_bytes_median": 610664448, + "postgres_connections_median": 41, + "prometheus_healthy_replicas_min": 4 + }, + { + "label": "pool-tuned-r4-c64", + "repeats": 3, + "concurrency": 64, + "replicas": 4, + "api_cpus_per_replica": "0.5", + "api_memory_per_replica": "256m", + "db_max_connections_per_replica": 4, + "successful_rps_median": 2178.974657000223, + "successful_rps_min": 2175.6291852005656, + "successful_rps_max": 2296.2928939242815, + "latency_p50_median_ms": 23.605, + "latency_p95_median_ms": 66.761, + "latency_p99_median_ms": 80.909, + "total_errors": 0, + "observed_instances_min": 4, + "api_cpu_cores_median": 0.5681059439110139, + "api_resident_bytes_median": 536719360, + "postgres_connections_median": 17, + "prometheus_healthy_replicas_min": 4 + } + ] +} diff --git a/evidence/lab3-scale-rps.svg b/evidence/lab3-scale-rps.svg new file mode 100644 index 0000000..7124497 --- /dev/null +++ b/evidence/lab3-scale-rps.svg @@ -0,0 +1 @@ +Median successful RPS by replica count06201240186024803100124API replicasfixed, c=32fixed, c=64fixed, c=8growing, c=32growing, c=64growing, c=8 \ No newline at end of file diff --git a/evidence/lab3-verification.txt b/evidence/lab3-verification.txt new file mode 100644 index 0000000..97710ec --- /dev/null +++ b/evidence/lab3-verification.txt @@ -0,0 +1,51 @@ +Lab 3 verification +Date: 2026-10-02 +Environment: Docker linux/arm64, Go 1.26.5 + +Contract and build +- make check: PASS +- docker compose --env-file .env config --quiet: PASS +- go test ./cmd/lab3experiment: PASS +- go test ./internal/messenger ./internal/model ./internal/password: PASS + +Integration +- docker compose --profile tests run --rm --build tests: PASS +- PostgreSQL concurrent idempotency: 8 calls through two Store instances, one message ID and one row +- black-box replica idempotency: 16 concurrent calls through Nginx, at least two instance IDs, one message ID and one history item +- PostgreSQL and Valkey stop/recovery: PASS +- persistence after restarting API, PostgreSQL and Valkey: PASS; the original session and data remained valid +- public smoke through Nginx: PASS + +API replica failover +- run 1: 2 instances observed, 131 successful writes, 0 transition errors, first recovered write after 1.353 s +- run 2: 2 instances observed, 125 successful writes, 1 transition error, first recovered write after 1.442 s +- confirmed messages remained readable and idempotent retries did not create duplicates + +Rolling update under Go load +- duration: 60.002 s +- concurrency: 16 +- successful requests: 134894 +- successful RPS: 2248.16 +- errors: 0 +- latency p50/p95/p99: 6.41/13.12/22.01 ms +- healthy API replicas after update: 2 + +Resource boundary +- 4 API replicas with 128 MiB each restarted while Argon2id processed registration +- idle process memory was about 71 MiB; Argon2id memory parameter was 64 MiB +- the fixed-budget experiment therefore uses 256 MiB per replica as its minimum + +Scale experiment +- command: DURATION=6s WARMUP=2s REPEATS=3 CONCURRENCIES='8 32 64' scripts/run-scale-experiment.sh +- configurations: fixed total API resources and growing per-replica resources, 1/2/4 replicas +- completed points: 60; HTTP errors: 0; every point observed all configured replicas +- CPU and Nginx RPS use counter deltas read immediately before and after measured load +- maximum difference between client RPS and Nginx counter RPS across 60 points: 0.75% +- reducing the pool from 10 to 4 connections per replica changed median RPS from 1918 to 2179 and PostgreSQL connections from 41 to 17 +- combined sanitized raw data and median aggregates: evidence/lab3-scale-results.json +- comparison chart: evidence/lab3-scale-rps.svg + +Configuration scope +- API receives only its application settings and PostgreSQL/Valkey connection URLs +- migrate receives only PostgreSQL pool, dependency and migration settings +- neither container receives the Grafana password or separate source passwords of unrelated services diff --git a/go.mod b/go.mod index 87b3785..68b77ae 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitea.privcloud.tech/dizi/mtusi2026lab2 +module gitea.privcloud.tech/dizi/mtusi2026lab3 go 1.26.0 @@ -7,6 +7,7 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/prometheus/client_golang v1.23.2 github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 golang.org/x/crypto v0.57.0 ) @@ -14,6 +15,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -21,6 +23,7 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -36,4 +39,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/config/config.go b/internal/config/config.go index 4804110..4457141 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,13 +16,13 @@ type Config struct { DatabaseURL string CacheURL string SessionTTL time.Duration + IdempotencyTTL time.Duration DBMaxConns int32 DBMinConns int32 DBMaxConnectionAge time.Duration DBMaxConnectionIdle time.Duration DBHealthCheckPeriod time.Duration DependencyTimeout time.Duration - MigrationTimeout time.Duration RequestTimeout time.Duration HTTPReadHeaderTimeout time.Duration HTTPReadTimeout time.Duration @@ -37,6 +37,27 @@ type Config struct { SwaggerPort string } +type MigrationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + MigrationTimeout time.Duration +} + +type databaseConfig struct { + url string + maxConns int + minConns int + maxConnectionAge time.Duration + maxConnectionIdle time.Duration + healthCheckPeriod time.Duration + dependencyTimeout time.Duration +} + func Load() (Config, error) { values := viper.New() values.AutomaticEnv() @@ -49,7 +70,7 @@ func Load() (Config, error) { if err != nil { return Config{}, err } - databaseURL, err := requiredString(values, "DATABASE_URL") + database, err := loadDatabaseConfig(values) if err != nil { return Config{}, err } @@ -69,31 +90,7 @@ func Load() (Config, error) { if err != nil { return Config{}, err } - maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) - if err != nil { - return Config{}, err - } - minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) - if err != nil { - return Config{}, err - } - maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) - if err != nil { - return Config{}, err - } - maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) - if err != nil { - return Config{}, err - } - healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) - if err != nil { - return Config{}, err - } - dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) - if err != nil { - return Config{}, err - } - migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) if err != nil { return Config{}, err } @@ -147,11 +144,12 @@ func Load() (Config, error) { } return Config{ - Address: net.JoinHostPort(host, port), DatabaseURL: databaseURL, CacheURL: cacheURL, - SessionTTL: sessionTTL, DBMaxConns: int32(maxConns), DBMinConns: int32(minConns), - DBMaxConnectionAge: maxConnectionAge, DBMaxConnectionIdle: maxConnectionIdle, - DBHealthCheckPeriod: healthCheckPeriod, DependencyTimeout: dependencyTimeout, - MigrationTimeout: migrationTimeout, RequestTimeout: requestTimeout, + Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, + SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, + DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + RequestTimeout: requestTimeout, HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout, HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout, HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), @@ -161,6 +159,61 @@ func Load() (Config, error) { }, nil } +func LoadMigration() (MigrationConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return MigrationConfig{}, err + } + migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + if err != nil { + return MigrationConfig{}, err + } + return MigrationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + MigrationTimeout: migrationTimeout, + }, nil +} + +func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { + databaseURL, err := requiredString(values, "DATABASE_URL") + if err != nil { + return databaseConfig{}, err + } + maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) + if err != nil { + return databaseConfig{}, err + } + minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) + if err != nil { + return databaseConfig{}, err + } + maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) + if err != nil { + return databaseConfig{}, err + } + maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) + if err != nil { + return databaseConfig{}, err + } + healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) + if err != nil { + return databaseConfig{}, err + } + dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) + if err != nil { + return databaseConfig{}, err + } + return databaseConfig{ + url: databaseURL, maxConns: maxConns, minConns: minConns, + maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle, + healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout, + }, nil +} + func requiredString(values *viper.Viper, name string) (string, error) { value := strings.TrimSpace(values.GetString(name)) if value == "" { diff --git a/internal/database/database.go b/internal/database/database.go index a690415..a02ff7c 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab2/migrations" + "gitea.privcloud.tech/dizi/mtusi2026lab3/migrations" ) var ( @@ -20,7 +20,6 @@ var ( errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive") errHealthCheckPeriod = errors.New("database health check period must be positive") errConnectionTimeout = errors.New("database connection timeout must be positive") - errMigrationTimeout = errors.New("database migration timeout must be positive") ) type Config struct { @@ -31,7 +30,6 @@ type Config struct { MaxConnectionIdle time.Duration HealthCheckPeriod time.Duration ConnectTimeout time.Duration - MigrationTimeout time.Duration } func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { @@ -59,15 +57,18 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { return nil, fmt.Errorf("ping database: %w", err) } - migrationCtx, cancel := context.WithTimeout(ctx, config.MigrationTimeout) - defer cancel() - if err := applyMigrations(migrationCtx, pool, migrations.FS); err != nil { - pool.Close() - return nil, err - } return pool, nil } +func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error { + if timeout <= 0 { + return errors.New("database migration timeout must be positive") + } + migrationCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return applyMigrations(migrationCtx, pool, migrations.FS) +} + func validateConfig(config Config) error { switch { case config.URL == "": @@ -86,8 +87,6 @@ func validateConfig(config Config) error { return errHealthCheckPeriod case config.ConnectTimeout <= 0: return errConnectionTimeout - case config.MigrationTimeout <= 0: - return errMigrationTimeout default: return nil } diff --git a/internal/messenger/auth.go b/internal/messenger/auth.go index 5bf621d..5c6eb57 100644 --- a/internal/messenger/auth.go +++ b/internal/messenger/auth.go @@ -6,9 +6,9 @@ import ( "net/http" "strings" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session" ) func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { diff --git a/internal/messenger/handlers.go b/internal/messenger/handlers.go index 1b0fc9b..e9ad0fd 100644 --- a/internal/messenger/handlers.go +++ b/internal/messenger/handlers.go @@ -9,8 +9,8 @@ import ( "time" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres" ) var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) @@ -202,6 +202,10 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { if r.Method == http.MethodGet { return s.listMessages(w, r, user, chatID) } + key, valid := idempotencyKey(r) + if !valid { + return invalidRequest() + } var input struct { Text string `json:"text"` } @@ -211,13 +215,21 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { if !validText(input.Text, 4000) { return invalidRequest() } - message, err := s.messengerStore.CreateMessage(r.Context(), chatID, user.ID, input.Text) + message, replay, err := s.messengerStore.CreateMessage( + r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL, + ) if errors.Is(err, postgres.ErrNotFound) { return notFound() } + if errors.Is(err, postgres.ErrIdempotencyConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"} + } if err != nil { return unavailable() } + if replay { + s.metrics.replays.Inc() + } writeJSON(w, http.StatusCreated, message) return nil } diff --git a/internal/messenger/messenger_interface.go b/internal/messenger/messenger_interface.go index ed56628..1761254 100644 --- a/internal/messenger/messenger_interface.go +++ b/internal/messenger/messenger_interface.go @@ -4,7 +4,7 @@ import ( "context" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" ) type MessengerStore interface { @@ -15,7 +15,7 @@ type MessengerStore interface { CreateChat(context.Context, string, []string, string) (model.Chat, error) ChatForUser(context.Context, string, string) (model.Chat, error) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) - CreateMessage(context.Context, string, string, string) (model.Message, error) + CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) } diff --git a/internal/messenger/metrics.go b/internal/messenger/metrics.go index 1f6272d..8073542 100644 --- a/internal/messenger/metrics.go +++ b/internal/messenger/metrics.go @@ -15,6 +15,7 @@ type metrics struct { requests *prometheus.CounterVec duration *prometheus.HistogramVec inFlight prometheus.Gauge + replays prometheus.Counter handler http.Handler } @@ -40,11 +41,18 @@ func newMetrics() *metrics { Name: "requests_in_flight", Help: "HTTP requests currently being served.", }), + replays: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", + Subsystem: "http", + Name: "idempotency_replays_total", + Help: "Message requests served from an idempotency record.", + }), } registry.MustRegister( result.requests, result.duration, result.inFlight, + result.replays, collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), ) diff --git a/internal/messenger/model.go b/internal/messenger/model.go index 65474b6..acbe99d 100644 --- a/internal/messenger/model.go +++ b/internal/messenger/model.go @@ -1,7 +1,7 @@ package messenger import ( - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" "github.com/google/uuid" ) diff --git a/internal/messenger/request.go b/internal/messenger/request.go index 8b9a7cc..da264ec 100644 --- a/internal/messenger/request.go +++ b/internal/messenger/request.go @@ -9,6 +9,7 @@ import ( "mime" "net/http" "net/url" + "regexp" "strconv" "strings" "time" @@ -17,6 +18,8 @@ import ( const maxRequestBytes = 1 << 20 +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError { mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { @@ -54,6 +57,14 @@ func validText(value string, maxRunes int) bool { return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes } +func idempotencyKey(r *http.Request) (string, bool) { + values := r.Header.Values("Idempotency-Key") + if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) { + return "", false + } + return values[0], true +} + type pageCursor struct { Kind string `json:"kind"` Resource string `json:"resource"` diff --git a/internal/messenger/server.go b/internal/messenger/server.go index da676b9..3cc2ba0 100644 --- a/internal/messenger/server.go +++ b/internal/messenger/server.go @@ -13,7 +13,9 @@ type Server struct { sessions SessionStore passwords PasswordService sessionTTL time.Duration + idempotencyTTL time.Duration requestTTL time.Duration + instanceID string logger *slog.Logger swaggerPort string metrics *metrics @@ -30,10 +32,21 @@ func (w *statusWriter) WriteHeader(status int) { w.ResponseWriter.WriteHeader(status) } -func NewServer(logger *slog.Logger, messages MessengerStore, sessions SessionStore, passwords PasswordService, sessionTTL, requestTTL time.Duration, swaggerPort string) *Server { +func NewServer( + logger *slog.Logger, + messages MessengerStore, + sessions SessionStore, + passwords PasswordService, + sessionTTL time.Duration, + idempotencyTTL time.Duration, + requestTTL time.Duration, + instanceID string, + swaggerPort string, +) *Server { s := &Server{ messengerStore: messages, sessions: sessions, passwords: passwords, - sessionTTL: sessionTTL, requestTTL: requestTTL, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(), + sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL, + instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(), } s.ready.Store(true) return s @@ -48,6 +61,7 @@ func (s *Server) Handler() http.Handler { } func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", s.instanceID) requestID, err := newID() if err != nil { http.Error(w, "server unavailable", http.StatusServiceUnavailable) @@ -70,8 +84,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) }() if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { - wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type") + wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key") wrapped.WriteHeader(http.StatusNoContent) return } @@ -92,7 +106,7 @@ func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool return false } w.Header().Set("Access-Control-Allow-Origin", origin) - w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id") + w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id") w.Header().Add("Vary", "Origin") return true } diff --git a/internal/messenger/server_test.go b/internal/messenger/server_test.go index 9d521c9..011fb0f 100644 --- a/internal/messenger/server_test.go +++ b/internal/messenger/server_test.go @@ -3,6 +3,7 @@ package messenger import ( "context" "errors" + "fmt" "io" "log/slog" "net/http" @@ -11,17 +12,36 @@ import ( "testing" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres" ) var errDependency = errors.New("dependency unavailable") +type MessengerSuite struct { + suite.Suite + logger *slog.Logger +} + +func TestMessengerSuite(t *testing.T) { + suite.Run(t, new(MessengerSuite)) +} + +func (s *MessengerSuite) SetupTest() { + s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) +} + type messengerStoreStub struct { pingErr error userByID model.User userByIDErr error credentials model.UserCredentials credentialsErr error + message model.Message + messageReplay bool + messageErr error } func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } @@ -43,8 +63,8 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) { return nil, false, nil } -func (messengerStoreStub) CreateMessage(context.Context, string, string, string) (model.Message, error) { - return model.Message{}, nil +func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) { + return s.message, s.messageReplay, s.messageErr } func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { return nil, false, nil @@ -74,7 +94,8 @@ func (s passwordsStub) Verify(string, string) (bool, error) { return s.verified, s.err } -func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) { +func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { + t := s.T() tests := []struct { name string store messengerStoreStub @@ -86,7 +107,7 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - server := newTestServer(tt.store, tt.sessions, passwordsStub{}) + server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) response := serveRequest(server, http.MethodGet, "/health/ready", "") if response.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) @@ -95,7 +116,8 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) { } } -func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) { +func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() { + t := s.T() tests := []struct { name string store messengerStoreStub @@ -107,7 +129,7 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - server := newTestServer(tt.store, tt.sessions, passwordsStub{}) + server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) request.Header.Set("Authorization", "Bearer valid-token") response := httptest.NewRecorder() @@ -119,15 +141,16 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) { } } -func TestAuthenticationChallenges(t *testing.T) { - server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) +func (s *MessengerSuite) TestAuthenticationChallenges() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "") if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` { t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate")) } store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}} - server = newTestServer(store, sessionStub{}, passwordsStub{verified: false}) + server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false}) request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) request.SetBasicAuth("user", "wrong-password") response := httptest.NewRecorder() @@ -137,8 +160,9 @@ func TestAuthenticationChallenges(t *testing.T) { } } -func TestSessionCreationRejectsChunkedBody(t *testing.T) { - server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) +func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) request.ContentLength = -1 request.SetBasicAuth("user", "password") @@ -149,17 +173,77 @@ func TestSessionCreationRejectsChunkedBody(t *testing.T) { } } -func TestRegistrationRejectsWhitespacePassword(t *testing.T) { - server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) +func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`) if response.Code != http.StatusBadRequest { t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) } } -func newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server { - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - return NewServer(logger, messages, sessions, passwords, time.Hour, time.Second, "8081") +func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + for _, response := range []*httptest.ResponseRecorder{ + serveRequest(server, http.MethodGet, "/health/live", ""), + serveRequest(server, http.MethodGet, "/api/v1/users/me", ""), + } { + if value := response.Header().Get("X-Instance-Id"); value != "test-instance" { + t.Fatalf("X-Instance-Id = %q, want test-instance", value) + } + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() { + t := s.T() + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + tests := []struct { + name string + key string + }{ + {name: "missing"}, + {name: "invalid", key: "invalid key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + if tt.key != "" { + request.Header.Set("Idempotency-Key", tt.key) + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } + }) + } +} + +func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() { + t := s.T() + store := messengerStoreStub{ + userByID: model.User{ID: "user-id"}, + messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict), + } + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "same-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server { + return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081") } func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 7c0f2e4..d3a54d9 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -4,15 +4,21 @@ import ( "encoding/json" "testing" "time" + + "github.com/stretchr/testify/suite" ) -func TestUserTimestampHasFixedMicroseconds(t *testing.T) { +type ModelSuite struct { + suite.Suite +} + +func TestModelSuite(t *testing.T) { + suite.Run(t, new(ModelSuite)) +} + +func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() { createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC) data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt}) - if err != nil { - t.Fatalf("Marshal() error = %v", err) - } - if string(data) != `{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}` { - t.Fatalf("Marshal() = %s", data) - } + s.Require().NoError(err) + s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data)) } diff --git a/internal/password/password_test.go b/internal/password/password_test.go index df25f40..8e88875 100644 --- a/internal/password/password_test.go +++ b/internal/password/password_test.go @@ -1,42 +1,42 @@ package password -import "testing" +import ( + "testing" -func TestHasher(t *testing.T) { - hasher, err := New(testConfig()) - if err != nil { - t.Fatalf("New() error = %v", err) - } - hash, err := hasher.Hash("T3st-password-123") - if err != nil { - t.Fatalf("Hash() error = %v", err) - } + "github.com/stretchr/testify/suite" +) - verified, err := hasher.Verify("T3st-password-123", hash) - if err != nil { - t.Fatalf("Verify() error = %v", err) - } - if !verified { - t.Fatal("Verify() = false for the original password") - } - - verified, err = hasher.Verify("wrong-password", hash) - if err != nil { - t.Fatalf("Verify() error = %v", err) - } - if verified { - t.Fatal("Verify() = true for a different password") - } +type PasswordSuite struct { + suite.Suite + hasher *Hasher } -func TestVerifyRejectsMalformedHash(t *testing.T) { +func TestPasswordSuite(t *testing.T) { + suite.Run(t, new(PasswordSuite)) +} + +func (s *PasswordSuite) SetupTest() { hasher, err := New(testConfig()) - if err != nil { - t.Fatalf("New() error = %v", err) - } - if _, err := hasher.Verify("password", "invalid"); err == nil { - t.Fatal("Verify() error = nil for malformed hash") - } + s.Require().NoError(err) + s.hasher = hasher +} + +func (s *PasswordSuite) TestHasher() { + hash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + + verified, err := s.hasher.Verify("T3st-password-123", hash) + s.Require().NoError(err) + s.True(verified) + + verified, err = s.hasher.Verify("wrong-password", hash) + s.Require().NoError(err) + s.False(verified) +} + +func (s *PasswordSuite) TestVerifyRejectsMalformedHash() { + _, err := s.hasher.Verify("password", "invalid") + s.Error(err) } func testConfig() Config { diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index f90becb..4e0c378 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -3,8 +3,9 @@ package postgres import "errors" var ( - ErrConflict = errors.New("conflict") - ErrNotFound = errors.New("not found") - ErrInvalidMembers = errors.New("invalid members") - ErrInvalidCursor = errors.New("invalid cursor") + ErrConflict = errors.New("conflict") + ErrNotFound = errors.New("not found") + ErrInvalidMembers = errors.New("invalid members") + ErrInvalidCursor = errors.New("invalid cursor") + ErrIdempotencyConflict = errors.New("idempotency conflict") ) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go new file mode 100644 index 0000000..a4843ad --- /dev/null +++ b/internal/postgres/idempotency.go @@ -0,0 +1,285 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" +) + +const expiredIdempotencyCleanupBatch = 100 + +func (s *Store) CreateMessage( + ctx context.Context, + chatID string, + senderID string, + text string, + idempotencyKey string, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + var ( + tx pgx.Tx + err error + visible bool + requestHash [sha256.Size]byte + message model.Message + replay bool + ) + + tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID) + if err != nil { + return model.Message{}, false, err + } + if !visible { + return model.Message{}, false, ErrNotFound + } + if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { + return model.Message{}, false, err + } + + requestHash = sha256.Sum256([]byte(text)) + message, replay, err = createIdempotentMessage( + ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err) + } + return message, replay, nil +} + +func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var visible bool + if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + return visible, nil +} + +func createIdempotentMessage( + ctx context.Context, + tx pgx.Tx, + chatID string, + senderID string, + text string, + idempotencyKey string, + requestHash []byte, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + for { + var ( + record idempotencyRecord + found bool + err error + messageID string + reserved bool + message model.Message + ) + + record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey) + if err != nil { + return model.Message{}, false, err + } + if found && record.active { + if !bytes.Equal(record.requestHash, requestHash) { + return model.Message{}, false, ErrIdempotencyConflict + } + message, err = messageByID(ctx, tx, record.messageID) + return message, true, err + } + if found { + if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil { + return model.Message{}, false, err + } + } + + messageID = uuid.NewString() + reserved, err = reserveIdempotencyKey( + ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if !reserved { + continue + } + + message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text) + return message, false, err + } +} + +func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error { + const query = ` + WITH expired AS ( + SELECT ctid + FROM message_idempotency + WHERE expires_at <= statement_timestamp() + ORDER BY expires_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM message_idempotency records + USING expired + WHERE records.ctid = expired.ctid` + + if _, err := tx.Exec(ctx, query, limit); err != nil { + return fmt.Errorf("delete expired idempotency records: %w", err) + } + return nil +} + +type idempotencyRecord struct { + requestHash []byte + messageID string + active bool +} + +func lockIdempotencyRecord( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, +) (idempotencyRecord, bool, error) { + const query = ` + SELECT request_hash, message_id, expires_at > statement_timestamp() + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + FOR UPDATE` + + var ( + record idempotencyRecord + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key).Scan( + &record.requestHash, + &record.messageID, + &record.active, + ) + if errors.Is(err, pgx.ErrNoRows) { + return idempotencyRecord{}, false, nil + } + if err != nil { + return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err) + } + return record, true, nil +} + +func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error { + const query = ` + DELETE FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3` + if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil { + return fmt.Errorf("delete expired idempotency key: %w", err) + } + return nil +} + +func reserveIdempotencyKey( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, + requestHash []byte, + messageID string, + ttl time.Duration, +) (bool, error) { + const query = ` + INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at) + VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second') + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING true` + + var ( + inserted bool + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reserve idempotency key: %w", err) + } + return inserted, nil +} + +func insertMessage( + ctx context.Context, + tx pgx.Tx, + messageID string, + chatID string, + senderID string, + text string, +) (model.Message, error) { + const query = ` + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + ) + if err != nil { + return model.Message{}, fmt.Errorf("insert message: %w", err) + } + return message, nil +} + +func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) { + const query = ` + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE id = $1` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, fmt.Errorf("idempotency result message not found") + } + if err != nil { + return model.Message{}, fmt.Errorf("get idempotency result message: %w", err) + } + return message, nil +} diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 949171d..2e07257 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" ) type Store struct { @@ -114,7 +114,7 @@ func (s *Store) CreateChat( if err != nil { return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err) } - defer func() { _ = tx.Rollback(ctx) }() + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() valid, err := allUsersExist(ctx, tx, memberIDs) if err != nil { @@ -319,40 +319,6 @@ func (s *Store) visibleChatPositionExists( return exists, nil } -func (s *Store) CreateMessage( - ctx context.Context, - chatID string, - senderID string, - text string, -) (model.Message, error) { - const query = ` - INSERT INTO messages (id, chat_id, sender_id, text) - SELECT $1, $2, $3, $4 - WHERE EXISTS ( - SELECT 1 - FROM chat_members - WHERE chat_id = $2 AND user_id = $3 - ) - RETURNING id, chat_id, sender_id, text, created_at` - - var message model.Message - err := s.pool.QueryRow(ctx, query, uuid.NewString(), chatID, senderID, text).Scan( - &message.ID, - &message.ChatID, - &message.SenderID, - &message.Text, - &message.CreatedAt, - ) - if errors.Is(err, pgx.ErrNoRows) { - return model.Message{}, ErrNotFound - } - if err != nil { - return model.Message{}, fmt.Errorf("create message: %w", err) - } - - return message, nil -} - func (s *Store) ListMessages( ctx context.Context, chatID string, diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index cc1112e..469f5a2 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -11,14 +11,37 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres" ) -func TestConcurrentUsernameCreation(t *testing.T) { - pool := openTestDatabase(t) - store := postgres.New(pool) +type PostgresSuite struct { + suite.Suite + pool *pgxpool.Pool + store *postgres.Store +} + +func TestPostgresSuite(t *testing.T) { + suite.Run(t, new(PostgresSuite)) +} + +func (s *PostgresSuite) SetupSuite() { + s.pool = openTestDatabase(s.T()) + s.store = postgres.New(s.pool) +} + +func (s *PostgresSuite) TearDownSuite() { + if s.pool != nil { + s.pool.Close() + } +} + +func (s *PostgresSuite) TestConcurrentUsernameCreation() { + t := s.T() + store := s.store username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] const workers = 8 @@ -51,9 +74,10 @@ func TestConcurrentUsernameCreation(t *testing.T) { } } -func TestCreateChatRollsBackForUnknownMember(t *testing.T) { - pool := openTestDatabase(t) - store := postgres.New(pool) +func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() { + t := s.T() + pool := s.pool + store := s.store owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") if err != nil { t.Fatalf("CreateUser() error = %v", err) @@ -69,6 +93,175 @@ func TestCreateChatRollsBackForUnknownMember(t *testing.T) { } } +func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + key := uuid.NewString() + + first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour) + if err != nil || replay { + t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err) + } + second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour) + if err != nil || !replay { + t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err) + } + if first != second { + t.Fatalf("replay message = %v, want %v", second, first) + } + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { + t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestConcurrentMessageIdempotency() { + t := s.T() + pool := s.pool + stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)} + owner, chatID := createOwnerAndChat(t, stores[0]) + key := uuid.NewString() + + const workers = 8 + type result struct { + message model.Message + replay bool + err error + } + results := make(chan result, workers) + var group sync.WaitGroup + for worker := range workers { + group.Add(1) + go func() { + defer group.Done() + message, replay, err := stores[worker%len(stores)].CreateMessage( + context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour, + ) + results <- result{message: message, replay: replay, err: err} + }() + } + group.Wait() + close(results) + + var messageID string + var created int + for result := range results { + if result.err != nil { + t.Fatalf("CreateMessage() error = %v", result.err) + } + if messageID == "" { + messageID = result.message.ID + } + if result.message.ID != messageID { + t.Fatalf("message ID = %s, want %s", result.message.ID, messageID) + } + if !result.replay { + created++ + } + } + if created != 1 { + t.Fatalf("new message results = %d, want 1", created) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + key := uuid.NewString() + + if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil { + t.Fatalf("owner CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil { + t.Fatalf("second user CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil { + t.Fatalf("other chat CreateMessage() error = %v", err) + } + if _, err := pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour) + if err != nil || replay || message.Text != "after expiry" { + t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err) + } +} + +func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { + t := s.T() + owner, chatID := createOwnerAndChat(t, s.store) + expiredKey := uuid.NewString() + if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil { + t.Fatalf("create expired message: %v", err) + } + if _, err := s.pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + if _, _, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour, + ); err != nil { + t.Fatalf("create new message: %v", err) + } + + var remaining int + if err := s.pool.QueryRow(context.Background(), ` + SELECT count(*) + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil { + t.Fatalf("count expired idempotency records: %v", err) + } + if remaining != 0 { + t.Fatalf("expired idempotency records = %d, want 0", remaining) + } +} + +func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) { + t.Helper() + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + return owner, chat.ID +} + +func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + return count +} + func openTestDatabase(t *testing.T) *pgxpool.Pool { t.Helper() url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE") @@ -76,12 +269,14 @@ func openTestDatabase(t *testing.T) *pgxpool.Pool { URL: url, MinConnections: 0, MaxConnections: 10, MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute, HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second, - MigrationTimeout: 30 * time.Second, }) if err != nil { t.Fatalf("database.Open() error = %v", err) } - t.Cleanup(pool.Close) + if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil { + pool.Close() + t.Fatalf("database.Migrate() error = %v", err) + } return pool } diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go index 8a19a4b..e22a63d 100644 --- a/internal/session/session_integration_test.go +++ b/internal/session/session_integration_test.go @@ -9,75 +9,64 @@ import ( "time" "github.com/google/uuid" + "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session" ) -func TestSessionLifecycle(t *testing.T) { - store := openTestStore(t) - userID := uuid.NewString() - token, expiresAt, err := store.Create(context.Background(), userID, time.Minute) - if err != nil { - t.Fatalf("Create() error = %v", err) - } - if token == "" || !expiresAt.After(time.Now()) { - t.Fatalf("invalid token or expiry: token length = %d, expires_at = %v", len(token), expiresAt) - } - - actualUserID, err := store.UserID(context.Background(), token) - if err != nil { - t.Fatalf("UserID() error = %v", err) - } - if actualUserID != userID { - t.Fatalf("UserID() = %q, want %q", actualUserID, userID) - } - if err := store.Delete(context.Background(), token); err != nil { - t.Fatalf("Delete() error = %v", err) - } - if _, err := store.UserID(context.Background(), token); !errors.Is(err, session.ErrNotFound) { - t.Fatalf("UserID() after delete error = %v, want ErrNotFound", err) - } +type SessionSuite struct { + suite.Suite + store *session.Store } -func TestSessionExpires(t *testing.T) { - store := openTestStore(t) - token, _, err := store.Create(context.Background(), uuid.NewString(), time.Second) - if err != nil { - t.Fatalf("Create() error = %v", err) - } - - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) { - _, err := store.UserID(context.Background(), token) - if errors.Is(err, session.ErrNotFound) { - return - } - if err != nil { - t.Fatalf("UserID() error = %v", err) - } - time.Sleep(50 * time.Millisecond) - } - t.Fatal("session did not expire within three seconds") +func TestSessionSuite(t *testing.T) { + suite.Run(t, new(SessionSuite)) } -func openTestStore(t *testing.T) *session.Store { - t.Helper() - url := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) - if url == "" { +func (s *SessionSuite) SetupSuite() { + rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) + if rawURL == "" { path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE")) if path == "" { - t.Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") + s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") } data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read TEST_CACHE_URL_FILE: %v", err) - } - url = strings.TrimSpace(string(data)) + s.Require().NoError(err) + rawURL = strings.TrimSpace(string(data)) } - store, err := session.New(url) - if err != nil { - t.Fatalf("session.New() error = %v", err) + store, err := session.New(rawURL) + s.Require().NoError(err) + s.store = store +} + +func (s *SessionSuite) TearDownSuite() { + if s.store != nil { + s.Require().NoError(s.store.Close()) } - t.Cleanup(func() { _ = store.Close() }) - return store +} + +func (s *SessionSuite) TestSessionLifecycle() { + userID := uuid.NewString() + token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute) + s.Require().NoError(err) + s.NotEmpty(token) + s.True(expiresAt.After(time.Now())) + + actualUserID, err := s.store.UserID(context.Background(), token) + s.Require().NoError(err) + s.Equal(userID, actualUserID) + s.Require().NoError(s.store.Delete(context.Background(), token)) + + _, err = s.store.UserID(context.Background(), token) + s.ErrorIs(err, session.ErrNotFound) +} + +func (s *SessionSuite) TestSessionExpires() { + token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second) + s.Require().NoError(err) + + s.Eventually(func() bool { + _, err = s.store.UserID(context.Background(), token) + return errors.Is(err, session.ErrNotFound) + }, 3*time.Second, 50*time.Millisecond) } diff --git a/lab3.env.example b/lab3.env.example index 5dd636b..447b3b7 100644 --- a/lab3.env.example +++ b/lab3.env.example @@ -8,10 +8,14 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 -# Время жизни сессии и используемое хранилище. +# Время жизни сессии, идемпотентности и используемое хранилище. SESSION_TTL_SECONDS=3600 +IDEMPOTENCY_TTL_SECONDS=86400 SESSION_STORE=valkey +# Число API-реплик за Nginx. +API_REPLICAS=2 + # Учётные данные локального PostgreSQL и строка подключения API. POSTGRES_USER=messenger POSTGRES_DB=messenger @@ -37,7 +41,7 @@ HTTP_READ_HEADER_TIMEOUT_SECONDS=5 HTTP_READ_TIMEOUT_SECONDS=10 HTTP_WRITE_TIMEOUT_SECONDS=10 HTTP_IDLE_TIMEOUT_SECONDS=60 -HTTP_SHUTDOWN_TIMEOUT_SECONDS=10 +HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 # Параметры Argon2id для новых хешей паролей. PASSWORD_MEMORY_KIB=65536 @@ -49,9 +53,23 @@ PASSWORD_KEY_LENGTH=32 # Порт Swagger внутри конфигурации CORS API. SWAGGER_PORT=8081 -# Ограничения ресурсов контейнеров приложения и хранилищ. -API_CPUS=2 -API_MEMORY=512m +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=256m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=0.5 +GRAFANA_MEMORY=256m POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 diff --git a/migrations/0002_message_idempotency.sql b/migrations/0002_message_idempotency.sql new file mode 100644 index 0000000..91443a4 --- /dev/null +++ b/migrations/0002_message_idempotency.sql @@ -0,0 +1,19 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL, + chat_id UUID NOT NULL, + key TEXT NOT NULL, + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32), + CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at) +); + +CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at); diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json index d724608..ad0b521 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger.json @@ -6,58 +6,81 @@ { "type": "timeseries", "title": "API requests per second", "id": 1, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, - "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "RPS", "refId": "A"}] + "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}] }, { - "type": "timeseries", "title": "API responses by status class", "id": 2, + "type": "timeseries", "title": "API RPS by replica", "id": 2, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API responses by status class", "id": 3, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, "targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}] }, { - "type": "timeseries", "title": "API latency p95", "id": 3, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "type": "timeseries", "title": "Healthy API replicas", "id": 4, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API latency p95 by replica and route", "id": 5, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, - "targets": [{"expr": "histogram_quantile(0.95, sum by (le, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{route}}", "refId": "A"}] + "targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}] }, { - "type": "timeseries", "title": "API in flight", "id": 4, - "gridPos": {"h": 8, "w": 6, "x": 12, "y": 8}, - "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "requests", "refId": "A"}] + "type": "timeseries", "title": "API in flight", "id": 6, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, + "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}] }, { - "type": "timeseries", "title": "Go heap", "id": 5, - "gridPos": {"h": 8, "w": 6, "x": 18, "y": 8}, + "type": "timeseries", "title": "Go heap by replica", "id": 7, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "heap", "refId": "A"}] + "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}] }, { - "type": "timeseries", "title": "PostgreSQL connections", "id": 6, - "gridPos": {"h": 8, "w": 8, "x": 0, "y": 16}, + "type": "timeseries", "title": "Idempotency replays", "id": 8, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, + "targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Nginx connections and requests", "id": 9, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, + "targets": [ + {"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"}, + {"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "PostgreSQL connections", "id": 10, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, "targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}] }, { - "type": "timeseries", "title": "PostgreSQL transactions", "id": 7, - "gridPos": {"h": 8, "w": 8, "x": 8, "y": 16}, + "type": "timeseries", "title": "PostgreSQL transactions", "id": 11, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, "targets": [ {"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"}, {"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"} ] }, { - "type": "timeseries", "title": "PostgreSQL database size", "id": 8, - "gridPos": {"h": 8, "w": 8, "x": 16, "y": 16}, + "type": "timeseries", "title": "PostgreSQL database size", "id": 12, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, "targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}] }, { - "type": "timeseries", "title": "Valkey memory", "id": 9, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24}, + "type": "timeseries", "title": "Valkey memory", "id": 13, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, "targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}] }, { - "type": "timeseries", "title": "Valkey keys and commands", "id": 10, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24}, + "type": "timeseries", "title": "Valkey keys and commands", "id": 14, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, "targets": [ {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} @@ -66,11 +89,11 @@ ], "refresh": "5s", "schemaVersion": 41, - "tags": ["highload", "lab2"], + "tags": ["highload", "lab3", "nginx", "failover"], "templating": {"list": []}, "time": {"from": "now-15m", "to": "now"}, "timezone": "browser", - "title": "Messenger overview", + "title": "Messenger lab3 overview", "uid": "messenger-overview", - "version": 1 + "version": 2 } diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index e23db13..2c0881e 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -4,8 +4,15 @@ global: scrape_configs: - job_name: messenger + dns_sd_configs: + - names: ["api"] + type: A + port: 8080 + refresh_interval: 2s + + - job_name: nginx static_configs: - - targets: ["api:8080"] + - targets: ["nginx-exporter:9113"] - job_name: postgres static_configs: diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..9dcebe8 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,55 @@ +user nginx; +worker_processes auto; +error_log /dev/stderr notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + + log_format structured escape=json + '{"time":"$time_iso8601","request_id":"$http_x_request_id",' + '"method":"$request_method","uri":"$request_uri","status":$status,' + '"request_time":$request_time,"upstream":"$upstream_addr",' + '"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}'; + + access_log /dev/stdout structured; + + upstream messenger { + zone messenger 64k; + server api:8080 resolve max_fails=1 fail_timeout=3s; + keepalive 32; + } + + server { + listen 8080; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } +} diff --git a/scripts/restart.sh b/scripts/restart.sh index 1954a36..cacb521 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,7 +2,7 @@ set -eu -docker compose restart postgres valkey -docker compose up -d --wait --wait-timeout 90 postgres valkey -docker compose restart api -docker compose up -d --wait --wait-timeout 90 api +docker compose stop nginx api valkey postgres +docker compose start postgres valkey +docker compose run --rm migrate +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..e32e297 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -eu + +wait_for_replicas() { + expected=$1 + deadline=$(($(date +%s) + 90)) + + while [ "$(date +%s)" -lt "$deadline" ]; do + containers=$(docker compose ps -q api) + count=0 + healthy=0 + for container in $containers; do + count=$((count + 1)) + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container") + if [ "$status" = "healthy" ]; then + healthy=$((healthy + 1)) + fi + done + if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then + return 0 + fi + sleep 1 + done + + printf '%s\n' "API replicas did not become healthy: expected $expected" >&2 + return 1 +} + +old_containers=$(docker compose ps -q api) +old_count=0 +for container in $old_containers; do + old_count=$((old_count + 1)) +done + +docker compose build +docker compose up -d postgres valkey +docker compose run --rm migrate + +if [ "$old_count" -lt 2 ]; then + docker compose up -d --scale api=2 --wait --wait-timeout 120 + exit 0 +fi + +docker compose up -d --no-deps --no-recreate --scale api=3 api +wait_for_replicas 3 + +for container in $old_containers; do + docker stop --time 6 "$container" >/dev/null + docker rm "$container" >/dev/null + docker compose up -d --no-deps --no-recreate --scale api=3 api + wait_for_replicas 3 +done + +docker compose up -d --no-deps --no-recreate --scale api=2 api +wait_for_replicas 2 +docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120 diff --git a/scripts/run-scale-experiment.sh b/scripts/run-scale-experiment.sh new file mode 100755 index 0000000..092713c --- /dev/null +++ b/scripts/run-scale-experiment.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +set -eu + +duration=${DURATION:-15s} +warmup=${WARMUP:-3s} +concurrencies=${CONCURRENCIES:-"8 32 64"} +repeats=${REPEATS:-3} +current_project= + +restore_default_stack() { + if [ -n "$current_project" ]; then + env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true + fi + docker compose up -d --scale api=2 --wait --wait-timeout 120 +} + +run_configuration() { + series=$1 + replicas=$2 + api_cpus=$3 + api_memory=$4 + db_max_conns=$5 + repeat=$6 + point_concurrencies=$7 + project="lab3exp_${series}_r${replicas}_n${repeat}" + + pending=false + for concurrency in $point_concurrencies; do + label="${series}-r${replicas}-c${concurrency}-n${repeat}" + if [ ! -s "evidence/lab3-${label}.json" ]; then + pending=true + fi + done + if [ "$pending" = false ]; then + echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}" + return + fi + + current_project=$project + + compose() { + env \ + COMPOSE_PROJECT_NAME="$project" \ + API_REPLICAS="$replicas" \ + API_CPUS="$api_cpus" \ + API_MEMORY="$api_memory" \ + DB_MIN_CONNS=0 \ + DB_MAX_CONNS="$db_max_conns" \ + docker compose "$@" + } + + compose up -d --scale api="$replicas" --wait --wait-timeout 120 \ + postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus + for concurrency in $point_concurrencies; do + label="${series}-r${replicas}-c${concurrency}-n${repeat}" + output="lab3-${label}.json" + if [ -s "evidence/$output" ]; then + echo "Skipping completed point $label" + continue + fi + compose --profile research run --rm experiment \ + /app/lab3experiment \ + -base-url=http://nginx:8080 \ + -prometheus-url=http://prometheus:9090 \ + -label="$label" \ + -duration="$duration" \ + -warmup="$warmup" \ + -concurrency="$concurrency" \ + -replicas="$replicas" \ + -api-cpus="$api_cpus" \ + -api-memory="$api_memory" \ + -db-max-conns="$db_max_conns" \ + -output="/output/$output" + done + compose --profile research down -v --remove-orphans + current_project= +} + +trap restore_default_stack EXIT INT TERM +docker compose build api nginx swagger prometheus grafana +docker compose stop + +repeat=1 +while [ "$repeat" -le "$repeats" ]; do + run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies" + run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies" + run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies" + + run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies" + run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies" + run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies" + + run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64" + run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64" + repeat=$((repeat + 1)) +done + +docker compose --profile research run --no-deps --rm experiment \ + /app/lab3summary \ + -input=/output \ + -output=/output/lab3-scale-results.json \ + -chart=/output/lab3-scale-rps.svg +rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..5327251 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2 +exit 1