forked from mtusi2026/mtusi2026lab3
Lab 3: Nginx balancing and API failover #1
@@ -0,0 +1,87 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.repository }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab3-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
git init .
|
||||
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||
git fetch --depth=1 origin ${{ github.sha }}
|
||||
git checkout --detach FETCH_HEAD
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Check contract
|
||||
run: make check
|
||||
|
||||
- name: Validate OpenAPI
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
- name: Check formatting
|
||||
run: test -z "$(gofmt -l cmd internal migrations)"
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
with:
|
||||
version: v2.13.2
|
||||
args: --timeout=5m
|
||||
skip-cache: true
|
||||
|
||||
- name: Unit tests and build
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build ./cmd/...
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab3.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 120
|
||||
network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)")
|
||||
docker network connect "$network" "$HOSTNAME"
|
||||
printf '%s\n' "$network" > .ci-network
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
make test BASE_URL=http://nginx:8080
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||
|
||||
- name: Stop local stack
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi
|
||||
if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi
|
||||
@@ -0,0 +1,117 @@
|
||||
name: Deploy production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: production
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Verify selected commit
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab3-deploy-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
git init .
|
||||
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||
git fetch --depth=1 origin ${{ github.sha }}
|
||||
git checkout --detach FETCH_HEAD
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Check contract
|
||||
run: make check
|
||||
|
||||
- name: Validate OpenAPI
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
- name: Check formatting
|
||||
run: test -z "$(gofmt -l cmd internal migrations)"
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
with:
|
||||
version: v2.13.2
|
||||
args: --timeout=5m
|
||||
skip-cache: true
|
||||
|
||||
- name: Unit tests and build
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build ./cmd/...
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab3.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 120
|
||||
network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)")
|
||||
docker network connect "$network" "$HOSTNAME"
|
||||
printf '%s\n' "$network" > .ci-network
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
make test BASE_URL=http://nginx:8080
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||
|
||||
- name: Stop local stack
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi
|
||||
if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi
|
||||
|
||||
deploy:
|
||||
name: Deploy selected commit
|
||||
needs: verify
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
steps:
|
||||
- name: Configure SSH
|
||||
run: |
|
||||
umask 077
|
||||
mkdir -p ~/.ssh
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||
|
||||
- name: Deploy and verify
|
||||
run: |
|
||||
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
|
||||
set -eu
|
||||
cd ~/mtusi2026lab3
|
||||
git fetch origin main
|
||||
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||
git reset --hard '$DEPLOY_SHA'
|
||||
git clean -fd
|
||||
scripts/rolling-update.sh
|
||||
test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA'
|
||||
docker compose ps
|
||||
"
|
||||
+10
-6
@@ -2,16 +2,14 @@ FROM golang:1.26.5-alpine3.24 AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN go build ./cmd/messenger
|
||||
RUN go build -o /out/messenger ./cmd/messenger
|
||||
RUN go build -o /out/lab3experiment ./cmd/lab3experiment
|
||||
RUN go build -o /out/lab3summary ./cmd/lab3summary
|
||||
|
||||
FROM build AS test
|
||||
|
||||
CMD ["go", "test", "./internal/postgres", "./internal/session"]
|
||||
|
||||
FROM build AS experiment
|
||||
|
||||
CMD ["go", "run", "./cmd/lab2experiment"]
|
||||
|
||||
FROM swaggerapi/swagger-ui:v5.31.2 AS swagger
|
||||
|
||||
COPY contracts/openapi.json /spec/openapi.json
|
||||
@@ -26,10 +24,16 @@ FROM grafana/grafana:13.1.6 AS grafana
|
||||
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
|
||||
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
|
||||
|
||||
FROM nginx:1.30.5-alpine AS loadbalancer
|
||||
|
||||
COPY nginx/nginx.conf /etc/nginx/nginx.conf
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
||||
COPY --from=build --chown=app:app /src/messenger /app/messenger
|
||||
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
||||
COPY --from=build --chown=app:app /out/lab3experiment /app/lab3experiment
|
||||
COPY --from=build --chown=app:app /out/lab3summary /app/lab3summary
|
||||
|
||||
USER app
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,37 +1,197 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Пластинин Денис, БВТ2303
|
||||
- Целевая оценка: 5
|
||||
- Выбранный трек: 1 — цена масштабирования
|
||||
- Ветка/PR: `solution/bvt2303-plastinin`, PR #1
|
||||
- Среда эксперимента: Docker, `linux/arm64`, Go 1.26.5
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Git, Docker Engine с Docker Compose v2 и свободные локальные порты 8080, 8081, 9090 и 3000. Все внешние образы закреплены по версиям. Локальные пароли из примера предназначены только для изолированного стенда. Compose передаёт каждому контейнеру только нужные ему переменные: API получает строки подключения PostgreSQL/Valkey без отдельных исходных паролей и пароля Grafana, а `migrate` — только настройки PostgreSQL и миграций.
|
||||
|
||||
```sh
|
||||
cp lab3.env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
|
||||
```
|
||||
|
||||
Compose сначала ожидает PostgreSQL, затем однократно запускает контейнер `migrate`. API-реплики стартуют только после успешного завершения миграций. Единственная публичная точка API — Nginx на `http://localhost:8080`; порты отдельных API-контейнеров на хост не публикуются.
|
||||
|
||||
Доступные интерфейсы:
|
||||
|
||||
- Swagger UI: `http://localhost:8081`;
|
||||
- Prometheus: `http://localhost:9090`;
|
||||
- Grafana: `http://localhost:3000`, логин `admin`, локальный пароль берётся из `.env`;
|
||||
- Nginx API: `http://localhost:8080`.
|
||||
|
||||
Проверки запущенного стенда:
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test
|
||||
docker compose --profile tests run --rm --build tests
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
```
|
||||
|
||||
Безопасная остановка сохраняет данные в named volumes:
|
||||
|
||||
```sh
|
||||
docker compose stop
|
||||
```
|
||||
|
||||
Повторный запуск выполняется командой `docker compose up -d --scale api=2 --wait`. Намеренное удаление всех учебных данных:
|
||||
|
||||
```sh
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Клиент] -->|HTTP :8080| N[Nginx]
|
||||
N --> A1[API replica 1]
|
||||
N --> A2[API replica 2]
|
||||
N -. scale .-> AN[API replica N]
|
||||
A1 --> P[(PostgreSQL)]
|
||||
A2 --> P
|
||||
AN --> P
|
||||
A1 --> V[(Valkey)]
|
||||
A2 --> V
|
||||
AN --> V
|
||||
M[Однократный migrate] --> P
|
||||
PR[Prometheus] --> A1
|
||||
PR --> A2
|
||||
PR --> NE[Nginx exporter]
|
||||
PR --> PE[PostgreSQL exporter]
|
||||
PR --> VE[Valkey exporter]
|
||||
G[Grafana] --> PR
|
||||
```
|
||||
|
||||
PostgreSQL является источником истины для пользователей, чатов, сообщений и записей идемпотентности. Valkey хранит общие сессии, поэтому запросы с одним bearer token могут обслуживаться разными API-репликами. В Valkey ключ имеет вид `session:<sha256(token)>`, а значением служит JSON с `user_id` и сроком действия. Исходный случайный 256-битный токен возвращается клиенту один раз и не записывается в хранилище.
|
||||
|
||||
API остаётся stateless относительно пользовательских данных. Локально в процессе находятся только конфигурация, подключения, метрики конкретной реплики и флаг readiness. Каждый HTTP-ответ содержит `X-Instance-Id`, равный hostname контейнера.
|
||||
|
||||
### Балансировка и отказ реплики
|
||||
|
||||
Nginx OSS 1.30.5 использует round-robin, Docker DNS `127.0.0.11`, повторное разрешение имени `api` каждые 2 секунды и общий upstream zone. Keep-alive соединения ограничены 32. Таймаут подключения равен 1 секунде, таймауты отправки и чтения — 10 секунд. Ошибка соединения или ответ 502/503/504 временно исключают backend на 3 секунды; небезопасные POST автоматически не повторяются.
|
||||
|
||||
Docker проверяет `/health/ready` каждой API-реплики раз в 2 секунды с таймаутом 1 секунду. Readiness обращается к PostgreSQL и Valkey. При SIGTERM приложение сначала снимает readiness, затем вызывает `http.Server.Shutdown` с лимитом 5 секунд; Compose даёт контейнеру 8 секунд до принудительной остановки.
|
||||
|
||||
Nginx OSS в этом стенде использует пассивное исключение backend. Docker healthcheck не передаёт своё состояние напрямую в Nginx. Поэтому верхняя практическая граница обнаружения отказа складывается из connect timeout и fail timeout; в проверках восстановление произошло примерно за 1.4 секунды, что ниже требуемых 10 секунд.
|
||||
|
||||
### Общая идемпотентность
|
||||
|
||||
Для `POST /api/v1/chats/{chat_id}/messages` обязателен один заголовок `Idempotency-Key` длиной до 128 символов. Область ключа — `(user_id, chat_id, key)`, поэтому одинаковая строка независима для другого пользователя или чата. Минимальный TTL равен 24 часам.
|
||||
|
||||
Запись сообщения и запись идемпотентности создаются в одной транзакции PostgreSQL:
|
||||
|
||||
1. Проверяется участие пользователя в чате.
|
||||
2. Из текста считается SHA-256 для выявления повторного использования ключа с другим запросом.
|
||||
3. Строка идемпотентности блокируется через `SELECT ... FOR UPDATE`; конкурентные вставки разрешаются первичным ключом и `ON CONFLICT DO NOTHING`.
|
||||
4. Победившая транзакция резервирует UUID сообщения и вставляет сообщение. Отложенный внешний ключ проверяется при commit.
|
||||
5. Повтор с тем же телом читает исходное сообщение и возвращает тот же объект. Повтор с другим телом получает `409 Conflict`.
|
||||
|
||||
Это позволяет безопасно повторить POST после timeout: подтверждённая ранее запись не дублируется, даже если повтор попал на другую реплику. Каждый успешный запрос также удаляет до 100 истёкших записей по индексу `expires_at`; `SKIP LOCKED` не даёт параллельным репликам ждать одну и ту же пачку.
|
||||
|
||||
### Миграции, обновление и развёртывание
|
||||
|
||||
API не выполняет DDL при старте. Отдельная команда `/app/messenger migrate` берёт advisory lock и применяет versioned SQL один раз. Миграция идемпотентности только добавляет таблицу и индекс, поэтому совместима с уже работающей предыдущей версией.
|
||||
|
||||
`scripts/rolling-update.sh` собирает образы, запускает миграции, временно увеличивает число API с двух до трёх и поочерёдно заменяет старые контейнеры. Скрипт ждёт health каждого состава и затем возвращает две реплики. Workflow deploy запускается только вручную из `main` через `workflow_dispatch`; перед подключением к серверу он повторяет check, formatting, Go tests и build для SHA запуска. На сервере выполняется reset именно на этот SHA, даже если `main` успел измениться. Для SSH используются environment secrets `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY` и заранее проверенный `DEPLOY_KNOWN_HOSTS`; изменяемых actions для SSH нет.
|
||||
|
||||
## Наблюдаемость
|
||||
|
||||
HTTP middleware каждой реплики публикует число запросов по route/status class, histogram latency, in-flight gauge, число replay идемпотентности, Go runtime и process metrics. Prometheus использует DNS service discovery имени `api` с периодом обновления 2 секунды и отдельно опрашивает Nginx, PostgreSQL и Valkey exporters каждые 5 секунд. Данные хранятся в Prometheus TSDB с retention 7 суток в named volume.
|
||||
|
||||
Provisioned dashboard Grafana показывает общий RPS, RPS и p95 по репликам, классы HTTP-ответов, число healthy replicas, in-flight, heap, replay rate, активные соединения и запросы Nginx, соединения PostgreSQL и состояние Valkey.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `make test` через Nginx | PASS: регистрация, сессия, чат, пагинация, доступ и идемпотентность | `evidence/lab3-verification.txt` |
|
||||
| Общие сессии и ≥2 реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Одна сессия обслужена двумя instance ID | `evidence/lab3-verification.txt` |
|
||||
| Конкурентная идемпотентность | Go black-box через Nginx и database integration suite | 16 одновременных HTTP-запросов обработаны минимум двумя instance ID и вернули один message ID; история содержит одну запись | `evidence/lab3-verification.txt` |
|
||||
| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | API, PostgreSQL и Valkey перезапущены; старые данные и token доступны | `evidence/lab3-verification.txt` |
|
||||
| Отказ API | Два запуска failover с остановкой разных реплик | 125–131 успешная запись; 0–1 переходная ошибка; восстановление 1.35–1.44 с; дублей нет | `evidence/lab3-verification.txt` |
|
||||
| Rolling update | 60 секунд Go-нагрузки и `scripts/rolling-update.sh` | 134894 подтверждения, 2248.16 RPS, 0 ошибок, p95 13.12 ms | `evidence/lab3-rolling-update.json` |
|
||||
| CPU/RAM-лимиты | `docker compose config` и scale experiment | Лимиты заданы всем сервисам; при 128 MiB API не хватает памяти для Argon2id, поэтому рабочий минимум эксперимента — 256 MiB | `evidence/lab3-verification.txt` |
|
||||
| Трек 1 | `scripts/run-scale-experiment.sh` | 1/2/4 реплики, два бюджета, concurrency 8/32/64, три повтора | `evidence/lab3-scale-results.json`, `evidence/lab3-scale-rps.svg` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Методика исследования масштабирования
|
||||
|
||||
Гипотеза: при фиксированном суммарном бюджете дробление ресурсов между большим числом процессов сначала улучшит распределение конкурентных запросов, но затем проиграет из-за меньших CPU-квот и пулов каждой реплики. При добавлении одинаковых ресурсов на реплику RPS должен расти до насыщения PostgreSQL или Nginx.
|
||||
|
||||
Используется Go-клиент `cmd/lab3experiment`. Он создаёт отдельного пользователя, сессию и чат, прогревает стенд 2 секунды, затем 6 секунд выполняет closed-loop POST сообщений через Nginx. Для каждого запроса генерируется уникальный idempotency key. Считаются только HTTP-подтверждения; автоматических retry измеряемого трафика нет. Для каждой точки используются concurrency 8, 32 и 64 и три изолированных повтора с новой базой.
|
||||
|
||||
CPU и Nginx RPS считаются по разности монотонных `process_cpu_seconds_total` и `nginx_http_requests_total`, прочитанных непосредственно до и после измеряемого интервала. Поэтому bootstrap, warmup и простой между точками в них не входят. Resident memory и число healthy replicas снимаются после нагрузки с `/metrics` каждой реплики, соединения PostgreSQL — instant query к Prometheus. Короткая контрольная точка дала 2355.97 client RPS и 2359.64 RPS по Nginx-счётчику, расхождение 0.16%.
|
||||
|
||||
Фиксированная серия имеет общий бюджет API 2 CPU и 1 GiB RAM: `1×(2 CPU, 1 GiB)`, `2×(1 CPU, 512 MiB)`, `4×(0.5 CPU, 256 MiB)`. Пулы PostgreSQL соответственно равны 16, 8 и 4 соединениям на реплику. В растущей серии каждая реплика получает 1 CPU, 256 MiB и пул 8, поэтому суммарный бюджет растёт вместе с числом процессов.
|
||||
|
||||
Предварительный прогон с общим лимитом 512 MiB показал, что вариант `4×128 MiB` не выполняет регистрацию: базовое потребление процесса около 71 MiB, а Argon2id временно требует ещё 64 MiB. Cgroup завершает реплику, Nginx наблюдает premature upstream close. Этот результат определил минимальный рабочий лимит 256 MiB и не включён в сравнение throughput.
|
||||
|
||||
Медианные результаты фиксированного суммарного бюджета:
|
||||
|
||||
| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms |
|
||||
| ---: | ---: | ---: | ---: |
|
||||
| 8 | 2412 / 5.86 | 2484 / 5.59 | 2242 / 6.35 |
|
||||
| 32 | 2731 / 18.63 | 2365 / 24.93 | 2013 / 34.58 |
|
||||
| 64 | 2733 / 32.99 | 2198 / 55.06 | 2103 / 67.89 |
|
||||
|
||||
При фиксированных 2 CPU и 1 GiB увеличение числа процессов не даёт линейного выигрыша. Две реплики дают небольшой выигрыш только при concurrency 8, одна быстрее при 32 и 64, а четыре проигрывают во всех профилях из-за квоты 0.5 CPU на процесс и расходов четырёх runtime. Для рабочего стенда две реплики остаются осознанным компромиссом ради отказоустойчивости, дальнейшее дробление бюджета не оправдано.
|
||||
|
||||
Медианные результаты серии с 1 CPU и 256 MiB на каждую реплику:
|
||||
|
||||
| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms |
|
||||
| ---: | ---: | ---: | ---: |
|
||||
| 8 | 2712 / 4.95 | 2554 / 5.28 | 2185 / 7.05 |
|
||||
| 32 | 2818 / 15.34 | 2471 / 21.94 | 2130 / 24.14 |
|
||||
| 64 | 2044 / 61.44 | 2239 / 57.76 | 1859 / 67.42 |
|
||||
|
||||
Даже при росте суммарного CPU практический предел не превышает двух реплик. При concurrency 64 переход `1→2` повышает медианный RPS на 9.6%, а `2→4` снижает его на 17.0%. Одновременно число соединений PostgreSQL растёт `9→17→33`, суммарная resident memory — примерно `36→65→397 MiB`, а p95 у четырёх реплик достигает 67.42 ms. Точная средняя загрузка API составляет примерно `0.43→0.52→0.54 core` при доступных `1→2→4 core`. API не упирается в CPU-квоту. Вместе с ростом DB connections и tail latency это указывает на насыщение общего пути PostgreSQL/Nginx/network и расходы дополнительных процессов, но данный эксперимент не разделяет вклад этих компонентов.
|
||||
|
||||
Во всех 54 основных точках было 0 HTTP-ошибок; каждая конфигурация наблюдала все заявленные реплики. Диапазоны трёх повторов заметно пересекаются и доступны в raw JSON, поэтому небольшие отличия нельзя считать точной оценкой абсолютной производительности.
|
||||
|
||||
Для настройки пула конфигурация `4×0.5 CPU, 256 MiB` при concurrency 64 дополнительно сравнивалась с 10 и 4 соединениями PostgreSQL на реплику:
|
||||
|
||||
| Пул на реплику | Median RPS | p50 / p95 / p99, ms | Соединения PostgreSQL | Ошибки |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 10 | 1918 | 30.00 / 59.52 / 101.70 | 41 | 0 |
|
||||
| 4 | 2179 | 23.61 / 66.76 / 80.91 | 17 | 0 |
|
||||
|
||||
Уменьшение пула `10→4` сократило число соединений на 58.5%, повысило медианный RPS на 13.6%, уменьшило p50 на 21.3% и p99 на 20.4%. p95 вырос на 12.2%, то есть распределение задержек изменилось неравномерно. Для данного стенда выбран пул 4: он соответствует CPU-квоте 0.5 core и заметно снижает давление на общую БД. При строгом SLO на tail latency значение следует подбирать повторно на более длинном open-loop тесте.
|
||||
|
||||
Итоговый график находится в `evidence/lab3-scale-rps.svg`, обезличенные исходные точки, медианы и диапазоны — в `evidence/lab3-scale-results.json`.
|
||||
|
||||
### Отказ и обновление
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| SIGTERM одной API-реплики | Nginx исключает backend, другая реплика продолжает работу | 0–1 переходная ошибка, старый instance ID исчез из последующих ответов | 1.35–1.44 с; подтверждённые сообщения сохранены, дублей нет |
|
||||
| Полный restart API, PostgreSQL и Valkey без удаления volumes | Данные и сессия сохраняются | Старый bearer token и созданные сущности доступны после запуска | В пределах 90-секундного окна теста; потерь не обнаружено |
|
||||
| Rolling update API 2→3→2 | Во время замены остаются healthy backend | 134894 успешных POST, 0 ошибок; наблюдались старые и новые instance ID | Перерыва по подтверждённым ответам не зафиксировано |
|
||||
| Остановка PostgreSQL или Valkey | Readiness становится отрицательной | Все API теряют готовность, потому что хранилища общие | Автоматический failover хранилищ не реализован |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Все компоненты работают на одном Docker-хосте. Отказ хоста, Nginx, PostgreSQL или Valkey останавливает сервис; выполнен отказ процесса API, а не отказ узла.
|
||||
- Nginx является одной публичной точкой и SPOF. Для эксплуатации потребуются несколько LB с внешним virtual IP или управляемый балансировщик.
|
||||
- PostgreSQL и Valkey не реплицированы. AOF Valkey с `appendfsync everysec` ограничивает возможную потерю примерно последней секундой при аварии хранилища, но этот сценарий в работе не заявляется как защищённый.
|
||||
- Нагрузочный клиент closed-loop и подвержен coordinated omission. Результаты сравнивают конфигурации на одной машине и не являются абсолютной производительностью production-системы.
|
||||
- Все реплики и клиент делят ресурсы одного хоста; дельта process CPU показывает потребление API, а не полную стоимость Docker/network/storage.
|
||||
- Для эксплуатации нужны TLS, внешнее управление secrets, репликация хранилищ, алерты и длительный soak test; при большом потоке следует отдельно измерить стоимость пакетной очистки idempotency records.
|
||||
|
||||
## Источники и инструменты
|
||||
|
||||
- Nginx upstream и динамическое DNS: <https://nginx.org/en/docs/http/ngx_http_upstream_module.html>
|
||||
- Docker Compose services и healthcheck: <https://docs.docker.com/reference/compose-file/services/>
|
||||
- PostgreSQL explicit locking: <https://www.postgresql.org/docs/current/explicit-locking.html>
|
||||
- Valkey persistence: <https://valkey.io/topics/persistence/>
|
||||
- Argon2: RFC 9106, <https://www.rfc-editor.org/rfc/rfc9106>
|
||||
- Go packages: `pgx/v5`, `valkey-go`, `prometheus/client_golang`, `google/uuid`, `viper`, `x/crypto/argon2`.
|
||||
- OpenAI Codex использовался для подготовки кода, конфигурации и текста; реализация и результаты проверены запуском тестов и экспериментов.
|
||||
|
||||
+168
-3
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -19,6 +20,7 @@ type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
userID string
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -26,7 +28,7 @@ func main() {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("PASS: PostgreSQL and Valkey outage recovery")
|
||||
fmt.Println("PASS: replica idempotency and dependency recovery")
|
||||
}
|
||||
|
||||
func run() error {
|
||||
@@ -39,6 +41,9 @@ func run() error {
|
||||
if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.verifyConcurrentIdempotency(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyOutage(client, "valkey"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -62,9 +67,21 @@ func (c *apiClient) registerAndLogin() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("register: %w", err)
|
||||
}
|
||||
if err := closeResponse(response, http.StatusCreated); err != nil {
|
||||
return fmt.Errorf("register: %w", err)
|
||||
registrationBody := response.Body
|
||||
defer func() { _ = registrationBody.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("register status = %d", response.StatusCode)
|
||||
}
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil {
|
||||
return fmt.Errorf("decode user: %w", err)
|
||||
}
|
||||
if user.ID == "" {
|
||||
return fmt.Errorf("register returned an empty user id")
|
||||
}
|
||||
c.userID = user.ID
|
||||
|
||||
request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil)
|
||||
if err != nil {
|
||||
@@ -92,6 +109,154 @@ func (c *apiClient) registerAndLogin() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type messageResult struct {
|
||||
messageID string
|
||||
instanceID string
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *apiClient) verifyConcurrentIdempotency() error {
|
||||
chatID, err := c.createChat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
const workers = 16
|
||||
var (
|
||||
key = uuid.NewString()
|
||||
start = make(chan struct{})
|
||||
results = make(chan messageResult, workers)
|
||||
group sync.WaitGroup
|
||||
instances = make(map[string]struct{})
|
||||
messageID string
|
||||
)
|
||||
transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers}
|
||||
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
|
||||
defer transport.CloseIdleConnections()
|
||||
|
||||
for range workers {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
<-start
|
||||
id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message")
|
||||
results <- messageResult{messageID: id, instanceID: instance, err: requestErr}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
group.Wait()
|
||||
close(results)
|
||||
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
return result.err
|
||||
}
|
||||
if result.instanceID == "" {
|
||||
return fmt.Errorf("message response has an empty X-Instance-Id")
|
||||
}
|
||||
instances[result.instanceID] = struct{}{}
|
||||
if messageID == "" {
|
||||
messageID = result.messageID
|
||||
}
|
||||
if result.messageID != messageID {
|
||||
return fmt.Errorf("idempotent responses returned different message ids")
|
||||
}
|
||||
}
|
||||
if len(instances) < 2 {
|
||||
return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances))
|
||||
}
|
||||
return c.verifyMessageHistory(chatID, messageID)
|
||||
}
|
||||
|
||||
func (c *apiClient) createChat() (string, error) {
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"title": "Replica check", "member_ids": []string{c.userID},
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create chat: %w", err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("create chat status = %d", response.StatusCode)
|
||||
}
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil {
|
||||
return "", fmt.Errorf("decode chat: %w", err)
|
||||
}
|
||||
return chat.ID, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) {
|
||||
payload, err := json.Marshal(map[string]string{"text": text})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
request, err := http.NewRequest(
|
||||
http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", key)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("create message: %w", err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode)
|
||||
}
|
||||
var message struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil {
|
||||
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err)
|
||||
}
|
||||
return message.ID, response.Header.Get("X-Instance-Id"), nil
|
||||
}
|
||||
|
||||
func (c *apiClient) verifyMessageHistory(chatID, messageID string) error {
|
||||
request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list messages: %w", err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("list messages status = %d", response.StatusCode)
|
||||
}
|
||||
var page struct {
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil {
|
||||
return fmt.Errorf("decode messages: %w", err)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].ID != messageID {
|
||||
return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyOutage(client *apiClient, service string) (returnErr error) {
|
||||
if err := compose("stop", service); err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type configuration struct {
|
||||
baseURL string
|
||||
prometheusURL string
|
||||
label string
|
||||
duration time.Duration
|
||||
warmup time.Duration
|
||||
concurrency int
|
||||
replicas int
|
||||
apiCPUs string
|
||||
apiMemory string
|
||||
dbMaxConns int
|
||||
output string
|
||||
}
|
||||
|
||||
type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
chatID string
|
||||
}
|
||||
|
||||
type collector struct {
|
||||
mu sync.Mutex
|
||||
latencies []time.Duration
|
||||
errors map[string]int
|
||||
instances map[string]struct{}
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
GOOS string `json:"goos"`
|
||||
GOARCH string `json:"goarch"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latencySummary `json:"latency_ms"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type latencySummary struct {
|
||||
P50 float64 `json:"p50"`
|
||||
P95 float64 `json:"p95"`
|
||||
P99 float64 `json:"p99"`
|
||||
}
|
||||
|
||||
type runtimeSnapshot struct {
|
||||
apiCPUSeconds float64
|
||||
apiResident float64
|
||||
healthyReplicas float64
|
||||
nginxRequests float64
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
settings := parseFlags()
|
||||
client := &apiClient{
|
||||
baseURL: strings.TrimRight(settings.baseURL, "/"),
|
||||
client: &http.Client{Transport: &http.Transport{
|
||||
MaxIdleConns: settings.concurrency * 2,
|
||||
MaxIdleConnsPerHost: settings.concurrency,
|
||||
MaxConnsPerHost: settings.concurrency,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
}, Timeout: 10 * time.Second},
|
||||
}
|
||||
if err := client.waitForReplicas(settings.replicas); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.bootstrap(); err != nil {
|
||||
return err
|
||||
}
|
||||
if settings.warmup > 0 {
|
||||
warmup := newCollector()
|
||||
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
|
||||
return fmt.Errorf("warmup: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var before runtimeSnapshot
|
||||
if settings.prometheusURL != "" {
|
||||
snapshot, err := collectRuntimeSnapshot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("collect metrics before load: %w", err)
|
||||
}
|
||||
before = snapshot
|
||||
}
|
||||
measurements := newCollector()
|
||||
started := time.Now()
|
||||
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
|
||||
return err
|
||||
}
|
||||
elapsed := time.Since(started)
|
||||
summary := measurements.result(settings, elapsed)
|
||||
if settings.prometheusURL != "" {
|
||||
after, err := collectRuntimeSnapshot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("collect metrics after load: %w", err)
|
||||
}
|
||||
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeResult(settings.output, summary)
|
||||
}
|
||||
|
||||
func (c *apiClient) waitForReplicas(expected int) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
instances := make(map[string]struct{}, expected)
|
||||
for ctx.Err() == nil {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err == nil {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
_ = response.Body.Close()
|
||||
if response.StatusCode == http.StatusOK {
|
||||
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
|
||||
instances[instanceID] = struct{}{}
|
||||
}
|
||||
if len(instances) == expected {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
|
||||
}
|
||||
|
||||
func parseFlags() configuration {
|
||||
var settings configuration
|
||||
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
|
||||
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
|
||||
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
|
||||
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
|
||||
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
|
||||
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
|
||||
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
|
||||
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
|
||||
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
|
||||
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
|
||||
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
|
||||
flag.Parse()
|
||||
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
|
||||
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
|
||||
os.Exit(2)
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
func (c *apiClient) bootstrap() error {
|
||||
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
|
||||
password := "Synthetic-" + uuid.NewString()
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
|
||||
"username": username, "display_name": "Scale experiment", "password": password,
|
||||
}, &user, http.StatusCreated); err != nil {
|
||||
return fmt.Errorf("register: %w", err)
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.SetBasicAuth(username, password)
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("login: %w", err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return responseError("login", response)
|
||||
}
|
||||
var session struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
||||
return fmt.Errorf("decode session: %w", err)
|
||||
}
|
||||
c.token = session.Token
|
||||
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
|
||||
"title": "Scale experiment", "member_ids": []string{user.ID},
|
||||
}, &chat, http.StatusCreated); err != nil {
|
||||
return fmt.Errorf("create chat: %w", err)
|
||||
}
|
||||
c.chatID = chat.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), duration)
|
||||
defer cancel()
|
||||
var sequence atomic.Uint64
|
||||
var group sync.WaitGroup
|
||||
for range concurrency {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for ctx.Err() == nil {
|
||||
number := sequence.Add(1)
|
||||
started := time.Now()
|
||||
instanceID, status, err := client.createMessage(ctx, number)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
measurements.observe(time.Since(started), instanceID, status, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
if len(measurements.latencies) == 0 {
|
||||
return errors.New("no successful requests")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
|
||||
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(
|
||||
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
|
||||
)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", uuid.NewString())
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
|
||||
}
|
||||
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
|
||||
}
|
||||
|
||||
func newCollector() *collector {
|
||||
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if err != nil {
|
||||
key := "network"
|
||||
if status != 0 {
|
||||
key = strconv.Itoa(status)
|
||||
}
|
||||
c.errors[key]++
|
||||
return
|
||||
}
|
||||
c.latencies = append(c.latencies, latency)
|
||||
if instanceID != "" {
|
||||
c.instances[instanceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *collector) result(settings configuration, elapsed time.Duration) result {
|
||||
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
|
||||
return result{
|
||||
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
|
||||
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
|
||||
Concurrency: settings.concurrency, Replicas: settings.replicas,
|
||||
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
|
||||
SuccessfulRequests: len(c.latencies),
|
||||
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
|
||||
LatencyMS: latencySummary{
|
||||
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
|
||||
},
|
||||
ObservedInstances: len(c.instances),
|
||||
}
|
||||
}
|
||||
|
||||
func percentile(values []time.Duration, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
index := int(float64(len(values)-1) * percentile)
|
||||
return float64(values[index].Microseconds()) / 1000
|
||||
}
|
||||
|
||||
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
|
||||
hosts, err := net.LookupHost("api")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
|
||||
}
|
||||
uniqueHosts := make(map[string]struct{}, len(hosts))
|
||||
var snapshot runtimeSnapshot
|
||||
for _, host := range hosts {
|
||||
if _, duplicate := uniqueHosts[host]; duplicate {
|
||||
continue
|
||||
}
|
||||
uniqueHosts[host] = struct{}{}
|
||||
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
|
||||
}
|
||||
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
snapshot.apiCPUSeconds += cpuSeconds
|
||||
snapshot.apiResident += resident
|
||||
snapshot.healthyReplicas++
|
||||
}
|
||||
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
|
||||
}
|
||||
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func scrapeMetrics(endpoint string) (map[string]float64, error) {
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Get(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("status %d", response.StatusCode)
|
||||
}
|
||||
metrics := make(map[string]float64)
|
||||
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
name, _, _ := strings.Cut(fields[0], "{")
|
||||
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse metric %s: %w", name, err)
|
||||
}
|
||||
metrics[name] += value
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
|
||||
value, found := metrics[name]
|
||||
if !found {
|
||||
return 0, fmt.Errorf("metric %s not found", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
|
||||
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
|
||||
}
|
||||
seconds := elapsed.Seconds()
|
||||
return map[string]float64{
|
||||
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
|
||||
"api_resident_bytes": after.apiResident,
|
||||
"healthy_replicas": after.healthyReplicas,
|
||||
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
|
||||
"postgres_connections": postgresConnections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func prometheusQuery(baseURL, query string) (float64, error) {
|
||||
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Get(endpoint)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total float64
|
||||
for _, item := range payload.Data.Result {
|
||||
if len(item.Value) != 2 {
|
||||
continue
|
||||
}
|
||||
var encoded string
|
||||
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.ParseFloat(encoded, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += value
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if token != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if idempotencyKey != "" {
|
||||
request.Header.Set("Idempotency-Key", idempotencyKey)
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != expected {
|
||||
return responseError(method+" "+path, response)
|
||||
}
|
||||
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
|
||||
}
|
||||
|
||||
func responseError(action string, response *http.Response) error {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
|
||||
}
|
||||
|
||||
func writeResult(path string, value result) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if path == "" {
|
||||
_, err = os.Stdout.Write(data)
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type latency struct {
|
||||
P50 float64 `json:"p50"`
|
||||
P95 float64 `json:"p95"`
|
||||
P99 float64 `json:"p99"`
|
||||
}
|
||||
|
||||
type inputResult struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latency `json:"latency_ms"`
|
||||
Instances []string `json:"instances"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type rawResult struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latency `json:"latency_ms"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type aggregate struct {
|
||||
Label string `json:"label"`
|
||||
Repeats int `json:"repeats"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRPSMedian float64 `json:"successful_rps_median"`
|
||||
SuccessfulRPSMin float64 `json:"successful_rps_min"`
|
||||
SuccessfulRPSMax float64 `json:"successful_rps_max"`
|
||||
LatencyP50MedianMS float64 `json:"latency_p50_median_ms"`
|
||||
LatencyP95MedianMS float64 `json:"latency_p95_median_ms"`
|
||||
LatencyP99MedianMS float64 `json:"latency_p99_median_ms"`
|
||||
TotalErrors int `json:"total_errors"`
|
||||
ObservedInstancesMin int `json:"observed_instances_min"`
|
||||
APICPUCoresMedian float64 `json:"api_cpu_cores_median"`
|
||||
APIResidentBytesMedian float64 `json:"api_resident_bytes_median"`
|
||||
PostgresConnectionsMedian float64 `json:"postgres_connections_median"`
|
||||
PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Method string `json:"method"`
|
||||
Raw []rawResult `json:"raw"`
|
||||
Summary []aggregate `json:"summary"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files")
|
||||
outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output")
|
||||
chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output")
|
||||
flag.Parse()
|
||||
|
||||
results, err := readResults(*inputDirectory)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
aggregates := summarize(results)
|
||||
if err := writeJSON(*outputPath, report{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Method: "closed-loop HTTP through Nginx; median of three isolated repeats",
|
||||
Raw: sanitize(results),
|
||||
Summary: aggregates,
|
||||
}); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := writeChart(*chartPath, aggregates); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readResults(directory string) ([]inputResult, error) {
|
||||
var paths []string
|
||||
for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} {
|
||||
matches, err := filepath.Glob(filepath.Join(directory, pattern))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths = append(paths, matches...)
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return nil, fmt.Errorf("no experiment results in %s", directory)
|
||||
}
|
||||
|
||||
results := make([]inputResult, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result inputResult
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %w", path, err)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label })
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func sanitize(results []inputResult) []rawResult {
|
||||
raw := make([]rawResult, 0, len(results))
|
||||
for _, value := range results {
|
||||
raw = append(raw, rawResult{
|
||||
Label: value.Label, Timestamp: value.Timestamp,
|
||||
DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds,
|
||||
Concurrency: value.Concurrency, Replicas: value.Replicas,
|
||||
APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns,
|
||||
SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS,
|
||||
Errors: value.Errors, LatencyMS: value.LatencyMS,
|
||||
ObservedInstances: instanceCount(value), Prometheus: value.Prometheus,
|
||||
})
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func summarize(results []inputResult) []aggregate {
|
||||
groups := make(map[string][]inputResult)
|
||||
for _, value := range results {
|
||||
groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value)
|
||||
}
|
||||
|
||||
labels := make([]string, 0, len(groups))
|
||||
for label := range groups {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
sort.Strings(labels)
|
||||
|
||||
aggregates := make([]aggregate, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
values := groups[label]
|
||||
first := values[0]
|
||||
rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS })
|
||||
p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 })
|
||||
p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 })
|
||||
p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 })
|
||||
cpu := metricNumbers(values, "api_cpu_cores")
|
||||
memory := metricNumbers(values, "api_resident_bytes")
|
||||
connections := metricNumbers(values, "postgres_connections")
|
||||
healthy := metricNumbers(values, "healthy_replicas")
|
||||
aggregate := aggregate{
|
||||
Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas,
|
||||
APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns,
|
||||
SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps),
|
||||
LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99),
|
||||
ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu),
|
||||
APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections),
|
||||
PrometheusHealthyMin: minimum(healthy),
|
||||
}
|
||||
for _, value := range values {
|
||||
if count := instanceCount(value); count < aggregate.ObservedInstancesMin {
|
||||
aggregate.ObservedInstancesMin = count
|
||||
}
|
||||
for _, count := range value.Errors {
|
||||
aggregate.TotalErrors += count
|
||||
}
|
||||
}
|
||||
aggregates = append(aggregates, aggregate)
|
||||
}
|
||||
return aggregates
|
||||
}
|
||||
|
||||
func instanceCount(value inputResult) int {
|
||||
if value.ObservedInstances > 0 {
|
||||
return value.ObservedInstances
|
||||
}
|
||||
return len(value.Instances)
|
||||
}
|
||||
|
||||
func withoutRepeat(label string) string {
|
||||
index := strings.LastIndex(label, "-n")
|
||||
if index == -1 {
|
||||
return label
|
||||
}
|
||||
if _, err := strconv.Atoi(label[index+2:]); err != nil {
|
||||
return label
|
||||
}
|
||||
return label[:index]
|
||||
}
|
||||
|
||||
func numbers(values []inputResult, value func(inputResult) float64) []float64 {
|
||||
result := make([]float64, 0, len(values))
|
||||
for _, item := range values {
|
||||
result = append(result, value(item))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func metricNumbers(values []inputResult, name string) []float64 {
|
||||
result := make([]float64, 0, len(values))
|
||||
for _, item := range values {
|
||||
if value, ok := item.Prometheus[name]; ok {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func median(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
values = append([]float64(nil), values...)
|
||||
sort.Float64s(values)
|
||||
middle := len(values) / 2
|
||||
if len(values)%2 == 1 {
|
||||
return values[middle]
|
||||
}
|
||||
return (values[middle-1] + values[middle]) / 2
|
||||
}
|
||||
|
||||
func minimum(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
minimum := values[0]
|
||||
for _, value := range values[1:] {
|
||||
if value < minimum {
|
||||
minimum = value
|
||||
}
|
||||
}
|
||||
return minimum
|
||||
}
|
||||
|
||||
func maximum(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
maximum := values[0]
|
||||
for _, value := range values[1:] {
|
||||
if value > maximum {
|
||||
maximum = value
|
||||
}
|
||||
}
|
||||
return maximum
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func writeChart(path string, values []aggregate) error {
|
||||
const width, height = 960, 560
|
||||
const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0
|
||||
plotWidth := width - left - right
|
||||
plotHeight := height - top - bottom
|
||||
|
||||
var selected []aggregate
|
||||
var maxRPS float64
|
||||
for _, value := range values {
|
||||
if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") {
|
||||
selected = append(selected, value)
|
||||
if value.SuccessfulRPSMedian > maxRPS {
|
||||
maxRPS = value.SuccessfulRPSMedian
|
||||
}
|
||||
}
|
||||
}
|
||||
if maxRPS == 0 {
|
||||
return fmt.Errorf("no scale results for chart")
|
||||
}
|
||||
maxRPS *= 1.1
|
||||
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, `<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">`, width, height, width, height)
|
||||
output.WriteString(`<rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style>`)
|
||||
fmt.Fprintf(&output, `<text x="%g" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text>`, left)
|
||||
|
||||
for step := 0; step <= 5; step++ {
|
||||
y := top + plotHeight*float64(5-step)/5
|
||||
value := maxRPS * float64(step) / 5
|
||||
fmt.Fprintf(&output, `<line class="grid" x1="%g" y1="%g" x2="%g" y2="%g"/><text x="%g" y="%g" text-anchor="end" font-size="12">%.0f</text>`, left, y, left+plotWidth, y, left-10, y+4, value)
|
||||
}
|
||||
fmt.Fprintf(&output, `<line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/><line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/>`, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight)
|
||||
for index, replicas := range []int{1, 2, 4} {
|
||||
x := left + plotWidth*float64(index)/2
|
||||
fmt.Fprintf(&output, `<text x="%g" y="%g" text-anchor="middle" font-size="13">%d</text>`, x, top+plotHeight+25, replicas)
|
||||
}
|
||||
fmt.Fprintf(&output, `<text x="%g" y="%d" text-anchor="middle" font-size="13">API replicas</text>`, left+plotWidth/2, height-15)
|
||||
|
||||
colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"}
|
||||
type series struct {
|
||||
name string
|
||||
values []aggregate
|
||||
}
|
||||
groups := make(map[string][]aggregate)
|
||||
for _, value := range selected {
|
||||
name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency)
|
||||
groups[name] = append(groups[name], value)
|
||||
}
|
||||
var names []string
|
||||
for name := range groups {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
allSeries := make([]series, 0, len(names))
|
||||
for _, name := range names {
|
||||
points := groups[name]
|
||||
sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas })
|
||||
allSeries = append(allSeries, series{name: name, values: points})
|
||||
}
|
||||
for index, current := range allSeries {
|
||||
color := colors[index%len(colors)]
|
||||
var coordinates []string
|
||||
for _, point := range current.values {
|
||||
xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas]
|
||||
x := left + plotWidth*xIndex/2
|
||||
y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS)
|
||||
coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y))
|
||||
fmt.Fprintf(&output, `<circle class="point" cx="%g" cy="%g" r="4" fill="%s"/>`, x, y, color)
|
||||
}
|
||||
fmt.Fprintf(&output, `<polyline class="line" points="%s" stroke="%s"/>`, strings.Join(coordinates, " "), color)
|
||||
legendY := top + 20 + float64(index*28)
|
||||
fmt.Fprintf(&output, `<line x1="%g" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="3"/><text x="%g" y="%g" font-size="12">%s</text>`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name))
|
||||
}
|
||||
output.WriteString(`</svg>`)
|
||||
return os.WriteFile(path, []byte(output.String()), 0o644)
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
+43
-10
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -9,33 +10,46 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/messenger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/messenger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
|
||||
)
|
||||
|
||||
func main() {
|
||||
appLogger := logger.New()
|
||||
if err := run(appLogger); err != nil {
|
||||
if err := run(appLogger, os.Args[1:]); err != nil {
|
||||
appLogger.Error("server stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger) error {
|
||||
func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 0 {
|
||||
return runServer(logger)
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return runMigrations()
|
||||
}
|
||||
return fmt.Errorf("usage: messenger [migrate]")
|
||||
}
|
||||
|
||||
func runServer(logger *slog.Logger) error {
|
||||
settings, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
MigrationTimeout: settings.MigrationTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -60,7 +74,9 @@ func run(logger *slog.Logger) error {
|
||||
sessions,
|
||||
passwords,
|
||||
settings.SessionTTL,
|
||||
settings.IdempotencyTTL,
|
||||
settings.RequestTimeout,
|
||||
instanceID,
|
||||
settings.SwaggerPort,
|
||||
)
|
||||
server := &http.Server{
|
||||
@@ -97,3 +113,20 @@ func run(logger *slog.Logger) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runMigrations() error {
|
||||
settings, err := config.LoadMigration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||
}
|
||||
|
||||
+125
-16
@@ -32,31 +32,107 @@ services:
|
||||
mem_limit: ${VALKEY_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab2}
|
||||
env_file: .env
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:${APP_PORT}"
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab3}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: ${MIGRATE_CPUS}
|
||||
mem_limit: ${MIGRATE_MEMORY}
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab3}
|
||||
environment:
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
CACHE_URL: ${CACHE_URL}
|
||||
SESSION_STORE: ${SESSION_STORE}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
|
||||
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
|
||||
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
|
||||
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
|
||||
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
|
||||
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
|
||||
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
|
||||
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
|
||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
start_period: 5s
|
||||
retries: 2
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
cpus: ${API_CPUS}
|
||||
mem_limit: ${API_MEMORY}
|
||||
stop_grace_period: 10s
|
||||
stop_grace_period: 8s
|
||||
restart: unless-stopped
|
||||
|
||||
nginx:
|
||||
build:
|
||||
context: .
|
||||
target: loadbalancer
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab3}
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
cpus: ${NGINX_CPUS}
|
||||
mem_limit: ${NGINX_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
swagger:
|
||||
build:
|
||||
context: .
|
||||
target: swagger
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab3}
|
||||
ports:
|
||||
- "${SWAGGER_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
api:
|
||||
nginx:
|
||||
condition: service_healthy
|
||||
cpus: ${SWAGGER_CPUS}
|
||||
mem_limit: ${SWAGGER_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
tests:
|
||||
@@ -68,23 +144,30 @@ services:
|
||||
TEST_DATABASE_URL: ${DATABASE_URL}
|
||||
TEST_CACHE_URL: ${CACHE_URL}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
cpus: ${TEST_CPUS}
|
||||
mem_limit: ${TEST_MEMORY}
|
||||
|
||||
experiment:
|
||||
profiles: ["research"]
|
||||
build:
|
||||
context: .
|
||||
target: experiment
|
||||
env_file: .env
|
||||
command: ["go", "run", "./cmd/lab2experiment", "-output=/output/lab2-track2.json"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab3}
|
||||
command:
|
||||
- /app/lab3experiment
|
||||
- -base-url=http://nginx:8080
|
||||
- -prometheus-url=http://prometheus:9090
|
||||
- -output=/output/lab3-scale.json
|
||||
volumes:
|
||||
- ./evidence:/output
|
||||
depends_on:
|
||||
postgres:
|
||||
nginx:
|
||||
condition: service_healthy
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
cpus: ${TEST_CPUS}
|
||||
mem_limit: ${TEST_MEMORY}
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
|
||||
@@ -96,6 +179,8 @@ services:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
valkey-exporter:
|
||||
@@ -107,12 +192,25 @@ services:
|
||||
depends_on:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
nginx-exporter:
|
||||
image: nginx/nginx-prometheus-exporter:1.5.1
|
||||
command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"]
|
||||
depends_on:
|
||||
nginx:
|
||||
condition: service_healthy
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
build:
|
||||
context: .
|
||||
target: prometheus
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab3}
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
@@ -121,16 +219,25 @@ services:
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "${PROMETHEUS_BIND_ADDRESS}:9090"
|
||||
healthcheck:
|
||||
test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
depends_on:
|
||||
- api
|
||||
- nginx-exporter
|
||||
- postgres-exporter
|
||||
- valkey-exporter
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
build:
|
||||
context: .
|
||||
target: grafana
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab3}
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
|
||||
@@ -141,6 +248,8 @@ services:
|
||||
- "${GRAFANA_BIND_ADDRESS}:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
cpus: ${GRAFANA_CPUS}
|
||||
mem_limit: ${GRAFANA_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"label": "rolling-update",
|
||||
"timestamp": "2026-10-02T09:54:15Z",
|
||||
"goos": "linux",
|
||||
"goarch": "arm64",
|
||||
"duration_seconds": 60.001996238,
|
||||
"warmup_seconds": 3,
|
||||
"concurrency": 16,
|
||||
"replicas": 2,
|
||||
"api_cpus_per_replica": "1",
|
||||
"api_memory_per_replica": "256m",
|
||||
"db_max_connections_per_replica": 10,
|
||||
"successful_requests": 134894,
|
||||
"successful_rps": 2248.158535675018,
|
||||
"errors": {},
|
||||
"latency_ms": {
|
||||
"p50": 6.41,
|
||||
"p95": 13.12,
|
||||
"p99": 22.006
|
||||
},
|
||||
"observed_instances": 5,
|
||||
"prometheus": {
|
||||
"api_cpu_cores": 0.4491430158747135,
|
||||
"api_resident_bytes": 58523648,
|
||||
"healthy_replicas": 2,
|
||||
"nginx_rps": 2253.547396138092,
|
||||
"postgres_connections": 21
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="960" height="560" viewBox="0 0 960 560"><rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style><text x="85" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text><line class="grid" x1="85" y1="490" x2="720" y2="490"/><text x="75" y="494" text-anchor="end" font-size="12">0</text><line class="grid" x1="85" y1="401" x2="720" y2="401"/><text x="75" y="405" text-anchor="end" font-size="12">620</text><line class="grid" x1="85" y1="312" x2="720" y2="312"/><text x="75" y="316" text-anchor="end" font-size="12">1240</text><line class="grid" x1="85" y1="223" x2="720" y2="223"/><text x="75" y="227" text-anchor="end" font-size="12">1860</text><line class="grid" x1="85" y1="134" x2="720" y2="134"/><text x="75" y="138" text-anchor="end" font-size="12">2480</text><line class="grid" x1="85" y1="45" x2="720" y2="45"/><text x="75" y="49" text-anchor="end" font-size="12">3100</text><line class="axis" x1="85" y1="45" x2="85" y2="490"/><line class="axis" x1="85" y1="490" x2="720" y2="490"/><text x="85" y="515" text-anchor="middle" font-size="13">1</text><text x="402.5" y="515" text-anchor="middle" font-size="13">2</text><text x="720" y="515" text-anchor="middle" font-size="13">4</text><text x="402.5" y="545" text-anchor="middle" font-size="13">API replicas</text><circle class="point" cx="85" cy="97.94469794128702" r="4" fill="#1a73e8"/><circle class="point" cx="402.5" cy="150.47324315126576" r="4" fill="#1a73e8"/><circle class="point" cx="720" cy="200.9869273874724" r="4" fill="#1a73e8"/><polyline class="line" points="85,97.94469794128702 402.5,150.47324315126576 720,200.9869273874724" stroke="#1a73e8"/><line x1="745" y1="65" x2="770" y2="65" stroke="#1a73e8" stroke-width="3"/><text x="778" y="69" font-size="12">fixed, c=32</text><circle class="point" cx="85" cy="97.59212639106859" r="4" fill="#d93025"/><circle class="point" cx="402.5" cy="174.50312647976264" r="4" fill="#d93025"/><circle class="point" cx="720" cy="188.14076549389546" r="4" fill="#d93025"/><polyline class="line" points="85,97.59212639106859 402.5,174.50312647976264 720,188.14076549389546" stroke="#d93025"/><line x1="745" y1="93" x2="770" y2="93" stroke="#d93025" stroke-width="3"/><text x="778" y="97" font-size="12">fixed, c=64</text><circle class="point" cx="85" cy="143.7066057420492" r="4" fill="#188038"/><circle class="point" cx="402.5" cy="133.35012431532638" r="4" fill="#188038"/><circle class="point" cx="720" cy="168.15646800421806" r="4" fill="#188038"/><polyline class="line" points="85,143.7066057420492 402.5,133.35012431532638 720,168.15646800421806" stroke="#188038"/><line x1="745" y1="121" x2="770" y2="121" stroke="#188038" stroke-width="3"/><text x="778" y="125" font-size="12">fixed, c=8</text><circle class="point" cx="85" cy="85.45454545454552" r="4" fill="#9334e6"/><circle class="point" cx="402.5" cy="135.30606743203387" r="4" fill="#9334e6"/><circle class="point" cx="720" cy="184.2610413680913" r="4" fill="#9334e6"/><polyline class="line" points="85,85.45454545454552 402.5,135.30606743203387 720,184.2610413680913" stroke="#9334e6"/><line x1="745" y1="149" x2="770" y2="149" stroke="#9334e6" stroke-width="3"/><text x="778" y="153" font-size="12">growing, c=32</text><circle class="point" cx="85" cy="196.57385409938001" r="4" fill="#f29900"/><circle class="point" cx="402.5" cy="168.53866266092896" r="4" fill="#f29900"/><circle class="point" cx="720" cy="223.13942705833085" r="4" fill="#f29900"/><polyline class="line" points="85,196.57385409938001 402.5,168.53866266092896 720,223.13942705833085" stroke="#f29900"/><line x1="745" y1="177" x2="770" y2="177" stroke="#f29900" stroke-width="3"/><text x="778" y="181" font-size="12">growing, c=64</text><circle class="point" cx="85" cy="100.60936627447775" r="4" fill="#00897b"/><circle class="point" cx="402.5" cy="123.35560037954652" r="4" fill="#00897b"/><circle class="point" cx="720" cy="176.3488242715569" r="4" fill="#00897b"/><polyline class="line" points="85,100.60936627447775 402.5,123.35560037954652 720,176.3488242715569" stroke="#00897b"/><line x1="745" y1="205" x2="770" y2="205" stroke="#00897b" stroke-width="3"/><text x="778" y="209" font-size="12">growing, c=8</text></svg>
|
||||
|
After Width: | Height: | Size: 4.3 KiB |
@@ -0,0 +1,51 @@
|
||||
Lab 3 verification
|
||||
Date: 2026-10-02
|
||||
Environment: Docker linux/arm64, Go 1.26.5
|
||||
|
||||
Contract and build
|
||||
- make check: PASS
|
||||
- docker compose --env-file .env config --quiet: PASS
|
||||
- go test ./cmd/lab3experiment: PASS
|
||||
- go test ./internal/messenger ./internal/model ./internal/password: PASS
|
||||
|
||||
Integration
|
||||
- docker compose --profile tests run --rm --build tests: PASS
|
||||
- PostgreSQL concurrent idempotency: 8 calls through two Store instances, one message ID and one row
|
||||
- black-box replica idempotency: 16 concurrent calls through Nginx, at least two instance IDs, one message ID and one history item
|
||||
- PostgreSQL and Valkey stop/recovery: PASS
|
||||
- persistence after restarting API, PostgreSQL and Valkey: PASS; the original session and data remained valid
|
||||
- public smoke through Nginx: PASS
|
||||
|
||||
API replica failover
|
||||
- run 1: 2 instances observed, 131 successful writes, 0 transition errors, first recovered write after 1.353 s
|
||||
- run 2: 2 instances observed, 125 successful writes, 1 transition error, first recovered write after 1.442 s
|
||||
- confirmed messages remained readable and idempotent retries did not create duplicates
|
||||
|
||||
Rolling update under Go load
|
||||
- duration: 60.002 s
|
||||
- concurrency: 16
|
||||
- successful requests: 134894
|
||||
- successful RPS: 2248.16
|
||||
- errors: 0
|
||||
- latency p50/p95/p99: 6.41/13.12/22.01 ms
|
||||
- healthy API replicas after update: 2
|
||||
|
||||
Resource boundary
|
||||
- 4 API replicas with 128 MiB each restarted while Argon2id processed registration
|
||||
- idle process memory was about 71 MiB; Argon2id memory parameter was 64 MiB
|
||||
- the fixed-budget experiment therefore uses 256 MiB per replica as its minimum
|
||||
|
||||
Scale experiment
|
||||
- command: DURATION=6s WARMUP=2s REPEATS=3 CONCURRENCIES='8 32 64' scripts/run-scale-experiment.sh
|
||||
- configurations: fixed total API resources and growing per-replica resources, 1/2/4 replicas
|
||||
- completed points: 60; HTTP errors: 0; every point observed all configured replicas
|
||||
- CPU and Nginx RPS use counter deltas read immediately before and after measured load
|
||||
- maximum difference between client RPS and Nginx counter RPS across 60 points: 0.75%
|
||||
- reducing the pool from 10 to 4 connections per replica changed median RPS from 1918 to 2179 and PostgreSQL connections from 41 to 17
|
||||
- combined sanitized raw data and median aggregates: evidence/lab3-scale-results.json
|
||||
- comparison chart: evidence/lab3-scale-rps.svg
|
||||
|
||||
Configuration scope
|
||||
- API receives only its application settings and PostgreSQL/Valkey connection URLs
|
||||
- migrate receives only PostgreSQL pool, dependency and migration settings
|
||||
- neither container receives the Grafana password or separate source passwords of unrelated services
|
||||
@@ -1,4 +1,4 @@
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab2
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab3
|
||||
|
||||
go 1.26.0
|
||||
|
||||
@@ -7,6 +7,7 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
golang.org/x/crypto v0.57.0
|
||||
)
|
||||
@@ -14,6 +15,7 @@ require (
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
@@ -21,6 +23,7 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
@@ -36,4 +39,5 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
+85
-32
@@ -16,13 +16,13 @@ type Config struct {
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
MigrationTimeout time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HTTPReadHeaderTimeout time.Duration
|
||||
HTTPReadTimeout time.Duration
|
||||
@@ -37,6 +37,27 @@ type Config struct {
|
||||
SwaggerPort string
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
MigrationTimeout time.Duration
|
||||
}
|
||||
|
||||
type databaseConfig struct {
|
||||
url string
|
||||
maxConns int
|
||||
minConns int
|
||||
maxConnectionAge time.Duration
|
||||
maxConnectionIdle time.Duration
|
||||
healthCheckPeriod time.Duration
|
||||
dependencyTimeout time.Duration
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
@@ -49,7 +70,7 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -69,31 +90,7 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
|
||||
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -147,11 +144,12 @@ func Load() (Config, error) {
|
||||
}
|
||||
|
||||
return Config{
|
||||
Address: net.JoinHostPort(host, port), DatabaseURL: databaseURL, CacheURL: cacheURL,
|
||||
SessionTTL: sessionTTL, DBMaxConns: int32(maxConns), DBMinConns: int32(minConns),
|
||||
DBMaxConnectionAge: maxConnectionAge, DBMaxConnectionIdle: maxConnectionIdle,
|
||||
DBHealthCheckPeriod: healthCheckPeriod, DependencyTimeout: dependencyTimeout,
|
||||
MigrationTimeout: migrationTimeout, RequestTimeout: requestTimeout,
|
||||
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
||||
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
|
||||
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
RequestTimeout: requestTimeout,
|
||||
HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout,
|
||||
HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout,
|
||||
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
|
||||
@@ -161,6 +159,61 @@ func Load() (Config, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadMigration() (MigrationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
return MigrationConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
MigrationTimeout: migrationTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
return databaseConfig{
|
||||
url: databaseURL, maxConns: maxConns, minConns: minConns,
|
||||
maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle,
|
||||
healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func requiredString(values *viper.Viper, name string) (string, error) {
|
||||
value := strings.TrimSpace(values.GetString(name))
|
||||
if value == "" {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/migrations"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/migrations"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -20,7 +20,6 @@ var (
|
||||
errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive")
|
||||
errHealthCheckPeriod = errors.New("database health check period must be positive")
|
||||
errConnectionTimeout = errors.New("database connection timeout must be positive")
|
||||
errMigrationTimeout = errors.New("database migration timeout must be positive")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -31,7 +30,6 @@ type Config struct {
|
||||
MaxConnectionIdle time.Duration
|
||||
HealthCheckPeriod time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
MigrationTimeout time.Duration
|
||||
}
|
||||
|
||||
func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
@@ -59,15 +57,18 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
return nil, fmt.Errorf("ping database: %w", err)
|
||||
}
|
||||
|
||||
migrationCtx, cancel := context.WithTimeout(ctx, config.MigrationTimeout)
|
||||
defer cancel()
|
||||
if err := applyMigrations(migrationCtx, pool, migrations.FS); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error {
|
||||
if timeout <= 0 {
|
||||
return errors.New("database migration timeout must be positive")
|
||||
}
|
||||
migrationCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
return applyMigrations(migrationCtx, pool, migrations.FS)
|
||||
}
|
||||
|
||||
func validateConfig(config Config) error {
|
||||
switch {
|
||||
case config.URL == "":
|
||||
@@ -86,8 +87,6 @@ func validateConfig(config Config) error {
|
||||
return errHealthCheckPeriod
|
||||
case config.ConnectTimeout <= 0:
|
||||
return errConnectionTimeout
|
||||
case config.MigrationTimeout <= 0:
|
||||
return errMigrationTimeout
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
|
||||
)
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
|
||||
)
|
||||
|
||||
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
@@ -202,6 +202,10 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if r.Method == http.MethodGet {
|
||||
return s.listMessages(w, r, user, chatID)
|
||||
}
|
||||
key, valid := idempotencyKey(r)
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
var input struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
@@ -211,13 +215,21 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if !validText(input.Text, 4000) {
|
||||
return invalidRequest()
|
||||
}
|
||||
message, err := s.messengerStore.CreateMessage(r.Context(), chatID, user.ID, input.Text)
|
||||
message, replay, err := s.messengerStore.CreateMessage(
|
||||
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
|
||||
)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
return notFound()
|
||||
}
|
||||
if errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
|
||||
}
|
||||
if err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
if replay {
|
||||
s.metrics.replays.Inc()
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, message)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
)
|
||||
|
||||
type MessengerStore interface {
|
||||
@@ -15,7 +15,7 @@ type MessengerStore interface {
|
||||
CreateChat(context.Context, string, []string, string) (model.Chat, error)
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
|
||||
CreateMessage(context.Context, string, string, string) (model.Message, error)
|
||||
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
|
||||
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
@@ -40,11 +41,18 @@ func newMetrics() *metrics {
|
||||
Name: "requests_in_flight",
|
||||
Help: "HTTP requests currently being served.",
|
||||
}),
|
||||
replays: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "idempotency_replays_total",
|
||||
Help: "Message requests served from an idempotency record.",
|
||||
}),
|
||||
}
|
||||
registry.MustRegister(
|
||||
result.requests,
|
||||
result.duration,
|
||||
result.inFlight,
|
||||
result.replays,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -17,6 +18,8 @@ import (
|
||||
|
||||
const maxRequestBytes = 1 << 20
|
||||
|
||||
var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError {
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
|
||||
@@ -54,6 +57,14 @@ func validText(value string, maxRunes int) bool {
|
||||
return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes
|
||||
}
|
||||
|
||||
func idempotencyKey(r *http.Request) (string, bool) {
|
||||
values := r.Header.Values("Idempotency-Key")
|
||||
if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) {
|
||||
return "", false
|
||||
}
|
||||
return values[0], true
|
||||
}
|
||||
|
||||
type pageCursor struct {
|
||||
Kind string `json:"kind"`
|
||||
Resource string `json:"resource"`
|
||||
|
||||
@@ -13,7 +13,9 @@ type Server struct {
|
||||
sessions SessionStore
|
||||
passwords PasswordService
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
requestTTL time.Duration
|
||||
instanceID string
|
||||
logger *slog.Logger
|
||||
swaggerPort string
|
||||
metrics *metrics
|
||||
@@ -30,10 +32,21 @@ func (w *statusWriter) WriteHeader(status int) {
|
||||
w.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func NewServer(logger *slog.Logger, messages MessengerStore, sessions SessionStore, passwords PasswordService, sessionTTL, requestTTL time.Duration, swaggerPort string) *Server {
|
||||
func NewServer(
|
||||
logger *slog.Logger,
|
||||
messages MessengerStore,
|
||||
sessions SessionStore,
|
||||
passwords PasswordService,
|
||||
sessionTTL time.Duration,
|
||||
idempotencyTTL time.Duration,
|
||||
requestTTL time.Duration,
|
||||
instanceID string,
|
||||
swaggerPort string,
|
||||
) *Server {
|
||||
s := &Server{
|
||||
messengerStore: messages, sessions: sessions, passwords: passwords,
|
||||
sessionTTL: sessionTTL, requestTTL: requestTTL, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
|
||||
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
|
||||
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
|
||||
}
|
||||
s.ready.Store(true)
|
||||
return s
|
||||
@@ -48,6 +61,7 @@ func (s *Server) Handler() http.Handler {
|
||||
}
|
||||
|
||||
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", s.instanceID)
|
||||
requestID, err := newID()
|
||||
if err != nil {
|
||||
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
|
||||
@@ -70,8 +84,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
|
||||
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
|
||||
}()
|
||||
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
|
||||
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
|
||||
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
|
||||
wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key")
|
||||
wrapped.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
@@ -92,7 +106,7 @@ func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool
|
||||
return false
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id")
|
||||
w.Header().Add("Vary", "Origin")
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package messenger
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -11,17 +12,36 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
|
||||
)
|
||||
|
||||
var errDependency = errors.New("dependency unavailable")
|
||||
|
||||
type MessengerSuite struct {
|
||||
suite.Suite
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func TestMessengerSuite(t *testing.T) {
|
||||
suite.Run(t, new(MessengerSuite))
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) SetupTest() {
|
||||
s.logger = slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
type messengerStoreStub struct {
|
||||
pingErr error
|
||||
userByID model.User
|
||||
userByIDErr error
|
||||
credentials model.UserCredentials
|
||||
credentialsErr error
|
||||
message model.Message
|
||||
messageReplay bool
|
||||
messageErr error
|
||||
}
|
||||
|
||||
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
|
||||
@@ -43,8 +63,8 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
|
||||
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
func (messengerStoreStub) CreateMessage(context.Context, string, string, string) (model.Message, error) {
|
||||
return model.Message{}, nil
|
||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
|
||||
return s.message, s.messageReplay, s.messageErr
|
||||
}
|
||||
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
|
||||
return nil, false, nil
|
||||
@@ -74,7 +94,8 @@ func (s passwordsStub) Verify(string, string) (bool, error) {
|
||||
return s.verified, s.err
|
||||
}
|
||||
|
||||
func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
|
||||
t := s.T()
|
||||
tests := []struct {
|
||||
name string
|
||||
store messengerStoreStub
|
||||
@@ -86,7 +107,7 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server := newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
response := serveRequest(server, http.MethodGet, "/health/ready", "")
|
||||
if response.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
|
||||
@@ -95,7 +116,8 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() {
|
||||
t := s.T()
|
||||
tests := []struct {
|
||||
name string
|
||||
store messengerStoreStub
|
||||
@@ -107,7 +129,7 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server := newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
response := httptest.NewRecorder()
|
||||
@@ -119,15 +141,16 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthenticationChallenges(t *testing.T) {
|
||||
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
func (s *MessengerSuite) TestAuthenticationChallenges() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "")
|
||||
if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` {
|
||||
t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate"))
|
||||
}
|
||||
|
||||
store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}}
|
||||
server = newTestServer(store, sessionStub{}, passwordsStub{verified: false})
|
||||
server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false})
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil)
|
||||
request.SetBasicAuth("user", "wrong-password")
|
||||
response := httptest.NewRecorder()
|
||||
@@ -137,8 +160,9 @@ func TestAuthenticationChallenges(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionCreationRejectsChunkedBody(t *testing.T) {
|
||||
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}"))
|
||||
request.ContentLength = -1
|
||||
request.SetBasicAuth("user", "password")
|
||||
@@ -149,17 +173,77 @@ func TestSessionCreationRejectsChunkedBody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationRejectsWhitespacePassword(t *testing.T) {
|
||||
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`)
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
return NewServer(logger, messages, sessions, passwords, time.Hour, time.Second, "8081")
|
||||
func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
for _, response := range []*httptest.ResponseRecorder{
|
||||
serveRequest(server, http.MethodGet, "/health/live", ""),
|
||||
serveRequest(server, http.MethodGet, "/api/v1/users/me", ""),
|
||||
} {
|
||||
if value := response.Header().Get("X-Instance-Id"); value != "test-instance" {
|
||||
t.Fatalf("X-Instance-Id = %q, want test-instance", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() {
|
||||
t := s.T()
|
||||
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
|
||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
}{
|
||||
{name: "missing"},
|
||||
{name: "invalid", key: "invalid key"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if tt.key != "" {
|
||||
request.Header.Set("Idempotency-Key", tt.key)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() {
|
||||
t := s.T()
|
||||
store := messengerStoreStub{
|
||||
userByID: model.User{ID: "user-id"},
|
||||
messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict),
|
||||
}
|
||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", "same-key")
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
if response.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
|
||||
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
|
||||
}
|
||||
|
||||
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
|
||||
|
||||
@@ -4,15 +4,21 @@ import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
func TestUserTimestampHasFixedMicroseconds(t *testing.T) {
|
||||
type ModelSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestModelSuite(t *testing.T) {
|
||||
suite.Run(t, new(ModelSuite))
|
||||
}
|
||||
|
||||
func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() {
|
||||
createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC)
|
||||
data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if string(data) != `{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}` {
|
||||
t.Fatalf("Marshal() = %s", data)
|
||||
}
|
||||
s.Require().NoError(err)
|
||||
s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data))
|
||||
}
|
||||
|
||||
@@ -1,42 +1,42 @@
|
||||
package password
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
func TestHasher(t *testing.T) {
|
||||
hasher, err := New(testConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
hash, err := hasher.Hash("T3st-password-123")
|
||||
if err != nil {
|
||||
t.Fatalf("Hash() error = %v", err)
|
||||
}
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
verified, err := hasher.Verify("T3st-password-123", hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Verify() error = %v", err)
|
||||
}
|
||||
if !verified {
|
||||
t.Fatal("Verify() = false for the original password")
|
||||
}
|
||||
|
||||
verified, err = hasher.Verify("wrong-password", hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Verify() error = %v", err)
|
||||
}
|
||||
if verified {
|
||||
t.Fatal("Verify() = true for a different password")
|
||||
}
|
||||
type PasswordSuite struct {
|
||||
suite.Suite
|
||||
hasher *Hasher
|
||||
}
|
||||
|
||||
func TestVerifyRejectsMalformedHash(t *testing.T) {
|
||||
func TestPasswordSuite(t *testing.T) {
|
||||
suite.Run(t, new(PasswordSuite))
|
||||
}
|
||||
|
||||
func (s *PasswordSuite) SetupTest() {
|
||||
hasher, err := New(testConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
if _, err := hasher.Verify("password", "invalid"); err == nil {
|
||||
t.Fatal("Verify() error = nil for malformed hash")
|
||||
}
|
||||
s.Require().NoError(err)
|
||||
s.hasher = hasher
|
||||
}
|
||||
|
||||
func (s *PasswordSuite) TestHasher() {
|
||||
hash, err := s.hasher.Hash("T3st-password-123")
|
||||
s.Require().NoError(err)
|
||||
|
||||
verified, err := s.hasher.Verify("T3st-password-123", hash)
|
||||
s.Require().NoError(err)
|
||||
s.True(verified)
|
||||
|
||||
verified, err = s.hasher.Verify("wrong-password", hash)
|
||||
s.Require().NoError(err)
|
||||
s.False(verified)
|
||||
}
|
||||
|
||||
func (s *PasswordSuite) TestVerifyRejectsMalformedHash() {
|
||||
_, err := s.hasher.Verify("password", "invalid")
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
func testConfig() Config {
|
||||
|
||||
@@ -3,8 +3,9 @@ package postgres
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrInvalidMembers = errors.New("invalid members")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrInvalidMembers = errors.New("invalid members")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrIdempotencyConflict = errors.New("idempotency conflict")
|
||||
)
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
)
|
||||
|
||||
const expiredIdempotencyCleanupBatch = 100
|
||||
|
||||
func (s *Store) CreateMessage(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
idempotencyKey string,
|
||||
idempotencyTTL time.Duration,
|
||||
) (model.Message, bool, error) {
|
||||
var (
|
||||
tx pgx.Tx
|
||||
err error
|
||||
visible bool
|
||||
requestHash [sha256.Size]byte
|
||||
message model.Message
|
||||
replay bool
|
||||
)
|
||||
|
||||
tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !visible {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
|
||||
requestHash = sha256.Sum256([]byte(text))
|
||||
message, replay, err = createIdempotentMessage(
|
||||
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||
)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err)
|
||||
}
|
||||
return message, replay, nil
|
||||
}
|
||||
|
||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||
const query = `
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM chat_members
|
||||
WHERE chat_id = $1 AND user_id = $2
|
||||
)`
|
||||
|
||||
var visible bool
|
||||
if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil {
|
||||
return false, fmt.Errorf("check chat visibility: %w", err)
|
||||
}
|
||||
return visible, nil
|
||||
}
|
||||
|
||||
func createIdempotentMessage(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
idempotencyKey string,
|
||||
requestHash []byte,
|
||||
idempotencyTTL time.Duration,
|
||||
) (model.Message, bool, error) {
|
||||
for {
|
||||
var (
|
||||
record idempotencyRecord
|
||||
found bool
|
||||
err error
|
||||
messageID string
|
||||
reserved bool
|
||||
message model.Message
|
||||
)
|
||||
|
||||
record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if found && record.active {
|
||||
if !bytes.Equal(record.requestHash, requestHash) {
|
||||
return model.Message{}, false, ErrIdempotencyConflict
|
||||
}
|
||||
message, err = messageByID(ctx, tx, record.messageID)
|
||||
return message, true, err
|
||||
}
|
||||
if found {
|
||||
if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
}
|
||||
|
||||
messageID = uuid.NewString()
|
||||
reserved, err = reserveIdempotencyKey(
|
||||
ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL,
|
||||
)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !reserved {
|
||||
continue
|
||||
}
|
||||
|
||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
|
||||
return message, false, err
|
||||
}
|
||||
}
|
||||
|
||||
func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error {
|
||||
const query = `
|
||||
WITH expired AS (
|
||||
SELECT ctid
|
||||
FROM message_idempotency
|
||||
WHERE expires_at <= statement_timestamp()
|
||||
ORDER BY expires_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM message_idempotency records
|
||||
USING expired
|
||||
WHERE records.ctid = expired.ctid`
|
||||
|
||||
if _, err := tx.Exec(ctx, query, limit); err != nil {
|
||||
return fmt.Errorf("delete expired idempotency records: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type idempotencyRecord struct {
|
||||
requestHash []byte
|
||||
messageID string
|
||||
active bool
|
||||
}
|
||||
|
||||
func lockIdempotencyRecord(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
userID string,
|
||||
chatID string,
|
||||
key string,
|
||||
) (idempotencyRecord, bool, error) {
|
||||
const query = `
|
||||
SELECT request_hash, message_id, expires_at > statement_timestamp()
|
||||
FROM message_idempotency
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3
|
||||
FOR UPDATE`
|
||||
|
||||
var (
|
||||
record idempotencyRecord
|
||||
err error
|
||||
)
|
||||
err = tx.QueryRow(ctx, query, userID, chatID, key).Scan(
|
||||
&record.requestHash,
|
||||
&record.messageID,
|
||||
&record.active,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return idempotencyRecord{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err)
|
||||
}
|
||||
return record, true, nil
|
||||
}
|
||||
|
||||
func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error {
|
||||
const query = `
|
||||
DELETE FROM message_idempotency
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`
|
||||
if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil {
|
||||
return fmt.Errorf("delete expired idempotency key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reserveIdempotencyKey(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
userID string,
|
||||
chatID string,
|
||||
key string,
|
||||
requestHash []byte,
|
||||
messageID string,
|
||||
ttl time.Duration,
|
||||
) (bool, error) {
|
||||
const query = `
|
||||
INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second')
|
||||
ON CONFLICT (user_id, chat_id, key) DO NOTHING
|
||||
RETURNING true`
|
||||
|
||||
var (
|
||||
inserted bool
|
||||
err error
|
||||
)
|
||||
err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reserve idempotency key: %w", err)
|
||||
}
|
||||
return inserted, nil
|
||||
}
|
||||
|
||||
func insertMessage(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
messageID string,
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
) (model.Message, error) {
|
||||
const query = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, chat_id, sender_id, text, created_at`
|
||||
|
||||
var (
|
||||
message model.Message
|
||||
err error
|
||||
)
|
||||
err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan(
|
||||
&message.ID,
|
||||
&message.ChatID,
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return model.Message{}, fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
return message, nil
|
||||
}
|
||||
|
||||
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
|
||||
const query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE id = $1`
|
||||
|
||||
var (
|
||||
message model.Message
|
||||
err error
|
||||
)
|
||||
err = tx.QueryRow(ctx, query, messageID).Scan(
|
||||
&message.ID,
|
||||
&message.ChatID,
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Message{}, fmt.Errorf("idempotency result message not found")
|
||||
}
|
||||
if err != nil {
|
||||
return model.Message{}, fmt.Errorf("get idempotency result message: %w", err)
|
||||
}
|
||||
return message, nil
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
@@ -114,7 +114,7 @@ func (s *Store) CreateChat(
|
||||
if err != nil {
|
||||
return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
valid, err := allUsersExist(ctx, tx, memberIDs)
|
||||
if err != nil {
|
||||
@@ -319,40 +319,6 @@ func (s *Store) visibleChatPositionExists(
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateMessage(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
) (model.Message, error) {
|
||||
const query = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
SELECT $1, $2, $3, $4
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM chat_members
|
||||
WHERE chat_id = $2 AND user_id = $3
|
||||
)
|
||||
RETURNING id, chat_id, sender_id, text, created_at`
|
||||
|
||||
var message model.Message
|
||||
err := s.pool.QueryRow(ctx, query, uuid.NewString(), chatID, senderID, text).Scan(
|
||||
&message.ID,
|
||||
&message.ChatID,
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Message{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Message{}, fmt.Errorf("create message: %w", err)
|
||||
}
|
||||
|
||||
return message, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListMessages(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
|
||||
@@ -11,14 +11,37 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
|
||||
)
|
||||
|
||||
func TestConcurrentUsernameCreation(t *testing.T) {
|
||||
pool := openTestDatabase(t)
|
||||
store := postgres.New(pool)
|
||||
type PostgresSuite struct {
|
||||
suite.Suite
|
||||
pool *pgxpool.Pool
|
||||
store *postgres.Store
|
||||
}
|
||||
|
||||
func TestPostgresSuite(t *testing.T) {
|
||||
suite.Run(t, new(PostgresSuite))
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) SetupSuite() {
|
||||
s.pool = openTestDatabase(s.T())
|
||||
s.store = postgres.New(s.pool)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TearDownSuite() {
|
||||
if s.pool != nil {
|
||||
s.pool.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestConcurrentUsernameCreation() {
|
||||
t := s.T()
|
||||
store := s.store
|
||||
username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16]
|
||||
|
||||
const workers = 8
|
||||
@@ -51,9 +74,10 @@ func TestConcurrentUsernameCreation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateChatRollsBackForUnknownMember(t *testing.T) {
|
||||
pool := openTestDatabase(t)
|
||||
store := postgres.New(pool)
|
||||
func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
store := s.store
|
||||
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateUser() error = %v", err)
|
||||
@@ -69,6 +93,175 @@ func TestCreateChatRollsBackForUnknownMember(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
store := s.store
|
||||
owner, chatID := createOwnerAndChat(t, store)
|
||||
key := uuid.NewString()
|
||||
|
||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
if err != nil || replay {
|
||||
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
|
||||
}
|
||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
if err != nil || !replay {
|
||||
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("replay message = %v, want %v", second, first)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
|
||||
}
|
||||
if count := messageCount(t, pool, chatID); count != 1 {
|
||||
t.Fatalf("message count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)}
|
||||
owner, chatID := createOwnerAndChat(t, stores[0])
|
||||
key := uuid.NewString()
|
||||
|
||||
const workers = 8
|
||||
type result struct {
|
||||
message model.Message
|
||||
replay bool
|
||||
err error
|
||||
}
|
||||
results := make(chan result, workers)
|
||||
var group sync.WaitGroup
|
||||
for worker := range workers {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
message, replay, err := stores[worker%len(stores)].CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
|
||||
)
|
||||
results <- result{message: message, replay: replay, err: err}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(results)
|
||||
|
||||
var messageID string
|
||||
var created int
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
t.Fatalf("CreateMessage() error = %v", result.err)
|
||||
}
|
||||
if messageID == "" {
|
||||
messageID = result.message.ID
|
||||
}
|
||||
if result.message.ID != messageID {
|
||||
t.Fatalf("message ID = %s, want %s", result.message.ID, messageID)
|
||||
}
|
||||
if !result.replay {
|
||||
created++
|
||||
}
|
||||
}
|
||||
if created != 1 {
|
||||
t.Fatalf("new message results = %d, want 1", created)
|
||||
}
|
||||
if count := messageCount(t, pool, chatID); count != 1 {
|
||||
t.Fatalf("message count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
store := s.store
|
||||
owner, chatID := createOwnerAndChat(t, store)
|
||||
second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateUser() error = %v", err)
|
||||
}
|
||||
chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateChat() error = %v", err)
|
||||
}
|
||||
key := uuid.NewString()
|
||||
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("owner CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("second user CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("other chat CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(context.Background(), `
|
||||
UPDATE message_idempotency
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 days',
|
||||
expires_at = clock_timestamp() - INTERVAL '1 day'
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
|
||||
if err != nil || replay || message.Text != "after expiry" {
|
||||
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
||||
t := s.T()
|
||||
owner, chatID := createOwnerAndChat(t, s.store)
|
||||
expiredKey := uuid.NewString()
|
||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
|
||||
t.Fatalf("create expired message: %v", err)
|
||||
}
|
||||
if _, err := s.pool.Exec(context.Background(), `
|
||||
UPDATE message_idempotency
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 days',
|
||||
expires_at = clock_timestamp() - INTERVAL '1 day'
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
if _, _, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
|
||||
); err != nil {
|
||||
t.Fatalf("create new message: %v", err)
|
||||
}
|
||||
|
||||
var remaining int
|
||||
if err := s.pool.QueryRow(context.Background(), `
|
||||
SELECT count(*)
|
||||
FROM message_idempotency
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil {
|
||||
t.Fatalf("count expired idempotency records: %v", err)
|
||||
}
|
||||
if remaining != 0 {
|
||||
t.Fatalf("expired idempotency records = %d, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) {
|
||||
t.Helper()
|
||||
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateUser() error = %v", err)
|
||||
}
|
||||
chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateChat() error = %v", err)
|
||||
}
|
||||
return owner, chat.ID
|
||||
}
|
||||
|
||||
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil {
|
||||
t.Fatalf("count messages: %v", err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func openTestDatabase(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE")
|
||||
@@ -76,12 +269,14 @@ func openTestDatabase(t *testing.T) *pgxpool.Pool {
|
||||
URL: url, MinConnections: 0, MaxConnections: 10,
|
||||
MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute,
|
||||
HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second,
|
||||
MigrationTimeout: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open() error = %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil {
|
||||
pool.Close()
|
||||
t.Fatalf("database.Migrate() error = %v", err)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
|
||||
@@ -9,75 +9,64 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
|
||||
)
|
||||
|
||||
func TestSessionLifecycle(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
userID := uuid.NewString()
|
||||
token, expiresAt, err := store.Create(context.Background(), userID, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
if token == "" || !expiresAt.After(time.Now()) {
|
||||
t.Fatalf("invalid token or expiry: token length = %d, expires_at = %v", len(token), expiresAt)
|
||||
}
|
||||
|
||||
actualUserID, err := store.UserID(context.Background(), token)
|
||||
if err != nil {
|
||||
t.Fatalf("UserID() error = %v", err)
|
||||
}
|
||||
if actualUserID != userID {
|
||||
t.Fatalf("UserID() = %q, want %q", actualUserID, userID)
|
||||
}
|
||||
if err := store.Delete(context.Background(), token); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
if _, err := store.UserID(context.Background(), token); !errors.Is(err, session.ErrNotFound) {
|
||||
t.Fatalf("UserID() after delete error = %v, want ErrNotFound", err)
|
||||
}
|
||||
type SessionSuite struct {
|
||||
suite.Suite
|
||||
store *session.Store
|
||||
}
|
||||
|
||||
func TestSessionExpires(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
token, _, err := store.Create(context.Background(), uuid.NewString(), time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
_, err := store.UserID(context.Background(), token)
|
||||
if errors.Is(err, session.ErrNotFound) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("UserID() error = %v", err)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("session did not expire within three seconds")
|
||||
func TestSessionSuite(t *testing.T) {
|
||||
suite.Run(t, new(SessionSuite))
|
||||
}
|
||||
|
||||
func openTestStore(t *testing.T) *session.Store {
|
||||
t.Helper()
|
||||
url := strings.TrimSpace(os.Getenv("TEST_CACHE_URL"))
|
||||
if url == "" {
|
||||
func (s *SessionSuite) SetupSuite() {
|
||||
rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL"))
|
||||
if rawURL == "" {
|
||||
path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE"))
|
||||
if path == "" {
|
||||
t.Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests")
|
||||
s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read TEST_CACHE_URL_FILE: %v", err)
|
||||
}
|
||||
url = strings.TrimSpace(string(data))
|
||||
s.Require().NoError(err)
|
||||
rawURL = strings.TrimSpace(string(data))
|
||||
}
|
||||
store, err := session.New(url)
|
||||
if err != nil {
|
||||
t.Fatalf("session.New() error = %v", err)
|
||||
store, err := session.New(rawURL)
|
||||
s.Require().NoError(err)
|
||||
s.store = store
|
||||
}
|
||||
|
||||
func (s *SessionSuite) TearDownSuite() {
|
||||
if s.store != nil {
|
||||
s.Require().NoError(s.store.Close())
|
||||
}
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
return store
|
||||
}
|
||||
|
||||
func (s *SessionSuite) TestSessionLifecycle() {
|
||||
userID := uuid.NewString()
|
||||
token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute)
|
||||
s.Require().NoError(err)
|
||||
s.NotEmpty(token)
|
||||
s.True(expiresAt.After(time.Now()))
|
||||
|
||||
actualUserID, err := s.store.UserID(context.Background(), token)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(userID, actualUserID)
|
||||
s.Require().NoError(s.store.Delete(context.Background(), token))
|
||||
|
||||
_, err = s.store.UserID(context.Background(), token)
|
||||
s.ErrorIs(err, session.ErrNotFound)
|
||||
}
|
||||
|
||||
func (s *SessionSuite) TestSessionExpires() {
|
||||
token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second)
|
||||
s.Require().NoError(err)
|
||||
|
||||
s.Eventually(func() bool {
|
||||
_, err = s.store.UserID(context.Background(), token)
|
||||
return errors.Is(err, session.ErrNotFound)
|
||||
}, 3*time.Second, 50*time.Millisecond)
|
||||
}
|
||||
|
||||
+23
-5
@@ -8,10 +8,14 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081
|
||||
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
|
||||
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
|
||||
|
||||
# Время жизни сессии и используемое хранилище.
|
||||
# Время жизни сессии, идемпотентности и используемое хранилище.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
SESSION_STORE=valkey
|
||||
|
||||
# Число API-реплик за Nginx.
|
||||
API_REPLICAS=2
|
||||
|
||||
# Учётные данные локального PostgreSQL и строка подключения API.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_DB=messenger
|
||||
@@ -37,7 +41,7 @@ HTTP_READ_HEADER_TIMEOUT_SECONDS=5
|
||||
HTTP_READ_TIMEOUT_SECONDS=10
|
||||
HTTP_WRITE_TIMEOUT_SECONDS=10
|
||||
HTTP_IDLE_TIMEOUT_SECONDS=60
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS=10
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
|
||||
# Параметры Argon2id для новых хешей паролей.
|
||||
PASSWORD_MEMORY_KIB=65536
|
||||
@@ -49,9 +53,23 @@ PASSWORD_KEY_LENGTH=32
|
||||
# Порт Swagger внутри конфигурации CORS API.
|
||||
SWAGGER_PORT=8081
|
||||
|
||||
# Ограничения ресурсов контейнеров приложения и хранилищ.
|
||||
API_CPUS=2
|
||||
API_MEMORY=512m
|
||||
# Ограничения ресурсов контейнеров.
|
||||
API_CPUS=1
|
||||
API_MEMORY=256m
|
||||
MIGRATE_CPUS=0.5
|
||||
MIGRATE_MEMORY=128m
|
||||
NGINX_CPUS=0.5
|
||||
NGINX_MEMORY=128m
|
||||
SWAGGER_CPUS=0.25
|
||||
SWAGGER_MEMORY=128m
|
||||
TEST_CPUS=2
|
||||
TEST_MEMORY=512m
|
||||
EXPORTER_CPUS=0.25
|
||||
EXPORTER_MEMORY=128m
|
||||
PROMETHEUS_CPUS=1
|
||||
PROMETHEUS_MEMORY=512m
|
||||
GRAFANA_CPUS=0.5
|
||||
GRAFANA_MEMORY=256m
|
||||
POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
request_hash BYTEA NOT NULL,
|
||||
message_id UUID NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key),
|
||||
CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id)
|
||||
ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
|
||||
CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at);
|
||||
@@ -6,58 +6,81 @@
|
||||
{
|
||||
"type": "timeseries", "title": "API requests per second", "id": 1,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "RPS", "refId": "A"}]
|
||||
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "API responses by status class", "id": 2,
|
||||
"type": "timeseries", "title": "API RPS by replica", "id": 2,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "API responses by status class", "id": 3,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "API latency p95", "id": 3,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"type": "timeseries", "title": "Healthy API replicas", "id": 4,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
||||
"targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "API latency p95 by replica and route", "id": 5,
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 16},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{route}}", "refId": "A"}]
|
||||
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "API in flight", "id": 4,
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 8},
|
||||
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "requests", "refId": "A"}]
|
||||
"type": "timeseries", "title": "API in flight", "id": 6,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
|
||||
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Go heap", "id": 5,
|
||||
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 8},
|
||||
"type": "timeseries", "title": "Go heap by replica", "id": 7,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "heap", "refId": "A"}]
|
||||
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "PostgreSQL connections", "id": 6,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16},
|
||||
"type": "timeseries", "title": "Idempotency replays", "id": 8,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
|
||||
"targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Nginx connections and requests", "id": 9,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
|
||||
"targets": [
|
||||
{"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"},
|
||||
{"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "PostgreSQL connections", "id": 10,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
|
||||
"targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "PostgreSQL transactions", "id": 7,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16},
|
||||
"type": "timeseries", "title": "PostgreSQL transactions", "id": 11,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 40},
|
||||
"targets": [
|
||||
{"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"},
|
||||
{"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "PostgreSQL database size", "id": 8,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16},
|
||||
"type": "timeseries", "title": "PostgreSQL database size", "id": 12,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 40},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Valkey memory", "id": 9,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 24},
|
||||
"type": "timeseries", "title": "Valkey memory", "id": 13,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 48},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Valkey keys and commands", "id": 10,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 24},
|
||||
"type": "timeseries", "title": "Valkey keys and commands", "id": 14,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 48},
|
||||
"targets": [
|
||||
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
|
||||
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
|
||||
@@ -66,11 +89,11 @@
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 41,
|
||||
"tags": ["highload", "lab2"],
|
||||
"tags": ["highload", "lab3", "nginx", "failover"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-15m", "to": "now"},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger overview",
|
||||
"title": "Messenger lab3 overview",
|
||||
"uid": "messenger-overview",
|
||||
"version": 1
|
||||
"version": 2
|
||||
}
|
||||
|
||||
@@ -4,8 +4,15 @@ global:
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
dns_sd_configs:
|
||||
- names: ["api"]
|
||||
type: A
|
||||
port: 8080
|
||||
refresh_interval: 2s
|
||||
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
- targets: ["api:8080"]
|
||||
- targets: ["nginx-exporter:9113"]
|
||||
|
||||
- job_name: postgres
|
||||
static_configs:
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
error_log /dev/stderr notice;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
|
||||
log_format structured escape=json
|
||||
'{"time":"$time_iso8601","request_id":"$http_x_request_id",'
|
||||
'"method":"$request_method","uri":"$request_uri","status":$status,'
|
||||
'"request_time":$request_time,"upstream":"$upstream_addr",'
|
||||
'"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}';
|
||||
|
||||
access_log /dev/stdout structured;
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 3s;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8081;
|
||||
access_log off;
|
||||
|
||||
location = /stub_status {
|
||||
stub_status;
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -2,7 +2,7 @@
|
||||
|
||||
set -eu
|
||||
|
||||
docker compose restart postgres valkey
|
||||
docker compose up -d --wait --wait-timeout 90 postgres valkey
|
||||
docker compose restart api
|
||||
docker compose up -d --wait --wait-timeout 90 api
|
||||
docker compose stop nginx api valkey postgres
|
||||
docker compose start postgres valkey
|
||||
docker compose run --rm migrate
|
||||
docker compose up -d --wait --wait-timeout 90
|
||||
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
wait_for_replicas() {
|
||||
expected=$1
|
||||
deadline=$(($(date +%s) + 90))
|
||||
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
containers=$(docker compose ps -q api)
|
||||
count=0
|
||||
healthy=0
|
||||
for container in $containers; do
|
||||
count=$((count + 1))
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
old_containers=$(docker compose ps -q api)
|
||||
old_count=0
|
||||
for container in $old_containers; do
|
||||
old_count=$((old_count + 1))
|
||||
done
|
||||
|
||||
docker compose build
|
||||
docker compose up -d postgres valkey
|
||||
docker compose run --rm migrate
|
||||
|
||||
if [ "$old_count" -lt 2 ]; then
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 120
|
||||
exit 0
|
||||
fi
|
||||
|
||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
||||
wait_for_replicas 3
|
||||
|
||||
for container in $old_containers; do
|
||||
docker stop --time 6 "$container" >/dev/null
|
||||
docker rm "$container" >/dev/null
|
||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
||||
wait_for_replicas 3
|
||||
done
|
||||
|
||||
docker compose up -d --no-deps --no-recreate --scale api=2 api
|
||||
wait_for_replicas 2
|
||||
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
|
||||
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
duration=${DURATION:-15s}
|
||||
warmup=${WARMUP:-3s}
|
||||
concurrencies=${CONCURRENCIES:-"8 32 64"}
|
||||
repeats=${REPEATS:-3}
|
||||
current_project=
|
||||
|
||||
restore_default_stack() {
|
||||
if [ -n "$current_project" ]; then
|
||||
env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true
|
||||
fi
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 120
|
||||
}
|
||||
|
||||
run_configuration() {
|
||||
series=$1
|
||||
replicas=$2
|
||||
api_cpus=$3
|
||||
api_memory=$4
|
||||
db_max_conns=$5
|
||||
repeat=$6
|
||||
point_concurrencies=$7
|
||||
project="lab3exp_${series}_r${replicas}_n${repeat}"
|
||||
|
||||
pending=false
|
||||
for concurrency in $point_concurrencies; do
|
||||
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
|
||||
if [ ! -s "evidence/lab3-${label}.json" ]; then
|
||||
pending=true
|
||||
fi
|
||||
done
|
||||
if [ "$pending" = false ]; then
|
||||
echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}"
|
||||
return
|
||||
fi
|
||||
|
||||
current_project=$project
|
||||
|
||||
compose() {
|
||||
env \
|
||||
COMPOSE_PROJECT_NAME="$project" \
|
||||
API_REPLICAS="$replicas" \
|
||||
API_CPUS="$api_cpus" \
|
||||
API_MEMORY="$api_memory" \
|
||||
DB_MIN_CONNS=0 \
|
||||
DB_MAX_CONNS="$db_max_conns" \
|
||||
docker compose "$@"
|
||||
}
|
||||
|
||||
compose up -d --scale api="$replicas" --wait --wait-timeout 120 \
|
||||
postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus
|
||||
for concurrency in $point_concurrencies; do
|
||||
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
|
||||
output="lab3-${label}.json"
|
||||
if [ -s "evidence/$output" ]; then
|
||||
echo "Skipping completed point $label"
|
||||
continue
|
||||
fi
|
||||
compose --profile research run --rm experiment \
|
||||
/app/lab3experiment \
|
||||
-base-url=http://nginx:8080 \
|
||||
-prometheus-url=http://prometheus:9090 \
|
||||
-label="$label" \
|
||||
-duration="$duration" \
|
||||
-warmup="$warmup" \
|
||||
-concurrency="$concurrency" \
|
||||
-replicas="$replicas" \
|
||||
-api-cpus="$api_cpus" \
|
||||
-api-memory="$api_memory" \
|
||||
-db-max-conns="$db_max_conns" \
|
||||
-output="/output/$output"
|
||||
done
|
||||
compose --profile research down -v --remove-orphans
|
||||
current_project=
|
||||
}
|
||||
|
||||
trap restore_default_stack EXIT INT TERM
|
||||
docker compose build api nginx swagger prometheus grafana
|
||||
docker compose stop
|
||||
|
||||
repeat=1
|
||||
while [ "$repeat" -le "$repeats" ]; do
|
||||
run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies"
|
||||
run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies"
|
||||
run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies"
|
||||
|
||||
run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies"
|
||||
run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies"
|
||||
run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies"
|
||||
|
||||
run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64"
|
||||
run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64"
|
||||
repeat=$((repeat + 1))
|
||||
done
|
||||
|
||||
docker compose --profile research run --no-deps --rm experiment \
|
||||
/app/lab3summary \
|
||||
-input=/output \
|
||||
-output=/output/lab3-scale-results.json \
|
||||
-chart=/output/lab3-scale-rps.svg
|
||||
rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
|
||||
for container in $(docker compose ps -q api); do
|
||||
instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 6 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user