Lab 3: Nginx balancing and API failover #1

Merged
dizi merged 5 commits from solution/bvt2303-plastinin into main 2026-10-02 17:42:41 +03:00
40 changed files with 4924 additions and 306 deletions
+87
View File
@@ -0,0 +1,87 @@
name: CI
on:
pull_request:
branches: [main]
push:
branches: [main]
workflow_dispatch:
concurrency:
group: ci-${{ github.repository }}-${{ github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
COMPOSE_PROJECT_NAME: mtusi2026lab3-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
git init .
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
git fetch --depth=1 origin ${{ github.sha }}
git checkout --detach FETCH_HEAD
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Check contract
run: make check
- name: Validate OpenAPI
run: |
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
- name: Check formatting
run: test -z "$(gofmt -l cmd internal migrations)"
- name: Lint
uses: golangci/golangci-lint-action@v7
with:
version: v2.13.2
args: --timeout=5m
skip-cache: true
- name: Unit tests and build
run: |
go test -race ./...
go build ./cmd/...
- name: Prepare local stack
run: |
cp lab3.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 120
network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)")
docker network connect "$network" "$HOSTNAME"
printf '%s\n' "$network" > .ci-network
- name: Integration checks
run: |
make test BASE_URL=http://nginx:8080
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
docker compose up -d --scale api=2 --wait --wait-timeout 90
- name: Container logs
if: failure()
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
- name: Stop local stack
if: always()
run: |
if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi
if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi
+117
View File
@@ -0,0 +1,117 @@
name: Deploy production
on:
workflow_dispatch:
concurrency:
group: production
cancel-in-progress: false
jobs:
verify:
name: Verify selected commit
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 20
env:
COMPOSE_PROJECT_NAME: mtusi2026lab3-deploy-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
git init .
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
git fetch --depth=1 origin ${{ github.sha }}
git checkout --detach FETCH_HEAD
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Check contract
run: make check
- name: Validate OpenAPI
run: |
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
- name: Check formatting
run: test -z "$(gofmt -l cmd internal migrations)"
- name: Lint
uses: golangci/golangci-lint-action@v7
with:
version: v2.13.2
args: --timeout=5m
skip-cache: true
- name: Unit tests and build
run: |
go test -race ./...
go build ./cmd/...
- name: Prepare local stack
run: |
cp lab3.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 120
network=$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}' "$(docker compose ps -q nginx)")
docker network connect "$network" "$HOSTNAME"
printf '%s\n' "$network" > .ci-network
- name: Integration checks
run: |
make test BASE_URL=http://nginx:8080
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
- name: Container logs
if: failure()
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
- name: Stop local stack
if: always()
run: |
if [ -f .ci-network ]; then docker network disconnect "$(cat .ci-network)" "$HOSTNAME"; fi
if [ -f .env ]; then docker compose --profile tests --profile research down -v --remove-orphans; fi
deploy:
name: Deploy selected commit
needs: verify
runs-on: ubuntu-latest
environment: production
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
DEPLOY_SHA: ${{ github.sha }}
steps:
- name: Configure SSH
run: |
umask 077
mkdir -p ~/.ssh
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
- name: Deploy and verify
run: |
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
set -eu
cd ~/mtusi2026lab3
git fetch origin main
git cat-file -e '$DEPLOY_SHA^{commit}'
git reset --hard '$DEPLOY_SHA'
git clean -fd
scripts/rolling-update.sh
test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA'
docker compose ps
"
+10 -6
View File
@@ -2,16 +2,14 @@ FROM golang:1.26.5-alpine3.24 AS build
WORKDIR /src
COPY . .
RUN go build ./cmd/messenger
RUN go build -o /out/messenger ./cmd/messenger
RUN go build -o /out/lab3experiment ./cmd/lab3experiment
RUN go build -o /out/lab3summary ./cmd/lab3summary
FROM build AS test
CMD ["go", "test", "./internal/postgres", "./internal/session"]
FROM build AS experiment
CMD ["go", "run", "./cmd/lab2experiment"]
FROM swaggerapi/swagger-ui:v5.31.2 AS swagger
COPY contracts/openapi.json /spec/openapi.json
@@ -26,10 +24,16 @@ FROM grafana/grafana:13.1.6 AS grafana
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
FROM nginx:1.30.5-alpine AS loadbalancer
COPY nginx/nginx.conf /etc/nginx/nginx.conf
FROM alpine:3.24.2
RUN addgroup -S app && adduser -S -G app -u 10001 app
COPY --from=build --chown=app:app /src/messenger /app/messenger
COPY --from=build --chown=app:app /out/messenger /app/messenger
COPY --from=build --chown=app:app /out/lab3experiment /app/lab3experiment
COPY --from=build --chown=app:app /out/lab3summary /app/lab3summary
USER app
WORKDIR /app
+175 -15
View File
@@ -1,37 +1,197 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Пластинин Денис, БВТ2303
- Целевая оценка: 5
- Выбранный трек: 1 — цена масштабирования
- Ветка/PR: `solution/bvt2303-plastinin`, PR #1
- Среда эксперимента: Docker, `linux/arm64`, Go 1.26.5
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Git, Docker Engine с Docker Compose v2 и свободные локальные порты 8080, 8081, 9090 и 3000. Все внешние образы закреплены по версиям. Локальные пароли из примера предназначены только для изолированного стенда. Compose передаёт каждому контейнеру только нужные ему переменные: API получает строки подключения PostgreSQL/Valkey без отдельных исходных паролей и пароля Grafana, а `migrate` — только настройки PostgreSQL и миграций.
```sh
cp lab3.env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
```
Compose сначала ожидает PostgreSQL, затем однократно запускает контейнер `migrate`. API-реплики стартуют только после успешного завершения миграций. Единственная публичная точка API — Nginx на `http://localhost:8080`; порты отдельных API-контейнеров на хост не публикуются.
Доступные интерфейсы:
- Swagger UI: `http://localhost:8081`;
- Prometheus: `http://localhost:9090`;
- Grafana: `http://localhost:3000`, логин `admin`, локальный пароль берётся из `.env`;
- Nginx API: `http://localhost:8080`.
Проверки запущенного стенда:
```sh
make check
make test
docker compose --profile tests run --rm --build tests
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
```
Безопасная остановка сохраняет данные в named volumes:
```sh
docker compose stop
```
Повторный запуск выполняется командой `docker compose up -d --scale api=2 --wait`. Намеренное удаление всех учебных данных:
```sh
docker compose down -v
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```mermaid
flowchart LR
C[Клиент] -->|HTTP :8080| N[Nginx]
N --> A1[API replica 1]
N --> A2[API replica 2]
N -. scale .-> AN[API replica N]
A1 --> P[(PostgreSQL)]
A2 --> P
AN --> P
A1 --> V[(Valkey)]
A2 --> V
AN --> V
M[Однократный migrate] --> P
PR[Prometheus] --> A1
PR --> A2
PR --> NE[Nginx exporter]
PR --> PE[PostgreSQL exporter]
PR --> VE[Valkey exporter]
G[Grafana] --> PR
```
PostgreSQL является источником истины для пользователей, чатов, сообщений и записей идемпотентности. Valkey хранит общие сессии, поэтому запросы с одним bearer token могут обслуживаться разными API-репликами. В Valkey ключ имеет вид `session:<sha256(token)>`, а значением служит JSON с `user_id` и сроком действия. Исходный случайный 256-битный токен возвращается клиенту один раз и не записывается в хранилище.
API остаётся stateless относительно пользовательских данных. Локально в процессе находятся только конфигурация, подключения, метрики конкретной реплики и флаг readiness. Каждый HTTP-ответ содержит `X-Instance-Id`, равный hostname контейнера.
### Балансировка и отказ реплики
Nginx OSS 1.30.5 использует round-robin, Docker DNS `127.0.0.11`, повторное разрешение имени `api` каждые 2 секунды и общий upstream zone. Keep-alive соединения ограничены 32. Таймаут подключения равен 1 секунде, таймауты отправки и чтения — 10 секунд. Ошибка соединения или ответ 502/503/504 временно исключают backend на 3 секунды; небезопасные POST автоматически не повторяются.
Docker проверяет `/health/ready` каждой API-реплики раз в 2 секунды с таймаутом 1 секунду. Readiness обращается к PostgreSQL и Valkey. При SIGTERM приложение сначала снимает readiness, затем вызывает `http.Server.Shutdown` с лимитом 5 секунд; Compose даёт контейнеру 8 секунд до принудительной остановки.
Nginx OSS в этом стенде использует пассивное исключение backend. Docker healthcheck не передаёт своё состояние напрямую в Nginx. Поэтому верхняя практическая граница обнаружения отказа складывается из connect timeout и fail timeout; в проверках восстановление произошло примерно за 1.4 секунды, что ниже требуемых 10 секунд.
### Общая идемпотентность
Для `POST /api/v1/chats/{chat_id}/messages` обязателен один заголовок `Idempotency-Key` длиной до 128 символов. Область ключа — `(user_id, chat_id, key)`, поэтому одинаковая строка независима для другого пользователя или чата. Минимальный TTL равен 24 часам.
Запись сообщения и запись идемпотентности создаются в одной транзакции PostgreSQL:
1. Проверяется участие пользователя в чате.
2. Из текста считается SHA-256 для выявления повторного использования ключа с другим запросом.
3. Строка идемпотентности блокируется через `SELECT ... FOR UPDATE`; конкурентные вставки разрешаются первичным ключом и `ON CONFLICT DO NOTHING`.
4. Победившая транзакция резервирует UUID сообщения и вставляет сообщение. Отложенный внешний ключ проверяется при commit.
5. Повтор с тем же телом читает исходное сообщение и возвращает тот же объект. Повтор с другим телом получает `409 Conflict`.
Это позволяет безопасно повторить POST после timeout: подтверждённая ранее запись не дублируется, даже если повтор попал на другую реплику. Каждый успешный запрос также удаляет до 100 истёкших записей по индексу `expires_at`; `SKIP LOCKED` не даёт параллельным репликам ждать одну и ту же пачку.
### Миграции, обновление и развёртывание
API не выполняет DDL при старте. Отдельная команда `/app/messenger migrate` берёт advisory lock и применяет versioned SQL один раз. Миграция идемпотентности только добавляет таблицу и индекс, поэтому совместима с уже работающей предыдущей версией.
`scripts/rolling-update.sh` собирает образы, запускает миграции, временно увеличивает число API с двух до трёх и поочерёдно заменяет старые контейнеры. Скрипт ждёт health каждого состава и затем возвращает две реплики. Workflow deploy запускается только вручную из `main` через `workflow_dispatch`; перед подключением к серверу он повторяет check, formatting, Go tests и build для SHA запуска. На сервере выполняется reset именно на этот SHA, даже если `main` успел измениться. Для SSH используются environment secrets `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY` и заранее проверенный `DEPLOY_KNOWN_HOSTS`; изменяемых actions для SSH нет.
## Наблюдаемость
HTTP middleware каждой реплики публикует число запросов по route/status class, histogram latency, in-flight gauge, число replay идемпотентности, Go runtime и process metrics. Prometheus использует DNS service discovery имени `api` с периодом обновления 2 секунды и отдельно опрашивает Nginx, PostgreSQL и Valkey exporters каждые 5 секунд. Данные хранятся в Prometheus TSDB с retention 7 суток в named volume.
Provisioned dashboard Grafana показывает общий RPS, RPS и p95 по репликам, классы HTTP-ответов, число healthy replicas, in-flight, heap, replay rate, активные соединения и запросы Nginx, соединения PostgreSQL и состояние Valkey.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke | `make test` через Nginx | PASS: регистрация, сессия, чат, пагинация, доступ и идемпотентность | `evidence/lab3-verification.txt` |
| Общие сессии и ≥2 реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Одна сессия обслужена двумя instance ID | `evidence/lab3-verification.txt` |
| Конкурентная идемпотентность | Go black-box через Nginx и database integration suite | 16 одновременных HTTP-запросов обработаны минимум двумя instance ID и вернули один message ID; история содержит одну запись | `evidence/lab3-verification.txt` |
| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | API, PostgreSQL и Valkey перезапущены; старые данные и token доступны | `evidence/lab3-verification.txt` |
| Отказ API | Два запуска failover с остановкой разных реплик | 125–131 успешная запись; 0–1 переходная ошибка; восстановление 1.35–1.44 с; дублей нет | `evidence/lab3-verification.txt` |
| Rolling update | 60 секунд Go-нагрузки и `scripts/rolling-update.sh` | 134894 подтверждения, 2248.16 RPS, 0 ошибок, p95 13.12 ms | `evidence/lab3-rolling-update.json` |
| CPU/RAM-лимиты | `docker compose config` и scale experiment | Лимиты заданы всем сервисам; при 128 MiB API не хватает памяти для Argon2id, поэтому рабочий минимум эксперимента — 256 MiB | `evidence/lab3-verification.txt` |
| Трек 1 | `scripts/run-scale-experiment.sh` | 1/2/4 реплики, два бюджета, concurrency 8/32/64, три повтора | `evidence/lab3-scale-results.json`, `evidence/lab3-scale-rps.svg` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Методика исследования масштабирования
Гипотеза: при фиксированном суммарном бюджете дробление ресурсов между большим числом процессов сначала улучшит распределение конкурентных запросов, но затем проиграет из-за меньших CPU-квот и пулов каждой реплики. При добавлении одинаковых ресурсов на реплику RPS должен расти до насыщения PostgreSQL или Nginx.
Используется Go-клиент `cmd/lab3experiment`. Он создаёт отдельного пользователя, сессию и чат, прогревает стенд 2 секунды, затем 6 секунд выполняет closed-loop POST сообщений через Nginx. Для каждого запроса генерируется уникальный idempotency key. Считаются только HTTP-подтверждения; автоматических retry измеряемого трафика нет. Для каждой точки используются concurrency 8, 32 и 64 и три изолированных повтора с новой базой.
CPU и Nginx RPS считаются по разности монотонных `process_cpu_seconds_total` и `nginx_http_requests_total`, прочитанных непосредственно до и после измеряемого интервала. Поэтому bootstrap, warmup и простой между точками в них не входят. Resident memory и число healthy replicas снимаются после нагрузки с `/metrics` каждой реплики, соединения PostgreSQL — instant query к Prometheus. Короткая контрольная точка дала 2355.97 client RPS и 2359.64 RPS по Nginx-счётчику, расхождение 0.16%.
Фиксированная серия имеет общий бюджет API 2 CPU и 1 GiB RAM: `1×(2 CPU, 1 GiB)`, `2×(1 CPU, 512 MiB)`, `4×(0.5 CPU, 256 MiB)`. Пулы PostgreSQL соответственно равны 16, 8 и 4 соединениям на реплику. В растущей серии каждая реплика получает 1 CPU, 256 MiB и пул 8, поэтому суммарный бюджет растёт вместе с числом процессов.
Предварительный прогон с общим лимитом 512 MiB показал, что вариант `4×128 MiB` не выполняет регистрацию: базовое потребление процесса около 71 MiB, а Argon2id временно требует ещё 64 MiB. Cgroup завершает реплику, Nginx наблюдает premature upstream close. Этот результат определил минимальный рабочий лимит 256 MiB и не включён в сравнение throughput.
Медианные результаты фиксированного суммарного бюджета:
| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms |
| ---: | ---: | ---: | ---: |
| 8 | 2412 / 5.86 | 2484 / 5.59 | 2242 / 6.35 |
| 32 | 2731 / 18.63 | 2365 / 24.93 | 2013 / 34.58 |
| 64 | 2733 / 32.99 | 2198 / 55.06 | 2103 / 67.89 |
При фиксированных 2 CPU и 1 GiB увеличение числа процессов не даёт линейного выигрыша. Две реплики дают небольшой выигрыш только при concurrency 8, одна быстрее при 32 и 64, а четыре проигрывают во всех профилях из-за квоты 0.5 CPU на процесс и расходов четырёх runtime. Для рабочего стенда две реплики остаются осознанным компромиссом ради отказоустойчивости, дальнейшее дробление бюджета не оправдано.
Медианные результаты серии с 1 CPU и 256 MiB на каждую реплику:
| Concurrency | 1 реплика: RPS / p95 ms | 2 реплики: RPS / p95 ms | 4 реплики: RPS / p95 ms |
| ---: | ---: | ---: | ---: |
| 8 | 2712 / 4.95 | 2554 / 5.28 | 2185 / 7.05 |
| 32 | 2818 / 15.34 | 2471 / 21.94 | 2130 / 24.14 |
| 64 | 2044 / 61.44 | 2239 / 57.76 | 1859 / 67.42 |
Даже при росте суммарного CPU практический предел не превышает двух реплик. При concurrency 64 переход `1→2` повышает медианный RPS на 9.6%, а `2→4` снижает его на 17.0%. Одновременно число соединений PostgreSQL растёт `9→17→33`, суммарная resident memory — примерно `36→65→397 MiB`, а p95 у четырёх реплик достигает 67.42 ms. Точная средняя загрузка API составляет примерно `0.43→0.52→0.54 core` при доступных `1→2→4 core`. API не упирается в CPU-квоту. Вместе с ростом DB connections и tail latency это указывает на насыщение общего пути PostgreSQL/Nginx/network и расходы дополнительных процессов, но данный эксперимент не разделяет вклад этих компонентов.
Во всех 54 основных точках было 0 HTTP-ошибок; каждая конфигурация наблюдала все заявленные реплики. Диапазоны трёх повторов заметно пересекаются и доступны в raw JSON, поэтому небольшие отличия нельзя считать точной оценкой абсолютной производительности.
Для настройки пула конфигурация `4×0.5 CPU, 256 MiB` при concurrency 64 дополнительно сравнивалась с 10 и 4 соединениями PostgreSQL на реплику:
| Пул на реплику | Median RPS | p50 / p95 / p99, ms | Соединения PostgreSQL | Ошибки |
| ---: | ---: | ---: | ---: | ---: |
| 10 | 1918 | 30.00 / 59.52 / 101.70 | 41 | 0 |
| 4 | 2179 | 23.61 / 66.76 / 80.91 | 17 | 0 |
Уменьшение пула `10→4` сократило число соединений на 58.5%, повысило медианный RPS на 13.6%, уменьшило p50 на 21.3% и p99 на 20.4%. p95 вырос на 12.2%, то есть распределение задержек изменилось неравномерно. Для данного стенда выбран пул 4: он соответствует CPU-квоте 0.5 core и заметно снижает давление на общую БД. При строгом SLO на tail latency значение следует подбирать повторно на более длинном open-loop тесте.
Итоговый график находится в `evidence/lab3-scale-rps.svg`, обезличенные исходные точки, медианы и диапазоны — в `evidence/lab3-scale-results.json`.
### Отказ и обновление
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| SIGTERM одной API-реплики | Nginx исключает backend, другая реплика продолжает работу | 0–1 переходная ошибка, старый instance ID исчез из последующих ответов | 1.35–1.44 с; подтверждённые сообщения сохранены, дублей нет |
| Полный restart API, PostgreSQL и Valkey без удаления volumes | Данные и сессия сохраняются | Старый bearer token и созданные сущности доступны после запуска | В пределах 90-секундного окна теста; потерь не обнаружено |
| Rolling update API 2→3→2 | Во время замены остаются healthy backend | 134894 успешных POST, 0 ошибок; наблюдались старые и новые instance ID | Перерыва по подтверждённым ответам не зафиксировано |
| Остановка PostgreSQL или Valkey | Readiness становится отрицательной | Все API теряют готовность, потому что хранилища общие | Автоматический failover хранилищ не реализован |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Все компоненты работают на одном Docker-хосте. Отказ хоста, Nginx, PostgreSQL или Valkey останавливает сервис; выполнен отказ процесса API, а не отказ узла.
- Nginx является одной публичной точкой и SPOF. Для эксплуатации потребуются несколько LB с внешним virtual IP или управляемый балансировщик.
- PostgreSQL и Valkey не реплицированы. AOF Valkey с `appendfsync everysec` ограничивает возможную потерю примерно последней секундой при аварии хранилища, но этот сценарий в работе не заявляется как защищённый.
- Нагрузочный клиент closed-loop и подвержен coordinated omission. Результаты сравнивают конфигурации на одной машине и не являются абсолютной производительностью production-системы.
- Все реплики и клиент делят ресурсы одного хоста; дельта process CPU показывает потребление API, а не полную стоимость Docker/network/storage.
- Для эксплуатации нужны TLS, внешнее управление secrets, репликация хранилищ, алерты и длительный soak test; при большом потоке следует отдельно измерить стоимость пакетной очистки idempotency records.
## Источники и инструменты
- Nginx upstream и динамическое DNS: <https://nginx.org/en/docs/http/ngx_http_upstream_module.html>
- Docker Compose services и healthcheck: <https://docs.docker.com/reference/compose-file/services/>
- PostgreSQL explicit locking: <https://www.postgresql.org/docs/current/explicit-locking.html>
- Valkey persistence: <https://valkey.io/topics/persistence/>
- Argon2: RFC 9106, <https://www.rfc-editor.org/rfc/rfc9106>
- Go packages: `pgx/v5`, `valkey-go`, `prometheus/client_golang`, `google/uuid`, `viper`, `x/crypto/argon2`.
- OpenAI Codex использовался для подготовки кода, конфигурации и текста; реализация и результаты проверены запуском тестов и экспериментов.
+168 -3
View File
@@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"strings"
"sync"
"time"
"github.com/google/uuid"
@@ -19,6 +20,7 @@ type apiClient struct {
baseURL string
client *http.Client
token string
userID string
}
func main() {
@@ -26,7 +28,7 @@ func main() {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println("PASS: PostgreSQL and Valkey outage recovery")
fmt.Println("PASS: replica idempotency and dependency recovery")
}
func run() error {
@@ -39,6 +41,9 @@ func run() error {
if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil {
return err
}
if err := client.verifyConcurrentIdempotency(); err != nil {
return err
}
if err := verifyOutage(client, "valkey"); err != nil {
return err
}
@@ -62,9 +67,21 @@ func (c *apiClient) registerAndLogin() error {
if err != nil {
return fmt.Errorf("register: %w", err)
}
if err := closeResponse(response, http.StatusCreated); err != nil {
return fmt.Errorf("register: %w", err)
registrationBody := response.Body
defer func() { _ = registrationBody.Close() }()
if response.StatusCode != http.StatusCreated {
return fmt.Errorf("register status = %d", response.StatusCode)
}
var user struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil {
return fmt.Errorf("decode user: %w", err)
}
if user.ID == "" {
return fmt.Errorf("register returned an empty user id")
}
c.userID = user.ID
request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil)
if err != nil {
@@ -92,6 +109,154 @@ func (c *apiClient) registerAndLogin() error {
return nil
}
type messageResult struct {
messageID string
instanceID string
err error
}
func (c *apiClient) verifyConcurrentIdempotency() error {
chatID, err := c.createChat()
if err != nil {
return err
}
const workers = 16
var (
key = uuid.NewString()
start = make(chan struct{})
results = make(chan messageResult, workers)
group sync.WaitGroup
instances = make(map[string]struct{})
messageID string
)
transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers}
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
defer transport.CloseIdleConnections()
for range workers {
group.Add(1)
go func() {
defer group.Done()
<-start
id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message")
results <- messageResult{messageID: id, instanceID: instance, err: requestErr}
}()
}
close(start)
group.Wait()
close(results)
for result := range results {
if result.err != nil {
return result.err
}
if result.instanceID == "" {
return fmt.Errorf("message response has an empty X-Instance-Id")
}
instances[result.instanceID] = struct{}{}
if messageID == "" {
messageID = result.messageID
}
if result.messageID != messageID {
return fmt.Errorf("idempotent responses returned different message ids")
}
}
if len(instances) < 2 {
return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances))
}
return c.verifyMessageHistory(chatID, messageID)
}
func (c *apiClient) createChat() (string, error) {
payload, err := json.Marshal(map[string]any{
"title": "Replica check", "member_ids": []string{c.userID},
})
if err != nil {
return "", err
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload))
if err != nil {
return "", err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
response, err := c.client.Do(request)
if err != nil {
return "", fmt.Errorf("create chat: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return "", fmt.Errorf("create chat status = %d", response.StatusCode)
}
var chat struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil {
return "", fmt.Errorf("decode chat: %w", err)
}
return chat.ID, nil
}
func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) {
payload, err := json.Marshal(map[string]string{"text": text})
if err != nil {
return "", "", err
}
request, err := http.NewRequest(
http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload),
)
if err != nil {
return "", "", err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", key)
response, err := client.Do(request)
if err != nil {
return "", "", fmt.Errorf("create message: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode)
}
var message struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil {
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err)
}
return message.ID, response.Header.Get("X-Instance-Id"), nil
}
func (c *apiClient) verifyMessageHistory(chatID, messageID string) error {
request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil)
if err != nil {
return err
}
request.Header.Set("Authorization", "Bearer "+c.token)
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("list messages: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("list messages status = %d", response.StatusCode)
}
var page struct {
Items []struct {
ID string `json:"id"`
} `json:"items"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil {
return fmt.Errorf("decode messages: %w", err)
}
if len(page.Items) != 1 || page.Items[0].ID != messageID {
return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items))
}
return nil
}
func verifyOutage(client *apiClient, service string) (returnErr error) {
if err := compose("stop", service); err != nil {
return err
+522
View File
@@ -0,0 +1,522 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
)
type configuration struct {
baseURL string
prometheusURL string
label string
duration time.Duration
warmup time.Duration
concurrency int
replicas int
apiCPUs string
apiMemory string
dbMaxConns int
output string
}
type apiClient struct {
baseURL string
client *http.Client
token string
chatID string
}
type collector struct {
mu sync.Mutex
latencies []time.Duration
errors map[string]int
instances map[string]struct{}
}
type result struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
GOOS string `json:"goos"`
GOARCH string `json:"goarch"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latencySummary `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type latencySummary struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type runtimeSnapshot struct {
apiCPUSeconds float64
apiResident float64
healthyReplicas float64
nginxRequests float64
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
settings := parseFlags()
client := &apiClient{
baseURL: strings.TrimRight(settings.baseURL, "/"),
client: &http.Client{Transport: &http.Transport{
MaxIdleConns: settings.concurrency * 2,
MaxIdleConnsPerHost: settings.concurrency,
MaxConnsPerHost: settings.concurrency,
IdleConnTimeout: 30 * time.Second,
}, Timeout: 10 * time.Second},
}
if err := client.waitForReplicas(settings.replicas); err != nil {
return err
}
if err := client.bootstrap(); err != nil {
return err
}
if settings.warmup > 0 {
warmup := newCollector()
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
return fmt.Errorf("warmup: %w", err)
}
}
var before runtimeSnapshot
if settings.prometheusURL != "" {
snapshot, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics before load: %w", err)
}
before = snapshot
}
measurements := newCollector()
started := time.Now()
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
return err
}
elapsed := time.Since(started)
summary := measurements.result(settings, elapsed)
if settings.prometheusURL != "" {
after, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics after load: %w", err)
}
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
if err != nil {
return err
}
}
return writeResult(settings.output, summary)
}
func (c *apiClient) waitForReplicas(expected int) error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
instances := make(map[string]struct{}, expected)
for ctx.Err() == nil {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
if err != nil {
return err
}
response, err := c.client.Do(request)
if err == nil {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
_ = response.Body.Close()
if response.StatusCode == http.StatusOK {
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
instances[instanceID] = struct{}{}
}
if len(instances) == expected {
return nil
}
}
}
time.Sleep(100 * time.Millisecond)
}
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
}
func parseFlags() configuration {
var settings configuration
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
flag.Parse()
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
os.Exit(2)
}
return settings
}
func (c *apiClient) bootstrap() error {
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
password := "Synthetic-" + uuid.NewString()
var user struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
"username": username, "display_name": "Scale experiment", "password": password,
}, &user, http.StatusCreated); err != nil {
return fmt.Errorf("register: %w", err)
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
if err != nil {
return err
}
request.SetBasicAuth(username, password)
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("login: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return responseError("login", response)
}
var session struct {
Token string `json:"token"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
return fmt.Errorf("decode session: %w", err)
}
c.token = session.Token
var chat struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
"title": "Scale experiment", "member_ids": []string{user.ID},
}, &chat, http.StatusCreated); err != nil {
return fmt.Errorf("create chat: %w", err)
}
c.chatID = chat.ID
return nil
}
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
var sequence atomic.Uint64
var group sync.WaitGroup
for range concurrency {
group.Add(1)
go func() {
defer group.Done()
for ctx.Err() == nil {
number := sequence.Add(1)
started := time.Now()
instanceID, status, err := client.createMessage(ctx, number)
if err != nil && ctx.Err() != nil {
return
}
measurements.observe(time.Since(started), instanceID, status, err)
}
}()
}
group.Wait()
if len(measurements.latencies) == 0 {
return errors.New("no successful requests")
}
return nil
}
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
if err != nil {
return "", 0, err
}
request, err := http.NewRequestWithContext(
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
)
if err != nil {
return "", 0, err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", uuid.NewString())
response, err := c.client.Do(request)
if err != nil {
return "", 0, err
}
defer func() { _ = response.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
if response.StatusCode != http.StatusCreated {
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
}
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
}
func newCollector() *collector {
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
}
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if err != nil {
key := "network"
if status != 0 {
key = strconv.Itoa(status)
}
c.errors[key]++
return
}
c.latencies = append(c.latencies, latency)
if instanceID != "" {
c.instances[instanceID] = struct{}{}
}
}
func (c *collector) result(settings configuration, elapsed time.Duration) result {
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
return result{
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
Concurrency: settings.concurrency, Replicas: settings.replicas,
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
SuccessfulRequests: len(c.latencies),
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
LatencyMS: latencySummary{
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
},
ObservedInstances: len(c.instances),
}
}
func percentile(values []time.Duration, percentile float64) float64 {
if len(values) == 0 {
return 0
}
index := int(float64(len(values)-1) * percentile)
return float64(values[index].Microseconds()) / 1000
}
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
hosts, err := net.LookupHost("api")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
}
uniqueHosts := make(map[string]struct{}, len(hosts))
var snapshot runtimeSnapshot
for _, host := range hosts {
if _, duplicate := uniqueHosts[host]; duplicate {
continue
}
uniqueHosts[host] = struct{}{}
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
}
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
if err != nil {
return runtimeSnapshot{}, err
}
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
if err != nil {
return runtimeSnapshot{}, err
}
snapshot.apiCPUSeconds += cpuSeconds
snapshot.apiResident += resident
snapshot.healthyReplicas++
}
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
}
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
if err != nil {
return runtimeSnapshot{}, err
}
return snapshot, nil
}
func scrapeMetrics(endpoint string) (map[string]float64, error) {
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %d", response.StatusCode)
}
metrics := make(map[string]float64)
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
name, _, _ := strings.Cut(fields[0], "{")
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
if err != nil {
return nil, fmt.Errorf("parse metric %s: %w", name, err)
}
metrics[name] += value
}
if err := scanner.Err(); err != nil {
return nil, err
}
return metrics, nil
}
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
value, found := metrics[name]
if !found {
return 0, fmt.Errorf("metric %s not found", name)
}
return value, nil
}
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
if err != nil {
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
}
seconds := elapsed.Seconds()
return map[string]float64{
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
"api_resident_bytes": after.apiResident,
"healthy_replicas": after.healthyReplicas,
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
"postgres_connections": postgresConnections,
}, nil
}
func prometheusQuery(baseURL, query string) (float64, error) {
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return 0, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
}
var payload struct {
Data struct {
Result []struct {
Value []json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
return 0, err
}
var total float64
for _, item := range payload.Data.Result {
if len(item.Value) != 2 {
continue
}
var encoded string
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
return 0, err
}
value, err := strconv.ParseFloat(encoded, 64)
if err != nil {
return 0, err
}
total += value
}
return total, nil
}
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
payload, err := json.Marshal(body)
if err != nil {
return err
}
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
if token != "" {
request.Header.Set("Authorization", "Bearer "+token)
}
if idempotencyKey != "" {
request.Header.Set("Idempotency-Key", idempotencyKey)
}
response, err := c.client.Do(request)
if err != nil {
return err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != expected {
return responseError(method+" "+path, response)
}
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
}
func responseError(action string, response *http.Response) error {
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
}
func writeResult(path string, value result) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
data = append(data, '\n')
if path == "" {
_, err = os.Stdout.Write(data)
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}
+366
View File
@@ -0,0 +1,366 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"html"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
type latency struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type inputResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
Instances []string `json:"instances"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type rawResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type aggregate struct {
Label string `json:"label"`
Repeats int `json:"repeats"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRPSMedian float64 `json:"successful_rps_median"`
SuccessfulRPSMin float64 `json:"successful_rps_min"`
SuccessfulRPSMax float64 `json:"successful_rps_max"`
LatencyP50MedianMS float64 `json:"latency_p50_median_ms"`
LatencyP95MedianMS float64 `json:"latency_p95_median_ms"`
LatencyP99MedianMS float64 `json:"latency_p99_median_ms"`
TotalErrors int `json:"total_errors"`
ObservedInstancesMin int `json:"observed_instances_min"`
APICPUCoresMedian float64 `json:"api_cpu_cores_median"`
APIResidentBytesMedian float64 `json:"api_resident_bytes_median"`
PostgresConnectionsMedian float64 `json:"postgres_connections_median"`
PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"`
}
type report struct {
GeneratedAt string `json:"generated_at"`
Method string `json:"method"`
Raw []rawResult `json:"raw"`
Summary []aggregate `json:"summary"`
}
func main() {
inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files")
outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output")
chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output")
flag.Parse()
results, err := readResults(*inputDirectory)
if err != nil {
fatal(err)
}
aggregates := summarize(results)
if err := writeJSON(*outputPath, report{
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Method: "closed-loop HTTP through Nginx; median of three isolated repeats",
Raw: sanitize(results),
Summary: aggregates,
}); err != nil {
fatal(err)
}
if err := writeChart(*chartPath, aggregates); err != nil {
fatal(err)
}
}
func readResults(directory string) ([]inputResult, error) {
var paths []string
for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} {
matches, err := filepath.Glob(filepath.Join(directory, pattern))
if err != nil {
return nil, err
}
paths = append(paths, matches...)
}
if len(paths) == 0 {
return nil, fmt.Errorf("no experiment results in %s", directory)
}
results := make([]inputResult, 0, len(paths))
for _, path := range paths {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var result inputResult
if err := json.Unmarshal(data, &result); err != nil {
return nil, fmt.Errorf("decode %s: %w", path, err)
}
results = append(results, result)
}
sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label })
return results, nil
}
func sanitize(results []inputResult) []rawResult {
raw := make([]rawResult, 0, len(results))
for _, value := range results {
raw = append(raw, rawResult{
Label: value.Label, Timestamp: value.Timestamp,
DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds,
Concurrency: value.Concurrency, Replicas: value.Replicas,
APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns,
SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS,
Errors: value.Errors, LatencyMS: value.LatencyMS,
ObservedInstances: instanceCount(value), Prometheus: value.Prometheus,
})
}
return raw
}
func summarize(results []inputResult) []aggregate {
groups := make(map[string][]inputResult)
for _, value := range results {
groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value)
}
labels := make([]string, 0, len(groups))
for label := range groups {
labels = append(labels, label)
}
sort.Strings(labels)
aggregates := make([]aggregate, 0, len(labels))
for _, label := range labels {
values := groups[label]
first := values[0]
rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS })
p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 })
p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 })
p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 })
cpu := metricNumbers(values, "api_cpu_cores")
memory := metricNumbers(values, "api_resident_bytes")
connections := metricNumbers(values, "postgres_connections")
healthy := metricNumbers(values, "healthy_replicas")
aggregate := aggregate{
Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas,
APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns,
SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps),
LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99),
ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu),
APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections),
PrometheusHealthyMin: minimum(healthy),
}
for _, value := range values {
if count := instanceCount(value); count < aggregate.ObservedInstancesMin {
aggregate.ObservedInstancesMin = count
}
for _, count := range value.Errors {
aggregate.TotalErrors += count
}
}
aggregates = append(aggregates, aggregate)
}
return aggregates
}
func instanceCount(value inputResult) int {
if value.ObservedInstances > 0 {
return value.ObservedInstances
}
return len(value.Instances)
}
func withoutRepeat(label string) string {
index := strings.LastIndex(label, "-n")
if index == -1 {
return label
}
if _, err := strconv.Atoi(label[index+2:]); err != nil {
return label
}
return label[:index]
}
func numbers(values []inputResult, value func(inputResult) float64) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
result = append(result, value(item))
}
return result
}
func metricNumbers(values []inputResult, name string) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
if value, ok := item.Prometheus[name]; ok {
result = append(result, value)
}
}
return result
}
func median(values []float64) float64 {
if len(values) == 0 {
return 0
}
values = append([]float64(nil), values...)
sort.Float64s(values)
middle := len(values) / 2
if len(values)%2 == 1 {
return values[middle]
}
return (values[middle-1] + values[middle]) / 2
}
func minimum(values []float64) float64 {
if len(values) == 0 {
return 0
}
minimum := values[0]
for _, value := range values[1:] {
if value < minimum {
minimum = value
}
}
return minimum
}
func maximum(values []float64) float64 {
if len(values) == 0 {
return 0
}
maximum := values[0]
for _, value := range values[1:] {
if value > maximum {
maximum = value
}
}
return maximum
}
func writeJSON(path string, value any) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
func writeChart(path string, values []aggregate) error {
const width, height = 960, 560
const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0
plotWidth := width - left - right
plotHeight := height - top - bottom
var selected []aggregate
var maxRPS float64
for _, value := range values {
if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") {
selected = append(selected, value)
if value.SuccessfulRPSMedian > maxRPS {
maxRPS = value.SuccessfulRPSMedian
}
}
}
if maxRPS == 0 {
return fmt.Errorf("no scale results for chart")
}
maxRPS *= 1.1
var output strings.Builder
fmt.Fprintf(&output, `<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">`, width, height, width, height)
output.WriteString(`<rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style>`)
fmt.Fprintf(&output, `<text x="%g" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text>`, left)
for step := 0; step <= 5; step++ {
y := top + plotHeight*float64(5-step)/5
value := maxRPS * float64(step) / 5
fmt.Fprintf(&output, `<line class="grid" x1="%g" y1="%g" x2="%g" y2="%g"/><text x="%g" y="%g" text-anchor="end" font-size="12">%.0f</text>`, left, y, left+plotWidth, y, left-10, y+4, value)
}
fmt.Fprintf(&output, `<line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/><line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/>`, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight)
for index, replicas := range []int{1, 2, 4} {
x := left + plotWidth*float64(index)/2
fmt.Fprintf(&output, `<text x="%g" y="%g" text-anchor="middle" font-size="13">%d</text>`, x, top+plotHeight+25, replicas)
}
fmt.Fprintf(&output, `<text x="%g" y="%d" text-anchor="middle" font-size="13">API replicas</text>`, left+plotWidth/2, height-15)
colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"}
type series struct {
name string
values []aggregate
}
groups := make(map[string][]aggregate)
for _, value := range selected {
name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency)
groups[name] = append(groups[name], value)
}
var names []string
for name := range groups {
names = append(names, name)
}
sort.Strings(names)
allSeries := make([]series, 0, len(names))
for _, name := range names {
points := groups[name]
sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas })
allSeries = append(allSeries, series{name: name, values: points})
}
for index, current := range allSeries {
color := colors[index%len(colors)]
var coordinates []string
for _, point := range current.values {
xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas]
x := left + plotWidth*xIndex/2
y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS)
coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y))
fmt.Fprintf(&output, `<circle class="point" cx="%g" cy="%g" r="4" fill="%s"/>`, x, y, color)
}
fmt.Fprintf(&output, `<polyline class="line" points="%s" stroke="%s"/>`, strings.Join(coordinates, " "), color)
legendY := top + 20 + float64(index*28)
fmt.Fprintf(&output, `<line x1="%g" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="3"/><text x="%g" y="%g" font-size="12">%s</text>`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name))
}
output.WriteString(`</svg>`)
return os.WriteFile(path, []byte(output.String()), 0o644)
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
+43 -10
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"fmt"
"log/slog"
"net"
"net/http"
@@ -9,33 +10,46 @@ import (
"os/signal"
"syscall"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/messenger"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/messenger"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
)
func main() {
appLogger := logger.New()
if err := run(appLogger); err != nil {
if err := run(appLogger, os.Args[1:]); err != nil {
appLogger.Error("server stopped", "error", err)
os.Exit(1)
}
}
func run(logger *slog.Logger) error {
func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 0 {
return runServer(logger)
}
if len(arguments) == 1 && arguments[0] == "migrate" {
return runMigrations()
}
return fmt.Errorf("usage: messenger [migrate]")
}
func runServer(logger *slog.Logger) error {
settings, err := config.Load()
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
MigrationTimeout: settings.MigrationTimeout,
})
if err != nil {
return err
@@ -60,7 +74,9 @@ func run(logger *slog.Logger) error {
sessions,
passwords,
settings.SessionTTL,
settings.IdempotencyTTL,
settings.RequestTimeout,
instanceID,
settings.SwaggerPort,
)
server := &http.Server{
@@ -97,3 +113,20 @@ func run(logger *slog.Logger) error {
return nil
}
}
func runMigrations() error {
settings, err := config.LoadMigration()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
+125 -16
View File
@@ -32,31 +32,107 @@ services:
mem_limit: ${VALKEY_MEMORY}
restart: unless-stopped
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab2}
env_file: .env
ports:
- "${API_BIND_ADDRESS}:${APP_PORT}"
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab3}
command: ["/app/messenger", "migrate"]
environment:
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
cpus: ${MIGRATE_CPUS}
mem_limit: ${MIGRATE_MEMORY}
restart: "no"
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab3}
environment:
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
DATABASE_URL: ${DATABASE_URL}
CACHE_URL: ${CACHE_URL}
SESSION_STORE: ${SESSION_STORE}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
SWAGGER_PORT: ${SWAGGER_PORT}
expose:
- "8080"
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
interval: 2s
timeout: 1s
start_period: 5s
retries: 2
deploy:
replicas: ${API_REPLICAS:-2}
cpus: ${API_CPUS}
mem_limit: ${API_MEMORY}
stop_grace_period: 10s
stop_grace_period: 8s
restart: unless-stopped
nginx:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab3}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"]
interval: 2s
timeout: 1s
retries: 10
cpus: ${NGINX_CPUS}
mem_limit: ${NGINX_MEMORY}
restart: unless-stopped
swagger:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab3}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
api:
nginx:
condition: service_healthy
cpus: ${SWAGGER_CPUS}
mem_limit: ${SWAGGER_MEMORY}
restart: unless-stopped
tests:
@@ -68,23 +144,30 @@ services:
TEST_DATABASE_URL: ${DATABASE_URL}
TEST_CACHE_URL: ${CACHE_URL}
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
cpus: ${TEST_CPUS}
mem_limit: ${TEST_MEMORY}
experiment:
profiles: ["research"]
build:
context: .
target: experiment
env_file: .env
command: ["go", "run", "./cmd/lab2experiment", "-output=/output/lab2-track2.json"]
image: ${API_IMAGE:-mtusi-messenger:lab3}
command:
- /app/lab3experiment
- -base-url=http://nginx:8080
- -prometheus-url=http://prometheus:9090
- -output=/output/lab3-scale.json
volumes:
- ./evidence:/output
depends_on:
postgres:
nginx:
condition: service_healthy
prometheus:
condition: service_healthy
cpus: ${TEST_CPUS}
mem_limit: ${TEST_MEMORY}
postgres-exporter:
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
@@ -96,6 +179,8 @@ services:
depends_on:
postgres:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
valkey-exporter:
@@ -107,12 +192,25 @@ services:
depends_on:
valkey:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
nginx-exporter:
image: nginx/nginx-prometheus-exporter:1.5.1
command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"]
depends_on:
nginx:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
prometheus:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab3}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
@@ -121,16 +219,25 @@ services:
- prometheus_data:/prometheus
ports:
- "${PROMETHEUS_BIND_ADDRESS}:9090"
healthcheck:
test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"]
interval: 5s
timeout: 3s
retries: 20
depends_on:
- api
- nginx-exporter
- postgres-exporter
- valkey-exporter
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
grafana:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab3}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
@@ -141,6 +248,8 @@ services:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
volumes:
+29
View File
@@ -0,0 +1,29 @@
{
"label": "rolling-update",
"timestamp": "2026-10-02T09:54:15Z",
"goos": "linux",
"goarch": "arm64",
"duration_seconds": 60.001996238,
"warmup_seconds": 3,
"concurrency": 16,
"replicas": 2,
"api_cpus_per_replica": "1",
"api_memory_per_replica": "256m",
"db_max_connections_per_replica": 10,
"successful_requests": 134894,
"successful_rps": 2248.158535675018,
"errors": {},
"latency_ms": {
"p50": 6.41,
"p95": 13.12,
"p99": 22.006
},
"observed_instances": 5,
"prometheus": {
"api_cpu_cores": 0.4491430158747135,
"api_resident_bytes": 58523648,
"healthy_replicas": 2,
"nginx_rps": 2253.547396138092,
"postgres_connections": 21
}
}
File diff suppressed because it is too large Load Diff
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="960" height="560" viewBox="0 0 960 560"><rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style><text x="85" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text><line class="grid" x1="85" y1="490" x2="720" y2="490"/><text x="75" y="494" text-anchor="end" font-size="12">0</text><line class="grid" x1="85" y1="401" x2="720" y2="401"/><text x="75" y="405" text-anchor="end" font-size="12">620</text><line class="grid" x1="85" y1="312" x2="720" y2="312"/><text x="75" y="316" text-anchor="end" font-size="12">1240</text><line class="grid" x1="85" y1="223" x2="720" y2="223"/><text x="75" y="227" text-anchor="end" font-size="12">1860</text><line class="grid" x1="85" y1="134" x2="720" y2="134"/><text x="75" y="138" text-anchor="end" font-size="12">2480</text><line class="grid" x1="85" y1="45" x2="720" y2="45"/><text x="75" y="49" text-anchor="end" font-size="12">3100</text><line class="axis" x1="85" y1="45" x2="85" y2="490"/><line class="axis" x1="85" y1="490" x2="720" y2="490"/><text x="85" y="515" text-anchor="middle" font-size="13">1</text><text x="402.5" y="515" text-anchor="middle" font-size="13">2</text><text x="720" y="515" text-anchor="middle" font-size="13">4</text><text x="402.5" y="545" text-anchor="middle" font-size="13">API replicas</text><circle class="point" cx="85" cy="97.94469794128702" r="4" fill="#1a73e8"/><circle class="point" cx="402.5" cy="150.47324315126576" r="4" fill="#1a73e8"/><circle class="point" cx="720" cy="200.9869273874724" r="4" fill="#1a73e8"/><polyline class="line" points="85,97.94469794128702 402.5,150.47324315126576 720,200.9869273874724" stroke="#1a73e8"/><line x1="745" y1="65" x2="770" y2="65" stroke="#1a73e8" stroke-width="3"/><text x="778" y="69" font-size="12">fixed, c=32</text><circle class="point" cx="85" cy="97.59212639106859" r="4" fill="#d93025"/><circle class="point" cx="402.5" cy="174.50312647976264" r="4" fill="#d93025"/><circle class="point" cx="720" cy="188.14076549389546" r="4" fill="#d93025"/><polyline class="line" points="85,97.59212639106859 402.5,174.50312647976264 720,188.14076549389546" stroke="#d93025"/><line x1="745" y1="93" x2="770" y2="93" stroke="#d93025" stroke-width="3"/><text x="778" y="97" font-size="12">fixed, c=64</text><circle class="point" cx="85" cy="143.7066057420492" r="4" fill="#188038"/><circle class="point" cx="402.5" cy="133.35012431532638" r="4" fill="#188038"/><circle class="point" cx="720" cy="168.15646800421806" r="4" fill="#188038"/><polyline class="line" points="85,143.7066057420492 402.5,133.35012431532638 720,168.15646800421806" stroke="#188038"/><line x1="745" y1="121" x2="770" y2="121" stroke="#188038" stroke-width="3"/><text x="778" y="125" font-size="12">fixed, c=8</text><circle class="point" cx="85" cy="85.45454545454552" r="4" fill="#9334e6"/><circle class="point" cx="402.5" cy="135.30606743203387" r="4" fill="#9334e6"/><circle class="point" cx="720" cy="184.2610413680913" r="4" fill="#9334e6"/><polyline class="line" points="85,85.45454545454552 402.5,135.30606743203387 720,184.2610413680913" stroke="#9334e6"/><line x1="745" y1="149" x2="770" y2="149" stroke="#9334e6" stroke-width="3"/><text x="778" y="153" font-size="12">growing, c=32</text><circle class="point" cx="85" cy="196.57385409938001" r="4" fill="#f29900"/><circle class="point" cx="402.5" cy="168.53866266092896" r="4" fill="#f29900"/><circle class="point" cx="720" cy="223.13942705833085" r="4" fill="#f29900"/><polyline class="line" points="85,196.57385409938001 402.5,168.53866266092896 720,223.13942705833085" stroke="#f29900"/><line x1="745" y1="177" x2="770" y2="177" stroke="#f29900" stroke-width="3"/><text x="778" y="181" font-size="12">growing, c=64</text><circle class="point" cx="85" cy="100.60936627447775" r="4" fill="#00897b"/><circle class="point" cx="402.5" cy="123.35560037954652" r="4" fill="#00897b"/><circle class="point" cx="720" cy="176.3488242715569" r="4" fill="#00897b"/><polyline class="line" points="85,100.60936627447775 402.5,123.35560037954652 720,176.3488242715569" stroke="#00897b"/><line x1="745" y1="205" x2="770" y2="205" stroke="#00897b" stroke-width="3"/><text x="778" y="209" font-size="12">growing, c=8</text></svg>

After

Width:  |  Height:  |  Size: 4.3 KiB

+51
View File
@@ -0,0 +1,51 @@
Lab 3 verification
Date: 2026-10-02
Environment: Docker linux/arm64, Go 1.26.5
Contract and build
- make check: PASS
- docker compose --env-file .env config --quiet: PASS
- go test ./cmd/lab3experiment: PASS
- go test ./internal/messenger ./internal/model ./internal/password: PASS
Integration
- docker compose --profile tests run --rm --build tests: PASS
- PostgreSQL concurrent idempotency: 8 calls through two Store instances, one message ID and one row
- black-box replica idempotency: 16 concurrent calls through Nginx, at least two instance IDs, one message ID and one history item
- PostgreSQL and Valkey stop/recovery: PASS
- persistence after restarting API, PostgreSQL and Valkey: PASS; the original session and data remained valid
- public smoke through Nginx: PASS
API replica failover
- run 1: 2 instances observed, 131 successful writes, 0 transition errors, first recovered write after 1.353 s
- run 2: 2 instances observed, 125 successful writes, 1 transition error, first recovered write after 1.442 s
- confirmed messages remained readable and idempotent retries did not create duplicates
Rolling update under Go load
- duration: 60.002 s
- concurrency: 16
- successful requests: 134894
- successful RPS: 2248.16
- errors: 0
- latency p50/p95/p99: 6.41/13.12/22.01 ms
- healthy API replicas after update: 2
Resource boundary
- 4 API replicas with 128 MiB each restarted while Argon2id processed registration
- idle process memory was about 71 MiB; Argon2id memory parameter was 64 MiB
- the fixed-budget experiment therefore uses 256 MiB per replica as its minimum
Scale experiment
- command: DURATION=6s WARMUP=2s REPEATS=3 CONCURRENCIES='8 32 64' scripts/run-scale-experiment.sh
- configurations: fixed total API resources and growing per-replica resources, 1/2/4 replicas
- completed points: 60; HTTP errors: 0; every point observed all configured replicas
- CPU and Nginx RPS use counter deltas read immediately before and after measured load
- maximum difference between client RPS and Nginx counter RPS across 60 points: 0.75%
- reducing the pool from 10 to 4 connections per replica changed median RPS from 1918 to 2179 and PostgreSQL connections from 41 to 17
- combined sanitized raw data and median aggregates: evidence/lab3-scale-results.json
- comparison chart: evidence/lab3-scale-rps.svg
Configuration scope
- API receives only its application settings and PostgreSQL/Valkey connection URLs
- migrate receives only PostgreSQL pool, dependency and migration settings
- neither container receives the Grafana password or separate source passwords of unrelated services
+5 -1
View File
@@ -1,4 +1,4 @@
module gitea.privcloud.tech/dizi/mtusi2026lab2
module gitea.privcloud.tech/dizi/mtusi2026lab3
go 1.26.0
@@ -7,6 +7,7 @@ require (
github.com/jackc/pgx/v5 v5.11.0
github.com/prometheus/client_golang v1.23.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
golang.org/x/crypto v0.57.0
)
@@ -14,6 +15,7 @@ require (
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -21,6 +23,7 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
@@ -36,4 +39,5 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+85 -32
View File
@@ -16,13 +16,13 @@ type Config struct {
DatabaseURL string
CacheURL string
SessionTTL time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
MigrationTimeout time.Duration
RequestTimeout time.Duration
HTTPReadHeaderTimeout time.Duration
HTTPReadTimeout time.Duration
@@ -37,6 +37,27 @@ type Config struct {
SwaggerPort string
}
type MigrationConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
MigrationTimeout time.Duration
}
type databaseConfig struct {
url string
maxConns int
minConns int
maxConnectionAge time.Duration
maxConnectionIdle time.Duration
healthCheckPeriod time.Duration
dependencyTimeout time.Duration
}
func Load() (Config, error) {
values := viper.New()
values.AutomaticEnv()
@@ -49,7 +70,7 @@ func Load() (Config, error) {
if err != nil {
return Config{}, err
}
databaseURL, err := requiredString(values, "DATABASE_URL")
database, err := loadDatabaseConfig(values)
if err != nil {
return Config{}, err
}
@@ -69,31 +90,7 @@ func Load() (Config, error) {
if err != nil {
return Config{}, err
}
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
if err != nil {
return Config{}, err
}
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
if err != nil {
return Config{}, err
}
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
if err != nil {
return Config{}, err
}
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
if err != nil {
return Config{}, err
}
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
if err != nil {
return Config{}, err
}
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
if err != nil {
return Config{}, err
}
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
if err != nil {
return Config{}, err
}
@@ -147,11 +144,12 @@ func Load() (Config, error) {
}
return Config{
Address: net.JoinHostPort(host, port), DatabaseURL: databaseURL, CacheURL: cacheURL,
SessionTTL: sessionTTL, DBMaxConns: int32(maxConns), DBMinConns: int32(minConns),
DBMaxConnectionAge: maxConnectionAge, DBMaxConnectionIdle: maxConnectionIdle,
DBHealthCheckPeriod: healthCheckPeriod, DependencyTimeout: dependencyTimeout,
MigrationTimeout: migrationTimeout, RequestTimeout: requestTimeout,
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
RequestTimeout: requestTimeout,
HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout,
HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout,
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
@@ -161,6 +159,61 @@ func Load() (Config, error) {
}, nil
}
func LoadMigration() (MigrationConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return MigrationConfig{}, err
}
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
if err != nil {
return MigrationConfig{}, err
}
return MigrationConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
MigrationTimeout: migrationTimeout,
}, nil
}
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
databaseURL, err := requiredString(values, "DATABASE_URL")
if err != nil {
return databaseConfig{}, err
}
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
if err != nil {
return databaseConfig{}, err
}
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
if err != nil {
return databaseConfig{}, err
}
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
if err != nil {
return databaseConfig{}, err
}
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
if err != nil {
return databaseConfig{}, err
}
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
if err != nil {
return databaseConfig{}, err
}
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
if err != nil {
return databaseConfig{}, err
}
return databaseConfig{
url: databaseURL, maxConns: maxConns, minConns: minConns,
maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle,
healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout,
}, nil
}
func requiredString(values *viper.Viper, name string) (string, error) {
value := strings.TrimSpace(values.GetString(name))
if value == "" {
+10 -11
View File
@@ -8,7 +8,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab2/migrations"
"gitea.privcloud.tech/dizi/mtusi2026lab3/migrations"
)
var (
@@ -20,7 +20,6 @@ var (
errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive")
errHealthCheckPeriod = errors.New("database health check period must be positive")
errConnectionTimeout = errors.New("database connection timeout must be positive")
errMigrationTimeout = errors.New("database migration timeout must be positive")
)
type Config struct {
@@ -31,7 +30,6 @@ type Config struct {
MaxConnectionIdle time.Duration
HealthCheckPeriod time.Duration
ConnectTimeout time.Duration
MigrationTimeout time.Duration
}
func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
@@ -59,15 +57,18 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
return nil, fmt.Errorf("ping database: %w", err)
}
migrationCtx, cancel := context.WithTimeout(ctx, config.MigrationTimeout)
defer cancel()
if err := applyMigrations(migrationCtx, pool, migrations.FS); err != nil {
pool.Close()
return nil, err
}
return pool, nil
}
func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error {
if timeout <= 0 {
return errors.New("database migration timeout must be positive")
}
migrationCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
return applyMigrations(migrationCtx, pool, migrations.FS)
}
func validateConfig(config Config) error {
switch {
case config.URL == "":
@@ -86,8 +87,6 @@ func validateConfig(config Config) error {
return errHealthCheckPeriod
case config.ConnectTimeout <= 0:
return errConnectionTimeout
case config.MigrationTimeout <= 0:
return errMigrationTimeout
default:
return nil
}
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"net/http"
"strings"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
)
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
+15 -3
View File
@@ -9,8 +9,8 @@ import (
"time"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
@@ -202,6 +202,10 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method == http.MethodGet {
return s.listMessages(w, r, user, chatID)
}
key, valid := idempotencyKey(r)
if !valid {
return invalidRequest()
}
var input struct {
Text string `json:"text"`
}
@@ -211,13 +215,21 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
if !validText(input.Text, 4000) {
return invalidRequest()
}
message, err := s.messengerStore.CreateMessage(r.Context(), chatID, user.ID, input.Text)
message, replay, err := s.messengerStore.CreateMessage(
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
)
if errors.Is(err, postgres.ErrNotFound) {
return notFound()
}
if errors.Is(err, postgres.ErrIdempotencyConflict) {
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
}
if err != nil {
return unavailable()
}
if replay {
s.metrics.replays.Inc()
}
writeJSON(w, http.StatusCreated, message)
return nil
}
+2 -2
View File
@@ -4,7 +4,7 @@ import (
"context"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
)
type MessengerStore interface {
@@ -15,7 +15,7 @@ type MessengerStore interface {
CreateChat(context.Context, string, []string, string) (model.Chat, error)
ChatForUser(context.Context, string, string) (model.Chat, error)
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
CreateMessage(context.Context, string, string, string) (model.Message, error)
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
}
+8
View File
@@ -15,6 +15,7 @@ type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
handler http.Handler
}
@@ -40,11 +41,18 @@ func newMetrics() *metrics {
Name: "requests_in_flight",
Help: "HTTP requests currently being served.",
}),
replays: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "idempotency_replays_total",
Help: "Message requests served from an idempotency record.",
}),
}
registry.MustRegister(
result.requests,
result.duration,
result.inFlight,
result.replays,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
+1 -1
View File
@@ -1,7 +1,7 @@
package messenger
import (
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
"github.com/google/uuid"
)
+11
View File
@@ -9,6 +9,7 @@ import (
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
@@ -17,6 +18,8 @@ import (
const maxRequestBytes = 1 << 20
var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError {
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
@@ -54,6 +57,14 @@ func validText(value string, maxRunes int) bool {
return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes
}
func idempotencyKey(r *http.Request) (string, bool) {
values := r.Header.Values("Idempotency-Key")
if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) {
return "", false
}
return values[0], true
}
type pageCursor struct {
Kind string `json:"kind"`
Resource string `json:"resource"`
+19 -5
View File
@@ -13,7 +13,9 @@ type Server struct {
sessions SessionStore
passwords PasswordService
sessionTTL time.Duration
idempotencyTTL time.Duration
requestTTL time.Duration
instanceID string
logger *slog.Logger
swaggerPort string
metrics *metrics
@@ -30,10 +32,21 @@ func (w *statusWriter) WriteHeader(status int) {
w.ResponseWriter.WriteHeader(status)
}
func NewServer(logger *slog.Logger, messages MessengerStore, sessions SessionStore, passwords PasswordService, sessionTTL, requestTTL time.Duration, swaggerPort string) *Server {
func NewServer(
logger *slog.Logger,
messages MessengerStore,
sessions SessionStore,
passwords PasswordService,
sessionTTL time.Duration,
idempotencyTTL time.Duration,
requestTTL time.Duration,
instanceID string,
swaggerPort string,
) *Server {
s := &Server{
messengerStore: messages, sessions: sessions, passwords: passwords,
sessionTTL: sessionTTL, requestTTL: requestTTL, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
}
s.ready.Store(true)
return s
@@ -48,6 +61,7 @@ func (s *Server) Handler() http.Handler {
}
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", s.instanceID)
requestID, err := newID()
if err != nil {
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
@@ -70,8 +84,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
}()
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key")
wrapped.WriteHeader(http.StatusNoContent)
return
}
@@ -92,7 +106,7 @@ func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool
return false
}
w.Header().Set("Access-Control-Allow-Origin", origin)
w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id")
w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id")
w.Header().Add("Vary", "Origin")
return true
}
+101 -17
View File
@@ -3,6 +3,7 @@ package messenger
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
@@ -11,17 +12,36 @@ import (
"testing"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
)
var errDependency = errors.New("dependency unavailable")
type MessengerSuite struct {
suite.Suite
logger *slog.Logger
}
func TestMessengerSuite(t *testing.T) {
suite.Run(t, new(MessengerSuite))
}
func (s *MessengerSuite) SetupTest() {
s.logger = slog.New(slog.NewTextHandler(io.Discard, nil))
}
type messengerStoreStub struct {
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
}
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
@@ -43,8 +63,8 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
return nil, false, nil
}
func (messengerStoreStub) CreateMessage(context.Context, string, string, string) (model.Message, error) {
return model.Message{}, nil
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
return s.message, s.messageReplay, s.messageErr
}
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
return nil, false, nil
@@ -74,7 +94,8 @@ func (s passwordsStub) Verify(string, string) (bool, error) {
return s.verified, s.err
}
func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
t := s.T()
tests := []struct {
name string
store messengerStoreStub
@@ -86,7 +107,7 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := newTestServer(tt.store, tt.sessions, passwordsStub{})
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
response := serveRequest(server, http.MethodGet, "/health/ready", "")
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
@@ -95,7 +116,8 @@ func TestReadinessReturnsUnavailableForDependencyFailure(t *testing.T) {
}
}
func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() {
t := s.T()
tests := []struct {
name string
store messengerStoreStub
@@ -107,7 +129,7 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := newTestServer(tt.store, tt.sessions, passwordsStub{})
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)
request.Header.Set("Authorization", "Bearer valid-token")
response := httptest.NewRecorder()
@@ -119,15 +141,16 @@ func TestBusinessRequestReturnsUnavailableForDependencyFailure(t *testing.T) {
}
}
func TestAuthenticationChallenges(t *testing.T) {
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
func (s *MessengerSuite) TestAuthenticationChallenges() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "")
if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` {
t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate"))
}
store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}}
server = newTestServer(store, sessionStub{}, passwordsStub{verified: false})
server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false})
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil)
request.SetBasicAuth("user", "wrong-password")
response := httptest.NewRecorder()
@@ -137,8 +160,9 @@ func TestAuthenticationChallenges(t *testing.T) {
}
}
func TestSessionCreationRejectsChunkedBody(t *testing.T) {
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}"))
request.ContentLength = -1
request.SetBasicAuth("user", "password")
@@ -149,17 +173,77 @@ func TestSessionCreationRejectsChunkedBody(t *testing.T) {
}
}
func TestRegistrationRejectsWhitespacePassword(t *testing.T) {
server := newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
}
}
func newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
return NewServer(logger, messages, sessions, passwords, time.Hour, time.Second, "8081")
func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
for _, response := range []*httptest.ResponseRecorder{
serveRequest(server, http.MethodGet, "/health/live", ""),
serveRequest(server, http.MethodGet, "/api/v1/users/me", ""),
} {
if value := response.Header().Get("X-Instance-Id"); value != "test-instance" {
t.Fatalf("X-Instance-Id = %q, want test-instance", value)
}
}
}
func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() {
t := s.T()
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
tests := []struct {
name string
key string
}{
{name: "missing"},
{name: "invalid", key: "invalid key"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/json")
if tt.key != "" {
request.Header.Set("Idempotency-Key", tt.key)
}
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
}
})
}
}
func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() {
t := s.T()
store := messengerStoreStub{
userByID: model.User{ID: "user-id"},
messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict),
}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", "same-key")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusConflict {
t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict)
}
}
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
}
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
+13 -7
View File
@@ -4,15 +4,21 @@ import (
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
func TestUserTimestampHasFixedMicroseconds(t *testing.T) {
type ModelSuite struct {
suite.Suite
}
func TestModelSuite(t *testing.T) {
suite.Run(t, new(ModelSuite))
}
func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() {
createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC)
data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt})
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if string(data) != `{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}` {
t.Fatalf("Marshal() = %s", data)
}
s.Require().NoError(err)
s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data))
}
+32 -32
View File
@@ -1,42 +1,42 @@
package password
import "testing"
import (
"testing"
func TestHasher(t *testing.T) {
hasher, err := New(testConfig())
if err != nil {
t.Fatalf("New() error = %v", err)
}
hash, err := hasher.Hash("T3st-password-123")
if err != nil {
t.Fatalf("Hash() error = %v", err)
}
"github.com/stretchr/testify/suite"
)
verified, err := hasher.Verify("T3st-password-123", hash)
if err != nil {
t.Fatalf("Verify() error = %v", err)
}
if !verified {
t.Fatal("Verify() = false for the original password")
}
verified, err = hasher.Verify("wrong-password", hash)
if err != nil {
t.Fatalf("Verify() error = %v", err)
}
if verified {
t.Fatal("Verify() = true for a different password")
}
type PasswordSuite struct {
suite.Suite
hasher *Hasher
}
func TestVerifyRejectsMalformedHash(t *testing.T) {
func TestPasswordSuite(t *testing.T) {
suite.Run(t, new(PasswordSuite))
}
func (s *PasswordSuite) SetupTest() {
hasher, err := New(testConfig())
if err != nil {
t.Fatalf("New() error = %v", err)
}
if _, err := hasher.Verify("password", "invalid"); err == nil {
t.Fatal("Verify() error = nil for malformed hash")
}
s.Require().NoError(err)
s.hasher = hasher
}
func (s *PasswordSuite) TestHasher() {
hash, err := s.hasher.Hash("T3st-password-123")
s.Require().NoError(err)
verified, err := s.hasher.Verify("T3st-password-123", hash)
s.Require().NoError(err)
s.True(verified)
verified, err = s.hasher.Verify("wrong-password", hash)
s.Require().NoError(err)
s.False(verified)
}
func (s *PasswordSuite) TestVerifyRejectsMalformedHash() {
_, err := s.hasher.Verify("password", "invalid")
s.Error(err)
}
func testConfig() Config {
+5 -4
View File
@@ -3,8 +3,9 @@ package postgres
import "errors"
var (
ErrConflict = errors.New("conflict")
ErrNotFound = errors.New("not found")
ErrInvalidMembers = errors.New("invalid members")
ErrInvalidCursor = errors.New("invalid cursor")
ErrConflict = errors.New("conflict")
ErrNotFound = errors.New("not found")
ErrInvalidMembers = errors.New("invalid members")
ErrInvalidCursor = errors.New("invalid cursor")
ErrIdempotencyConflict = errors.New("idempotency conflict")
)
+285
View File
@@ -0,0 +1,285 @@
package postgres
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
)
const expiredIdempotencyCleanupBatch = 100
func (s *Store) CreateMessage(
ctx context.Context,
chatID string,
senderID string,
text string,
idempotencyKey string,
idempotencyTTL time.Duration,
) (model.Message, bool, error) {
var (
tx pgx.Tx
err error
visible bool
requestHash [sha256.Size]byte
message model.Message
replay bool
)
tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID)
if err != nil {
return model.Message{}, false, err
}
if !visible {
return model.Message{}, false, ErrNotFound
}
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
return model.Message{}, false, err
}
requestHash = sha256.Sum256([]byte(text))
message, replay, err = createIdempotentMessage(
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
)
if err != nil {
return model.Message{}, false, err
}
if err := tx.Commit(ctx); err != nil {
return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err)
}
return message, replay, nil
}
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM chat_members
WHERE chat_id = $1 AND user_id = $2
)`
var visible bool
if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil {
return false, fmt.Errorf("check chat visibility: %w", err)
}
return visible, nil
}
func createIdempotentMessage(
ctx context.Context,
tx pgx.Tx,
chatID string,
senderID string,
text string,
idempotencyKey string,
requestHash []byte,
idempotencyTTL time.Duration,
) (model.Message, bool, error) {
for {
var (
record idempotencyRecord
found bool
err error
messageID string
reserved bool
message model.Message
)
record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey)
if err != nil {
return model.Message{}, false, err
}
if found && record.active {
if !bytes.Equal(record.requestHash, requestHash) {
return model.Message{}, false, ErrIdempotencyConflict
}
message, err = messageByID(ctx, tx, record.messageID)
return message, true, err
}
if found {
if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil {
return model.Message{}, false, err
}
}
messageID = uuid.NewString()
reserved, err = reserveIdempotencyKey(
ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL,
)
if err != nil {
return model.Message{}, false, err
}
if !reserved {
continue
}
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
return message, false, err
}
}
func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error {
const query = `
WITH expired AS (
SELECT ctid
FROM message_idempotency
WHERE expires_at <= statement_timestamp()
ORDER BY expires_at
LIMIT $1
FOR UPDATE SKIP LOCKED
)
DELETE FROM message_idempotency records
USING expired
WHERE records.ctid = expired.ctid`
if _, err := tx.Exec(ctx, query, limit); err != nil {
return fmt.Errorf("delete expired idempotency records: %w", err)
}
return nil
}
type idempotencyRecord struct {
requestHash []byte
messageID string
active bool
}
func lockIdempotencyRecord(
ctx context.Context,
tx pgx.Tx,
userID string,
chatID string,
key string,
) (idempotencyRecord, bool, error) {
const query = `
SELECT request_hash, message_id, expires_at > statement_timestamp()
FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3
FOR UPDATE`
var (
record idempotencyRecord
err error
)
err = tx.QueryRow(ctx, query, userID, chatID, key).Scan(
&record.requestHash,
&record.messageID,
&record.active,
)
if errors.Is(err, pgx.ErrNoRows) {
return idempotencyRecord{}, false, nil
}
if err != nil {
return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err)
}
return record, true, nil
}
func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error {
const query = `
DELETE FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3`
if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil {
return fmt.Errorf("delete expired idempotency key: %w", err)
}
return nil
}
func reserveIdempotencyKey(
ctx context.Context,
tx pgx.Tx,
userID string,
chatID string,
key string,
requestHash []byte,
messageID string,
ttl time.Duration,
) (bool, error) {
const query = `
INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at)
VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second')
ON CONFLICT (user_id, chat_id, key) DO NOTHING
RETURNING true`
var (
inserted bool
err error
)
err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("reserve idempotency key: %w", err)
}
return inserted, nil
}
func insertMessage(
ctx context.Context,
tx pgx.Tx,
messageID string,
chatID string,
senderID string,
text string,
) (model.Message, error) {
const query = `
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES ($1, $2, $3, $4)
RETURNING id, chat_id, sender_id, text, created_at`
var (
message model.Message
err error
)
err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
)
if err != nil {
return model.Message{}, fmt.Errorf("insert message: %w", err)
}
return message, nil
}
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
const query = `
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE id = $1`
var (
message model.Message
err error
)
err = tx.QueryRow(ctx, query, messageID).Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, fmt.Errorf("idempotency result message not found")
}
if err != nil {
return model.Message{}, fmt.Errorf("get idempotency result message: %w", err)
}
return message, nil
}
+2 -36
View File
@@ -10,7 +10,7 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
)
type Store struct {
@@ -114,7 +114,7 @@ func (s *Store) CreateChat(
if err != nil {
return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
valid, err := allUsersExist(ctx, tx, memberIDs)
if err != nil {
@@ -319,40 +319,6 @@ func (s *Store) visibleChatPositionExists(
return exists, nil
}
func (s *Store) CreateMessage(
ctx context.Context,
chatID string,
senderID string,
text string,
) (model.Message, error) {
const query = `
INSERT INTO messages (id, chat_id, sender_id, text)
SELECT $1, $2, $3, $4
WHERE EXISTS (
SELECT 1
FROM chat_members
WHERE chat_id = $2 AND user_id = $3
)
RETURNING id, chat_id, sender_id, text, created_at`
var message model.Message
err := s.pool.QueryRow(ctx, query, uuid.NewString(), chatID, senderID, text).Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, ErrNotFound
}
if err != nil {
return model.Message{}, fmt.Errorf("create message: %w", err)
}
return message, nil
}
func (s *Store) ListMessages(
ctx context.Context,
chatID string,
+205 -10
View File
@@ -11,14 +11,37 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/postgres"
)
func TestConcurrentUsernameCreation(t *testing.T) {
pool := openTestDatabase(t)
store := postgres.New(pool)
type PostgresSuite struct {
suite.Suite
pool *pgxpool.Pool
store *postgres.Store
}
func TestPostgresSuite(t *testing.T) {
suite.Run(t, new(PostgresSuite))
}
func (s *PostgresSuite) SetupSuite() {
s.pool = openTestDatabase(s.T())
s.store = postgres.New(s.pool)
}
func (s *PostgresSuite) TearDownSuite() {
if s.pool != nil {
s.pool.Close()
}
}
func (s *PostgresSuite) TestConcurrentUsernameCreation() {
t := s.T()
store := s.store
username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16]
const workers = 8
@@ -51,9 +74,10 @@ func TestConcurrentUsernameCreation(t *testing.T) {
}
}
func TestCreateChatRollsBackForUnknownMember(t *testing.T) {
pool := openTestDatabase(t)
store := postgres.New(pool)
func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() {
t := s.T()
pool := s.pool
store := s.store
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
@@ -69,6 +93,175 @@ func TestCreateChatRollsBackForUnknownMember(t *testing.T) {
}
}
func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
t := s.T()
pool := s.pool
store := s.store
owner, chatID := createOwnerAndChat(t, store)
key := uuid.NewString()
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
if err != nil || replay {
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
}
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
if err != nil || !replay {
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
}
if first != second {
t.Fatalf("replay message = %v, want %v", second, first)
}
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
}
if count := messageCount(t, pool, chatID); count != 1 {
t.Fatalf("message count = %d, want 1", count)
}
}
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
t := s.T()
pool := s.pool
stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)}
owner, chatID := createOwnerAndChat(t, stores[0])
key := uuid.NewString()
const workers = 8
type result struct {
message model.Message
replay bool
err error
}
results := make(chan result, workers)
var group sync.WaitGroup
for worker := range workers {
group.Add(1)
go func() {
defer group.Done()
message, replay, err := stores[worker%len(stores)].CreateMessage(
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
)
results <- result{message: message, replay: replay, err: err}
}()
}
group.Wait()
close(results)
var messageID string
var created int
for result := range results {
if result.err != nil {
t.Fatalf("CreateMessage() error = %v", result.err)
}
if messageID == "" {
messageID = result.message.ID
}
if result.message.ID != messageID {
t.Fatalf("message ID = %s, want %s", result.message.ID, messageID)
}
if !result.replay {
created++
}
}
if created != 1 {
t.Fatalf("new message results = %d, want 1", created)
}
if count := messageCount(t, pool, chatID); count != 1 {
t.Fatalf("message count = %d, want 1", count)
}
}
func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
t := s.T()
pool := s.pool
store := s.store
owner, chatID := createOwnerAndChat(t, store)
second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
}
chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID)
if err != nil {
t.Fatalf("CreateChat() error = %v", err)
}
key := uuid.NewString()
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
t.Fatalf("owner CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
t.Fatalf("second user CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
t.Fatalf("other chat CreateMessage() error = %v", err)
}
if _, err := pool.Exec(context.Background(), `
UPDATE message_idempotency
SET created_at = clock_timestamp() - INTERVAL '2 days',
expires_at = clock_timestamp() - INTERVAL '1 day'
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
t.Fatalf("expire idempotency key: %v", err)
}
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
if err != nil || replay || message.Text != "after expiry" {
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
}
}
func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
t := s.T()
owner, chatID := createOwnerAndChat(t, s.store)
expiredKey := uuid.NewString()
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
t.Fatalf("create expired message: %v", err)
}
if _, err := s.pool.Exec(context.Background(), `
UPDATE message_idempotency
SET created_at = clock_timestamp() - INTERVAL '2 days',
expires_at = clock_timestamp() - INTERVAL '1 day'
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil {
t.Fatalf("expire idempotency key: %v", err)
}
if _, _, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
); err != nil {
t.Fatalf("create new message: %v", err)
}
var remaining int
if err := s.pool.QueryRow(context.Background(), `
SELECT count(*)
FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil {
t.Fatalf("count expired idempotency records: %v", err)
}
if remaining != 0 {
t.Fatalf("expired idempotency records = %d, want 0", remaining)
}
}
func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) {
t.Helper()
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
}
chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID)
if err != nil {
t.Fatalf("CreateChat() error = %v", err)
}
return owner, chat.ID
}
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
t.Helper()
var count int
if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil {
t.Fatalf("count messages: %v", err)
}
return count
}
func openTestDatabase(t *testing.T) *pgxpool.Pool {
t.Helper()
url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE")
@@ -76,12 +269,14 @@ func openTestDatabase(t *testing.T) *pgxpool.Pool {
URL: url, MinConnections: 0, MaxConnections: 10,
MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute,
HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second,
MigrationTimeout: 30 * time.Second,
})
if err != nil {
t.Fatalf("database.Open() error = %v", err)
}
t.Cleanup(pool.Close)
if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil {
pool.Close()
t.Fatalf("database.Migrate() error = %v", err)
}
return pool
}
+47 -58
View File
@@ -9,75 +9,64 @@ import (
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab2/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab3/internal/session"
)
func TestSessionLifecycle(t *testing.T) {
store := openTestStore(t)
userID := uuid.NewString()
token, expiresAt, err := store.Create(context.Background(), userID, time.Minute)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if token == "" || !expiresAt.After(time.Now()) {
t.Fatalf("invalid token or expiry: token length = %d, expires_at = %v", len(token), expiresAt)
}
actualUserID, err := store.UserID(context.Background(), token)
if err != nil {
t.Fatalf("UserID() error = %v", err)
}
if actualUserID != userID {
t.Fatalf("UserID() = %q, want %q", actualUserID, userID)
}
if err := store.Delete(context.Background(), token); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if _, err := store.UserID(context.Background(), token); !errors.Is(err, session.ErrNotFound) {
t.Fatalf("UserID() after delete error = %v, want ErrNotFound", err)
}
type SessionSuite struct {
suite.Suite
store *session.Store
}
func TestSessionExpires(t *testing.T) {
store := openTestStore(t)
token, _, err := store.Create(context.Background(), uuid.NewString(), time.Second)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
_, err := store.UserID(context.Background(), token)
if errors.Is(err, session.ErrNotFound) {
return
}
if err != nil {
t.Fatalf("UserID() error = %v", err)
}
time.Sleep(50 * time.Millisecond)
}
t.Fatal("session did not expire within three seconds")
func TestSessionSuite(t *testing.T) {
suite.Run(t, new(SessionSuite))
}
func openTestStore(t *testing.T) *session.Store {
t.Helper()
url := strings.TrimSpace(os.Getenv("TEST_CACHE_URL"))
if url == "" {
func (s *SessionSuite) SetupSuite() {
rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL"))
if rawURL == "" {
path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE"))
if path == "" {
t.Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests")
s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read TEST_CACHE_URL_FILE: %v", err)
}
url = strings.TrimSpace(string(data))
s.Require().NoError(err)
rawURL = strings.TrimSpace(string(data))
}
store, err := session.New(url)
if err != nil {
t.Fatalf("session.New() error = %v", err)
store, err := session.New(rawURL)
s.Require().NoError(err)
s.store = store
}
func (s *SessionSuite) TearDownSuite() {
if s.store != nil {
s.Require().NoError(s.store.Close())
}
t.Cleanup(func() { _ = store.Close() })
return store
}
func (s *SessionSuite) TestSessionLifecycle() {
userID := uuid.NewString()
token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute)
s.Require().NoError(err)
s.NotEmpty(token)
s.True(expiresAt.After(time.Now()))
actualUserID, err := s.store.UserID(context.Background(), token)
s.Require().NoError(err)
s.Equal(userID, actualUserID)
s.Require().NoError(s.store.Delete(context.Background(), token))
_, err = s.store.UserID(context.Background(), token)
s.ErrorIs(err, session.ErrNotFound)
}
func (s *SessionSuite) TestSessionExpires() {
token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second)
s.Require().NoError(err)
s.Eventually(func() bool {
_, err = s.store.UserID(context.Background(), token)
return errors.Is(err, session.ErrNotFound)
}, 3*time.Second, 50*time.Millisecond)
}
+23 -5
View File
@@ -8,10 +8,14 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
# Время жизни сессии и используемое хранилище.
# Время жизни сессии, идемпотентности и используемое хранилище.
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
SESSION_STORE=valkey
# Число API-реплик за Nginx.
API_REPLICAS=2
# Учётные данные локального PostgreSQL и строка подключения API.
POSTGRES_USER=messenger
POSTGRES_DB=messenger
@@ -37,7 +41,7 @@ HTTP_READ_HEADER_TIMEOUT_SECONDS=5
HTTP_READ_TIMEOUT_SECONDS=10
HTTP_WRITE_TIMEOUT_SECONDS=10
HTTP_IDLE_TIMEOUT_SECONDS=60
HTTP_SHUTDOWN_TIMEOUT_SECONDS=10
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
# Параметры Argon2id для новых хешей паролей.
PASSWORD_MEMORY_KIB=65536
@@ -49,9 +53,23 @@ PASSWORD_KEY_LENGTH=32
# Порт Swagger внутри конфигурации CORS API.
SWAGGER_PORT=8081
# Ограничения ресурсов контейнеров приложения и хранилищ.
API_CPUS=2
API_MEMORY=512m
# Ограничения ресурсов контейнеров.
API_CPUS=1
API_MEMORY=256m
MIGRATE_CPUS=0.5
MIGRATE_MEMORY=128m
NGINX_CPUS=0.5
NGINX_MEMORY=128m
SWAGGER_CPUS=0.25
SWAGGER_MEMORY=128m
TEST_CPUS=2
TEST_MEMORY=512m
EXPORTER_CPUS=0.25
EXPORTER_MEMORY=128m
PROMETHEUS_CPUS=1
PROMETHEUS_MEMORY=512m
GRAFANA_CPUS=0.5
GRAFANA_MEMORY=256m
POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
+19
View File
@@ -0,0 +1,19 @@
CREATE TABLE message_idempotency (
user_id UUID NOT NULL,
chat_id UUID NOT NULL,
key TEXT NOT NULL,
request_hash BYTEA NOT NULL,
message_id UUID NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
expires_at TIMESTAMPTZ NOT NULL,
CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key),
CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE,
CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id)
ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32),
CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at)
);
CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at);
+47 -24
View File
@@ -6,58 +6,81 @@
{
"type": "timeseries", "title": "API requests per second", "id": 1,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "RPS", "refId": "A"}]
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}]
},
{
"type": "timeseries", "title": "API responses by status class", "id": 2,
"type": "timeseries", "title": "API RPS by replica", "id": 2,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API responses by status class", "id": 3,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API latency p95", "id": 3,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"type": "timeseries", "title": "Healthy API replicas", "id": 4,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}]
},
{
"type": "timeseries", "title": "API latency p95 by replica and route", "id": 5,
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 16},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{route}}", "refId": "A"}]
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API in flight", "id": 4,
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 8},
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "requests", "refId": "A"}]
"type": "timeseries", "title": "API in flight", "id": 6,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Go heap", "id": 5,
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 8},
"type": "timeseries", "title": "Go heap by replica", "id": 7,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "heap", "refId": "A"}]
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "PostgreSQL connections", "id": 6,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16},
"type": "timeseries", "title": "Idempotency replays", "id": 8,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
"targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}]
},
{
"type": "timeseries", "title": "Nginx connections and requests", "id": 9,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
"targets": [
{"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"},
{"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL connections", "id": 10,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
"targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}]
},
{
"type": "timeseries", "title": "PostgreSQL transactions", "id": 7,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16},
"type": "timeseries", "title": "PostgreSQL transactions", "id": 11,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 40},
"targets": [
{"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"},
{"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL database size", "id": 8,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16},
"type": "timeseries", "title": "PostgreSQL database size", "id": 12,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 40},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey memory", "id": 9,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 24},
"type": "timeseries", "title": "Valkey memory", "id": 13,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 48},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey keys and commands", "id": 10,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 24},
"type": "timeseries", "title": "Valkey keys and commands", "id": 14,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 48},
"targets": [
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
@@ -66,11 +89,11 @@
],
"refresh": "5s",
"schemaVersion": 41,
"tags": ["highload", "lab2"],
"tags": ["highload", "lab3", "nginx", "failover"],
"templating": {"list": []},
"time": {"from": "now-15m", "to": "now"},
"timezone": "browser",
"title": "Messenger overview",
"title": "Messenger lab3 overview",
"uid": "messenger-overview",
"version": 1
"version": 2
}
+8 -1
View File
@@ -4,8 +4,15 @@ global:
scrape_configs:
- job_name: messenger
dns_sd_configs:
- names: ["api"]
type: A
port: 8080
refresh_interval: 2s
- job_name: nginx
static_configs:
- targets: ["api:8080"]
- targets: ["nginx-exporter:9113"]
- job_name: postgres
static_configs:
+55
View File
@@ -0,0 +1,55 @@
user nginx;
worker_processes auto;
error_log /dev/stderr notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
resolver 127.0.0.11 valid=2s ipv6=off;
log_format structured escape=json
'{"time":"$time_iso8601","request_id":"$http_x_request_id",'
'"method":"$request_method","uri":"$request_uri","status":$status,'
'"request_time":$request_time,"upstream":"$upstream_addr",'
'"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}';
access_log /dev/stdout structured;
upstream messenger {
zone messenger 64k;
server api:8080 resolve max_fails=1 fail_timeout=3s;
keepalive 32;
}
server {
listen 8080;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 3s;
}
}
server {
listen 8081;
access_log off;
location = /stub_status {
stub_status;
}
}
}
+4 -4
View File
@@ -2,7 +2,7 @@
set -eu
docker compose restart postgres valkey
docker compose up -d --wait --wait-timeout 90 postgres valkey
docker compose restart api
docker compose up -d --wait --wait-timeout 90 api
docker compose stop nginx api valkey postgres
docker compose start postgres valkey
docker compose run --rm migrate
docker compose up -d --wait --wait-timeout 90
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
set -eu
wait_for_replicas() {
expected=$1
deadline=$(($(date +%s) + 90))
while [ "$(date +%s)" -lt "$deadline" ]; do
containers=$(docker compose ps -q api)
count=0
healthy=0
for container in $containers; do
count=$((count + 1))
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
if [ "$status" = "healthy" ]; then
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
return 0
fi
sleep 1
done
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
return 1
}
old_containers=$(docker compose ps -q api)
old_count=0
for container in $old_containers; do
old_count=$((old_count + 1))
done
docker compose build
docker compose up -d postgres valkey
docker compose run --rm migrate
if [ "$old_count" -lt 2 ]; then
docker compose up -d --scale api=2 --wait --wait-timeout 120
exit 0
fi
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
for container in $old_containers; do
docker stop --time 6 "$container" >/dev/null
docker rm "$container" >/dev/null
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
done
docker compose up -d --no-deps --no-recreate --scale api=2 api
wait_for_replicas 2
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
+104
View File
@@ -0,0 +1,104 @@
#!/bin/sh
set -eu
duration=${DURATION:-15s}
warmup=${WARMUP:-3s}
concurrencies=${CONCURRENCIES:-"8 32 64"}
repeats=${REPEATS:-3}
current_project=
restore_default_stack() {
if [ -n "$current_project" ]; then
env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true
fi
docker compose up -d --scale api=2 --wait --wait-timeout 120
}
run_configuration() {
series=$1
replicas=$2
api_cpus=$3
api_memory=$4
db_max_conns=$5
repeat=$6
point_concurrencies=$7
project="lab3exp_${series}_r${replicas}_n${repeat}"
pending=false
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
if [ ! -s "evidence/lab3-${label}.json" ]; then
pending=true
fi
done
if [ "$pending" = false ]; then
echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}"
return
fi
current_project=$project
compose() {
env \
COMPOSE_PROJECT_NAME="$project" \
API_REPLICAS="$replicas" \
API_CPUS="$api_cpus" \
API_MEMORY="$api_memory" \
DB_MIN_CONNS=0 \
DB_MAX_CONNS="$db_max_conns" \
docker compose "$@"
}
compose up -d --scale api="$replicas" --wait --wait-timeout 120 \
postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
output="lab3-${label}.json"
if [ -s "evidence/$output" ]; then
echo "Skipping completed point $label"
continue
fi
compose --profile research run --rm experiment \
/app/lab3experiment \
-base-url=http://nginx:8080 \
-prometheus-url=http://prometheus:9090 \
-label="$label" \
-duration="$duration" \
-warmup="$warmup" \
-concurrency="$concurrency" \
-replicas="$replicas" \
-api-cpus="$api_cpus" \
-api-memory="$api_memory" \
-db-max-conns="$db_max_conns" \
-output="/output/$output"
done
compose --profile research down -v --remove-orphans
current_project=
}
trap restore_default_stack EXIT INT TERM
docker compose build api nginx swagger prometheus grafana
docker compose stop
repeat=1
while [ "$repeat" -le "$repeats" ]; do
run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies"
run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies"
run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies"
run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies"
run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64"
run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64"
repeat=$((repeat + 1))
done
docker compose --profile research run --no-deps --rm experiment \
/app/lab3summary \
-input=/output \
-output=/output/lab3-scale-results.json \
-chart=/output/lab3-scale-rps.svg
rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for container in $(docker compose ps -q api); do
instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 6 "$container" >/dev/null
exit 0
fi
done
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
exit 1