ЛР 4 — БВТ2303 — Пластинин — на 5 #1

Closed
dizi wants to merge 16 commits from dizi/mtusi2026lab4:solution/bvt2303-plastinin into main
73 changed files with 8812 additions and 21 deletions
+1
View File
@@ -17,4 +17,5 @@ backups
__pycache__
node_modules
evidence
/messenger
.DS_Store
+93
View File
@@ -0,0 +1,93 @@
name: CI
on:
pull_request:
branches: [main]
push:
branches: [main]
workflow_dispatch:
concurrency:
group: ci-${{ github.repository }}-${{ github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
COMPOSE_PROJECT_NAME: mtusi2026lab4-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
git init .
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
git fetch --depth=1 origin ${{ github.sha }}
git checkout --detach FETCH_HEAD
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Check contract
run: make check
- name: Validate OpenAPI
run: |
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
- name: Lint
uses: golangci/golangci-lint-action@v7
with:
version: v2.13.2
args: --timeout=5m
skip-cache: true
- name: Unit tests and build
run: |
go test -race ./...
go build ./cmd/...
- name: Prepare local stack
run: |
cp lab4.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
- name: Integration checks
run: |
set -a
. ./.env
set +a
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
make test BASE_URL=http://nginx:8080
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
docker compose up -d --scale api=2 --wait --wait-timeout 90
- name: Container logs
if: failure()
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
- name: Stop local stack
if: always()
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
+192
View File
@@ -0,0 +1,192 @@
name: Deploy production
on:
workflow_dispatch:
concurrency:
group: production
cancel-in-progress: false
jobs:
verify:
name: Verify selected commit
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 45
outputs:
api_image: ${{ steps.publish.outputs.api_image }}
nginx_image: ${{ steps.publish.outputs.nginx_image }}
swagger_image: ${{ steps.publish.outputs.swagger_image }}
prometheus_image: ${{ steps.publish.outputs.prometheus_image }}
grafana_image: ${{ steps.publish.outputs.grafana_image }}
s3_image: ${{ steps.publish.outputs.s3_image }}
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
env:
COMPOSE_PROJECT_NAME: mtusi2026lab4-deploy-${{ github.run_id }}
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-api:${{ github.sha }}
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-nginx:${{ github.sha }}
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-swagger:${{ github.sha }}
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-prometheus:${{ github.sha }}
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-grafana:${{ github.sha }}
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio:${{ github.sha }}
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio-client:${{ github.sha }}
steps:
- name: Checkout exact commit
run: |
git init .
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
git fetch --depth=1 origin ${{ github.sha }}
git checkout --detach FETCH_HEAD
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Check contract
run: make check
- name: Validate OpenAPI
run: |
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
- name: Lint
uses: golangci/golangci-lint-action@v7
with:
version: v2.13.2
args: --timeout=5m
skip-cache: true
- name: Unit tests and build
run: |
go test -race ./...
go build ./cmd/...
- name: Prepare local stack
run: |
cp lab4.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
- name: Integration checks
run: |
set -a
. ./.env
set +a
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
make test BASE_URL=http://nginx:8080
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
- name: Publish verified images
id: publish
env:
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
run: |
printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin
publish() {
output=$1
image=$2
push_output=$(docker push "$image")
printf '%s\n' "$push_output"
digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1)
test -n "$digest"
repository=${image%:*}
printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT"
}
publish api_image "$API_IMAGE"
publish nginx_image "$NGINX_IMAGE"
publish swagger_image "$SWAGGER_IMAGE"
publish prometheus_image "$PROMETHEUS_IMAGE"
publish grafana_image "$GRAFANA_IMAGE"
publish s3_image "$S3_IMAGE"
publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE"
- name: Container logs
if: failure()
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
- name: Stop local stack
if: always()
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
deploy:
name: Deploy selected commit
needs: verify
runs-on: ubuntu-latest
environment: production
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
DEPLOY_SHA: ${{ github.sha }}
steps:
- name: Configure SSH
run: |
umask 077
mkdir -p ~/.ssh
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
- name: Prepare immutable image manifest
run: |
cat > deploy-images.env <<EOF
API_IMAGE=${{ needs.verify.outputs.api_image }}
NGINX_IMAGE=${{ needs.verify.outputs.nginx_image }}
SWAGGER_IMAGE=${{ needs.verify.outputs.swagger_image }}
PROMETHEUS_IMAGE=${{ needs.verify.outputs.prometheus_image }}
GRAFANA_IMAGE=${{ needs.verify.outputs.grafana_image }}
S3_IMAGE=${{ needs.verify.outputs.s3_image }}
S3_BOOTSTRAP_IMAGE=${{ needs.verify.outputs.s3_bootstrap_image }}
EOF
- name: Authenticate registry on deployment host
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
- name: Upload image manifest
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab4-${{ github.run_id }}.env"
- name: Deploy and verify
run: |
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
set -eu
cd ~/mtusi2026lab4
git fetch origin main
git cat-file -e '$DEPLOY_SHA^{commit}'
git reset --hard '$DEPLOY_SHA'
manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env'
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
rm -f \"\$manifest\"
else
status=\$?
rm -f \"\$manifest\"
exit \"\$status\"
fi
test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA'
docker compose ps
"
+1
View File
@@ -22,6 +22,7 @@ dist/
build/
target/
bin/
/messenger
obj/
coverage/
*.log
+44
View File
@@ -0,0 +1,44 @@
FROM golang:1.26.5-alpine3.24 AS build
WORKDIR /src
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -p=1 -o /out/messenger ./cmd/messenger
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
FROM build AS test
CMD ["go", "test", "./internal/postgres", "./internal/session"]
FROM swaggerapi/swagger-ui:v5.31.2 AS swagger
COPY contracts/openapi.json /spec/openapi.json
ENV SWAGGER_JSON=/spec/openapi.json
FROM prom/prometheus:v3.13.3 AS prometheus
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
FROM grafana/grafana:13.1.6 AS grafana
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
FROM nginx:1.30.5-alpine AS loadbalancer
COPY nginx/nginx.conf /etc/nginx/nginx.conf
COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf
FROM alpine:3.24.2
RUN addgroup -S app && adduser -S -G app -u 10001 app
COPY --from=build --chown=app:app /out/messenger /app/messenger
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
USER app
WORKDIR /app
HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1
CMD ["/app/messenger"]
+165 -21
View File
@@ -1,37 +1,181 @@
# Отчёт — лабораторная 4
# Отчёт — лабораторная работа № 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: Пластинин Денис, БВТ2303.
- Целевая оценка: 5.
- Выбранный трек: 2, согласованность PostgreSQL и S3.
- Ветка: `solution/bvt2303-plastinin`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Engine с Compose V2, Git и свободные локальные порты 8080, 8081, 9090 и 3000. Для запуска Go-проверок вне контейнера нужен Go 1.26. Образы приложения и MinIO собираются из исходников внутри Docker. Версии Go, PostgreSQL, Valkey, Nginx, MinIO и остальных компонентов закреплены в Dockerfile и `compose.yaml`.
## Архитектура и решения
```sh
cp lab4.env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
После запуска доступны:
- API через Nginx: `http://localhost:8080`;
- Swagger UI: `http://localhost:8081`;
- Prometheus: `http://localhost:9090`;
- Grafana: `http://localhost:3000`, пользователь `admin`, пароль из `GRAFANA_ADMIN_PASSWORD`.
Публичные и внутренние проверки:
```sh
make check
make test BASE_URL=http://localhost:8080
docker compose --profile tests run --rm --build tests
go test -race ./...
go vet ./...
```
Сверка PostgreSQL и S3 выполняется внутри запущенной API-реплики. Без `--apply` команда ничего не удаляет:
```sh
docker compose exec -T api /app/messenger reconcile-attachments --grace-period=1h
docker compose exec -T api /app/messenger reconcile-attachments --apply --grace-period=1h
```
Эксперименты запускаются на чистом стенде:
```sh
docker compose exec -T api /app/lab4experiment memory
docker compose exec -T api /app/lab4experiment consistency
```
Безопасная остановка сохраняет данные в Docker volumes:
```sh
docker compose down
```
Полное удаление учебных данных выполняется только явно:
```sh
docker compose down -v
```
## Архитектура
```text
клиент
|
v
Nginx :8080
|
+---- API replica 1 ----+
| |
+---- API replica 2 ----+---- PostgreSQL: пользователи, чаты,
| сообщения, metadata
+---- Valkey: сессии
+---- MinIO: приватные байты изображений
Prometheus <---- API /metrics, MinIO, Nginx, PostgreSQL, Valkey
|
+---- Grafana
```
Из лабораторной № 3 сохранены две и более API-реплики, Nginx, общие PostgreSQL и Valkey, Bearer-сессии, keyset pagination, идемпотентность сообщений, graceful shutdown, failover, метрики и управляемое развёртывание. В лабораторной № 4 добавлены приватное объектное хранилище, модель вложения, бинарная загрузка и скачивание, привязка вложений к сообщениям и сверка двух хранилищ.
Код разделён по ответственности. `internal/messenger/httpapi` содержит HTTP-маршруты, middleware и преобразование ошибок в ответы. `internal/messenger/attachment` владеет сценарием загрузки и чтения вложений. `internal/postgres`, `internal/objectstore` и `internal/session` являются адаптерами PostgreSQL, S3 и Valkey. `internal/reconcile` выполняет сверку хранилищ, а `internal/bootstrap` собирает зависимости для сервера, миграций и служебной команды. HTTP-слой зависит от узких интерфейсов и общих ошибок модели, поэтому детали PostgreSQL и S3 в handlers не попадают.
PostgreSQL остаётся источником истины для прав доступа и метаданных. Таблица `attachments` хранит владельца, чат, отображаемое имя, MIME, размер, SHA-256 и внутренний object key. Таблица `message_attachments` задаёт упорядоченную связь многие ко многим с сообщениями. Object key генерируется сервером и не зависит от имени файла.
MinIO хранит только байты оригиналов. Bucket закрыт для анонимного доступа. Bootstrap повторяемо создаёт bucket, policy и отдельного пользователя приложения. API получает только действия list/get/put/delete в одном bucket. Root credentials используются bootstrap-контейнером и не передаются API. Web-консоль MinIO отключена и её порт не опубликован.
## Загрузка и чтение вложения
Клиент передаёт сырые байты PNG или JPEG. Сервер ограничивает HTTP body до 10 MiB до полного чтения, проверяет заявленный и фактический MIME, декодирует конфигурацию изображения, ограничивает число пикселей 40 миллионами и проверяет полную декодируемость. Filename содержит от 1 до 255 Unicode-символов и не может содержать разделители пути, NUL, управляющие символы или состоять только из пробелов.
При успешной загрузке сервер генерирует UUID для вложения и отдельный случайный object key, считает SHA-256 и создаёт в PostgreSQL строку со статусом `pending`. Затем он записывает объект в MinIO и переводит строку в `ready`. Только после этого возвращается `201`. При отказе S3 строка `pending` удаляется. Если результат перевода в `ready` неоднозначен, сервер повторно читает строку по attachment ID и object key: уже сохранённый `ready` считается успехом, а подтверждённый `pending` компенсируется удалением объекта и metadata.
Метаданные и содержимое читаются только после проверки Bearer-сессии и членства пользователя в чате. Посторонний пользователь и неизвестный идентификатор дают одинаковый `404`. API отдаёт байты самостоятельно без redirect и возвращает сохранённый `Content-Type` и точный `Content-Length`.
Сообщение принимает до десяти уникальных `attachment_ids`. В одной транзакции PostgreSQL проверяются членство отправителя и принадлежность каждого вложения нужному чату, создаются сообщение, связи и запись идемпотентности. Отсутствующее поле `attachment_ids` и пустой массив дают одинаковый семантический hash. Пустой текст допустим только при наличии вложения.
## Согласованность PostgreSQL и S3
Операции PostgreSQL и S3 не входят в общую распределённую транзакцию. Возможны два вида расхождений:
1. object существует в S3, но metadata отсутствует;
2. metadata существует, но object отсутствует.
Команда `reconcile-attachments` проходит оба набора в порядке object key постранично, поэтому объём памяти ограничен размером двух страниц. По умолчанию включён dry-run. Grace period защищает свежие объекты и строки, которые могут относиться к ещё выполняющейся операции.
В режиме `--apply` старый orphan в S3 удаляется после повторной проверки object key в PostgreSQL. Старые `pending` и готовые, но неиспользуемые вложения сначала атомарно переводятся в `deleting` под блокировкой строки. Под той же блокировкой повторно проверяются ожидаемый статус, возраст и отсутствие ссылки из `message_attachments`. После удаления объекта строка удаляется только в статусе `deleting`. Если upload успел изменить `pending` на `ready`, устаревший кандидат не захватывается. Используемая строка с отсутствующим объектом сохраняется и попадает в отчёт для ручного восстановления. Повторный запуск продолжает незавершённый `deleting` и не повторяет уже выполненные удаления.
## Ограничение памяти
До чтения body weighted semaphore резервирует полный лимит одной загрузки — 10 MiB. Поэтому отсутствующий или заниженный `Content-Length` не позволяет одновременно накопить неограниченное число буферов. После чтения дополнительно резервируется восемь байт на пиксель перед полной декодировкой. Бюджеты одной реплики разделены: 64 MiB для исходных body и 384 MiB для декодирования; лимит контейнера равен 512 MiB. Проверки размеров выполняются до умножения, чтобы исключить integer overflow. Если одного из бюджетов недостаточно, API отвечает `429` с `Retry-After: 1`.
Go-инструмент `cmd/lab4experiment` создаёт случайное JPEG-изображение и выполняет четыре волны по 12 одновременных загрузок. Передача каждой копии ограничена по скорости, чтобы нагрузка длилась дольше интервала scrape. Наблюдаемый пик heap и resident memory двух API-процессов читается из Prometheus с интервалом одна секунда. Это измерение локального стенда, а не универсальная оценка производительности S3.
В эксперименте успешно завершились все 48 загрузок файла размером 3 759 914 байт. Четыре волны заняли 12,974 секунды. После трёх секунд стабилизации суммарный heap двух реплик вырос с 153 985 824 до 202 505 208 байт, то есть на 48 519 384 байта. Суммарный RSS вырос с 335 298 560 до 345 976 832 байт, то есть на 10 678 272 байта. Наблюдаемый пик остался ниже общего контейнерного лимита двух реплик 1 GiB.
## Наблюдаемость
API экспортирует общие HTTP-метрики и отдельные показатели загрузок:
- `messenger_attachment_uploads_total{result}`;
- `messenger_attachment_upload_duration_seconds`;
- `messenger_attachment_upload_bytes_total`.
Prometheus обнаруживает все API-реплики через Docker DNS и отдельно собирает MinIO, Nginx, PostgreSQL и Valkey. Grafana показывает частоту и p95 загрузок, объём принятых байтов, число и ёмкость объектов MinIO, ошибки S3, доступность компонентов, RPS, классы HTTP-ответов и память Go.
`/health/live` проверяет только процесс. `/health/ready` требует доступных PostgreSQL, Valkey и приватного bucket S3. Недоступность Grafana или Prometheus не останавливает API.
## CI/CD и blue/green deploy
CI проверяет contract, OpenAPI, форматирование, lint, unit и integration tests, persistence и отказ одной API-реплики. После успешных проверок образы публикуются в Gitea Container Registry. В manifest передаются ссылки вида `repository@sha256:digest`, поэтому сервер запускает те же байты, которые прошли проверки.
Production workflow запускается вручную и использует защищённое environment `production`. Перед переключением применяются только обратно совместимые миграции. Скрипт поднимает новый пул API под отдельными DNS-именами и ждёт readiness каждой реплики. Nginx сначала проверяет новую конфигурацию через `nginx -t`, затем атомарно выполняет reload upstream. Старый пул остаётся запущенным до успешного smoke-check нового и только после этого останавливается. При ошибке до завершения переключения восстанавливается предыдущий upstream, а candidate-контейнеры удаляются.
Сам балансировщик относится к стабильному control plane. Если его verified digest изменился, обычный API deploy завершается до переключения и требует отдельного обслуживания Nginx. Это не подменяет новый образ старым и не создаёт скрытое окно недоступности внутри blue/green обновления API.
Локальная проверка механизма подняла две отдельные candidate-реплики, переключила upstream и выполнила 20 запросов `/health/live`: все ответы имели статус `200`, а `X-Instance-Id` чередовался только между двумя контейнерами нового пула.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда или сценарий | Результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт и OpenAPI | `make check`, `make validate` | offline check PASS; validate входит в CI | `evidence/lab4-verification.txt` |
| Unit и integration | `go test ./...`, race/tests profile в CI | unit PASS | `evidence/lab4-verification.txt` |
| Upload/download и ACL | `make test` | PASS | `evidence/lab4-verification.txt` |
| Межрепличный failover и restart | public resilience hooks | проверяются CI точного commit | `.gitea/workflows/ci.yml` |
| Приватный bucket и повторный bootstrap | anonymous HTTP check, два запуска bootstrap | PASS | `evidence/lab4-verification.txt` |
| 12 конкурентных загрузок и память | `lab4experiment memory` | 48/48 PASS, observed RSS peak 345 976 832 B | `evidence/lab4-memory.json` |
| Трек 2: dry-run/apply/repeat | `lab4experiment consistency` | PASS, повторный apply: 0 удалений | `evidence/lab4-consistency.json` |
| Blue/green API switch | два candidate-контейнера, `nginx -t`, reload, 20 health-запросов | 20/20 HTTP 200, только новый пул | `evidence/lab4-verification.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперимент согласованности
## Эксперименты и отказы
Гипотеза: постраничная сверка с grace period может автоматически удалить старые безопасные расхождения, не удаляя свежие данные и metadata используемого вложения; второй apply будет идемпотентным.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Методика на чистом стенде:
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
1. создать согласованное вложение;
2. удалить S3 object у непривязанного вложения;
3. привязать другое вложение к сообщению и удалить его S3 object;
4. создать object без metadata;
5. создать незавершённое вложение `pending`;
6. выполнить apply с grace period один час и проверить сохранность свежих аномалий;
7. состарить `pending`, выполнить dry-run с нулевым grace period;
8. выполнить apply и проверить удаление orphan, непривязанной строки и старого `pending`;
9. повторить apply и проверить отсутствие новых действий.
## Ограничения и следующие шаги
Фактическая стоимость обхода записывается как число просмотренных объектов и строк, размер страницы и wall time. Алгоритм выполняет линейный merge двух отсортированных потоков; дополнительные запросы проверки ссылок нужны только для metadata с отсутствующим объектом.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
При размере страницы 10 фактический запуск пересёк границу страницы. Apply с часовым grace period просмотрел 14 строк metadata и 13 objects за 4 миллисекунды, сохранил свежие orphan, missing metadata и `pending`. После искусственного старения dry-run за 5 миллисекунд обнаружил один старый orphan, одну непривязанную строку без объекта, одну используемую строку без объекта и один старый `pending`. Apply за 13 миллисекунд удалил object и metadata старого `pending`, orphan object и непривязанную missing metadata. Повторный apply занял 1 миллисекунду и выполнил ноль удалений; используемая строка осталась в PostgreSQL.
## Ограничения
- Между PostgreSQL и S3 нет общей транзакции; lifecycle `pending/ready/deleting`, компенсация и reconciler обеспечивают eventual consistency.
- Удалённый object используемого вложения автоматически не восстанавливается: строка сохраняется и требует восстановления байтов из резервной копии либо решения оператора.
- Grace period основан на времени PostgreSQL и S3; для удалённого стенда требуется синхронизация часов.
- Admission control действует отдельно в каждой API-реплике; при изменении контейнерного лимита нужно совместно настраивать `ATTACHMENT_BODY_MEMORY_BYTES` и `ATTACHMENT_DECODE_MEMORY_BYTES`.
- Автоматический rollback переключает API обратно на предыдущий пул, но не откатывает уже применённые миграции и stateful-хранилища. Поэтому миграции должны оставаться обратно совместимыми как минимум с предыдущей версией приложения.
- MinIO собирается из официального закреплённого source tag, поскольку готовые community images больше не публикуются. Это увеличивает время первого CI build.
- Локальный HTTP допустим только для учебного стенда с синтетическими данными; удалённое развёртывание требует TLS.
+330
View File
@@ -0,0 +1,330 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"sync"
"time"
"github.com/google/uuid"
)
type apiClient struct {
baseURL string
client *http.Client
token string
userID string
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println("PASS: replica idempotency and dependency recovery")
}
func run() error {
baseURL := flag.String("base-url", "http://localhost:8080", "API base URL")
flag.Parse()
client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}}
if err := client.registerAndLogin(); err != nil {
return err
}
if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil {
return err
}
if err := client.verifyConcurrentIdempotency(); err != nil {
return err
}
if err := verifyOutage(client, "valkey"); err != nil {
return err
}
return verifyOutage(client, "postgres")
}
func (c *apiClient) registerAndLogin() error {
username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16]
password := "T3st-" + uuid.NewString()
body := map[string]string{"username": username, "display_name": "Dependency check", "password": password}
payload, err := json.Marshal(body)
if err != nil {
return err
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("register: %w", err)
}
registrationBody := response.Body
defer func() { _ = registrationBody.Close() }()
if response.StatusCode != http.StatusCreated {
return fmt.Errorf("register status = %d", response.StatusCode)
}
var user struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil {
return fmt.Errorf("decode user: %w", err)
}
if user.ID == "" {
return fmt.Errorf("register returned an empty user id")
}
c.userID = user.ID
request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil)
if err != nil {
return err
}
request.SetBasicAuth(username, password)
response, err = c.client.Do(request)
if err != nil {
return fmt.Errorf("login: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return fmt.Errorf("login status = %d", response.StatusCode)
}
var session struct {
Token string `json:"token"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
return fmt.Errorf("decode session: %w", err)
}
if session.Token == "" {
return fmt.Errorf("login returned an empty token")
}
c.token = session.Token
return nil
}
type messageResult struct {
messageID string
instanceID string
err error
}
func (c *apiClient) verifyConcurrentIdempotency() error {
chatID, err := c.createChat()
if err != nil {
return err
}
const workers = 16
var (
key = uuid.NewString()
start = make(chan struct{})
results = make(chan messageResult, workers)
group sync.WaitGroup
instances = make(map[string]struct{})
messageID string
)
transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers}
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
defer transport.CloseIdleConnections()
for range workers {
group.Add(1)
go func() {
defer group.Done()
<-start
id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message")
results <- messageResult{messageID: id, instanceID: instance, err: requestErr}
}()
}
close(start)
group.Wait()
close(results)
for result := range results {
if result.err != nil {
return result.err
}
if result.instanceID == "" {
return fmt.Errorf("message response has an empty X-Instance-Id")
}
instances[result.instanceID] = struct{}{}
if messageID == "" {
messageID = result.messageID
}
if result.messageID != messageID {
return fmt.Errorf("idempotent responses returned different message ids")
}
}
if len(instances) < 2 {
return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances))
}
return c.verifyMessageHistory(chatID, messageID)
}
func (c *apiClient) createChat() (string, error) {
payload, err := json.Marshal(map[string]any{
"title": "Replica check", "member_ids": []string{c.userID},
})
if err != nil {
return "", err
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload))
if err != nil {
return "", err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
response, err := c.client.Do(request)
if err != nil {
return "", fmt.Errorf("create chat: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return "", fmt.Errorf("create chat status = %d", response.StatusCode)
}
var chat struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil {
return "", fmt.Errorf("decode chat: %w", err)
}
return chat.ID, nil
}
func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) {
payload, err := json.Marshal(map[string]string{"text": text})
if err != nil {
return "", "", err
}
request, err := http.NewRequest(
http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload),
)
if err != nil {
return "", "", err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", key)
response, err := client.Do(request)
if err != nil {
return "", "", fmt.Errorf("create message: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode)
}
var message struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil {
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err)
}
return message.ID, response.Header.Get("X-Instance-Id"), nil
}
func (c *apiClient) verifyMessageHistory(chatID, messageID string) error {
request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil)
if err != nil {
return err
}
request.Header.Set("Authorization", "Bearer "+c.token)
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("list messages: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("list messages status = %d", response.StatusCode)
}
var page struct {
Items []struct {
ID string `json:"id"`
} `json:"items"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil {
return fmt.Errorf("decode messages: %w", err)
}
if len(page.Items) != 1 || page.Items[0].ID != messageID {
return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items))
}
return nil
}
func verifyOutage(client *apiClient, service string) (returnErr error) {
if err := compose("stop", service); err != nil {
return err
}
defer func() {
if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil {
returnErr = err
}
}()
if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil {
return fmt.Errorf("%s readiness outage: %w", service, err)
}
if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil {
return fmt.Errorf("%s business outage: %w", service, err)
}
if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil {
return err
}
if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil {
return fmt.Errorf("%s readiness recovery: %w", service, err)
}
if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil {
return fmt.Errorf("%s session recovery: %w", service, err)
}
return nil
}
func compose(arguments ...string) error {
command := exec.Command("docker", append([]string{"compose"}, arguments...)...)
if output, err := command.CombinedOutput(); err != nil {
return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output))
}
return nil
}
func (c *apiClient) expectEventually(path string, status int, authenticated bool) error {
deadline := time.Now().Add(20 * time.Second)
var lastErr error
for time.Now().Before(deadline) {
lastErr = c.expect(http.MethodGet, path, status, authenticated)
if lastErr == nil {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return lastErr
}
func (c *apiClient) expect(method, path string, status int, authenticated bool) error {
request, err := http.NewRequest(method, c.baseURL+path, nil)
if err != nil {
return err
}
if authenticated {
request.Header.Set("Authorization", "Bearer "+c.token)
}
response, err := c.client.Do(request)
if err != nil {
return err
}
return closeResponse(response, status)
}
func closeResponse(response *http.Response, status int) error {
defer func() { _ = response.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
if response.StatusCode != status {
return fmt.Errorf("status = %d, want %d", response.StatusCode, status)
}
return nil
}
+676
View File
@@ -0,0 +1,676 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"image"
"image/jpeg"
"io"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
)
const experimentPassword = "Lab4-experiment-password"
type apiClient struct {
baseURL string
client *http.Client
token string
}
type attachment struct {
ID string `json:"id"`
}
type setup struct {
client *apiClient
chatID string
}
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(arguments []string, output io.Writer) error {
if len(arguments) == 0 {
return errors.New("usage: lab4experiment memory|consistency [flags]")
}
switch arguments[0] {
case "memory":
return runMemory(arguments[1:], output)
case "consistency":
return runConsistency(arguments[1:], output)
default:
return errors.New("usage: lab4experiment memory|consistency [flags]")
}
}
type memoryResult struct {
Replicas int `json:"replicas"`
Concurrency int `json:"concurrency"`
Rounds int `json:"rounds"`
SettleTimeMillis int64 `json:"settle_time_ms"`
ImageBytes int `json:"image_bytes"`
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
SuccessfulUploads int64 `json:"successful_uploads"`
FailedUploads int64 `json:"failed_uploads"`
DurationMillis int64 `json:"duration_ms"`
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
HeapPeakBytes int64 `json:"heap_peak_bytes"`
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
RSSPeakBytes int64 `json:"rss_peak_bytes"`
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
}
func runMemory(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
flags.SetOutput(io.Discard)
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
replicas := flags.Int("replicas", 2, "expected API replicas")
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
rounds := flags.Int("rounds", 4, "number of upload waves")
width := flags.Int("width", 2048, "image width")
height := flags.Int("height", 2048, "image height")
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("invalid memory experiment flags")
}
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
fixture, err := jpegFixture(*width, *height)
if err != nil {
return err
}
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
return err
}
settle := time.NewTimer(*settleTime)
defer settle.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-settle.C:
}
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
if err != nil {
return err
}
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
if err != nil {
return err
}
environment, err := prepare(ctx, *baseURL)
if err != nil {
return err
}
var successful atomic.Int64
var failed atomic.Int64
var firstError struct {
sync.Mutex
err error
}
heapPeak := heapBaseline
rssPeak := rssBaseline
loadDone := make(chan struct{})
sampleDone := make(chan error, 1)
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
started := time.Now()
for round := range *rounds {
var workers sync.WaitGroup
workers.Add(*concurrency)
for index := range *concurrency {
go func(round, index int) {
defer workers.Done()
reader := newRateReader(fixture, *uploadRate)
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
failed.Add(1)
firstError.Lock()
if firstError.err == nil {
firstError.err = err
}
firstError.Unlock()
return
}
successful.Add(1)
}(round, index)
}
workers.Wait()
}
close(loadDone)
if err := <-sampleDone; err != nil {
return err
}
result := memoryResult{
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
}
if result.FailedUploads > 0 {
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
}
return writeJSON(output, result)
}
func sampleMemory(
ctx context.Context,
prometheusURL string,
expectedReplicas int,
done <-chan struct{},
heapPeak *int64,
rssPeak *int64,
result chan<- error,
) {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
result <- ctx.Err()
return
case <-done:
result <- nil
return
case <-ticker.C:
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
if err != nil {
result <- err
return
}
if replicas != int64(expectedReplicas) {
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
return
}
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
if err != nil {
result <- err
return
}
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
if err != nil {
result <- err
return
}
*heapPeak = max(*heapPeak, heap)
*rssPeak = max(*rssPeak, rss)
}
}
}
type consistencyResult struct {
PageSize int `json:"page_size"`
SeededConsistent int `json:"seeded_consistent"`
FreshApply reconciliationRun `json:"fresh_apply"`
DryRun reconciliationRun `json:"dry_run"`
Apply reconciliationRun `json:"apply"`
RepeatedApply reconciliationRun `json:"repeated_apply"`
FreshPreserved bool `json:"fresh_preserved"`
OrphanDeleted bool `json:"orphan_deleted"`
UnusedDeleted bool `json:"unused_metadata_deleted"`
InUsePreserved bool `json:"in_use_metadata_preserved"`
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
TotalDurationMillis int64 `json:"total_duration_ms"`
}
type reconciliationRun struct {
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func runConsistency(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
flags.SetOutput(io.Discard)
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
pageSize := flags.Int("page-size", 10, "reconciliation page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("invalid consistency experiment flags")
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
settings, err := config.LoadReconciliation()
if err != nil {
return err
}
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
metadata := postgres.New(pool)
clean, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
initial, err := clean.Run(ctx)
if err != nil {
return err
}
if hasAnomalies(initial) {
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
}
started := time.Now()
environment, err := prepare(ctx, *baseURL)
if err != nil {
return err
}
fixture, err := jpegFixture(320, 200)
if err != nil {
return err
}
seededConsistent := *pageSize + 1
for index := range seededConsistent {
filename := fmt.Sprintf("consistent-%03d.jpg", index)
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
return err
}
}
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
return err
}
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
}
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
if err != nil {
return err
}
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
return err
}
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
return err
}
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
return err
}
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
}
freshRun, err := measuredReconciliation(ctx, fresh)
if err != nil {
return err
}
freshPreserved := objectExists(ctx, objects, orphanKey) &&
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
attachmentExists(ctx, pool, abandoned.ID)
if _, err := pool.Exec(ctx, `
UPDATE attachments
SET created_at = clock_timestamp() - INTERVAL '2 hours'
WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("age abandoned attachment: %w", err)
}
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
dryRunResult, err := measuredReconciliation(ctx, dryRun)
if err != nil {
return err
}
apply, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
}
applyResult, err := measuredReconciliation(ctx, apply)
if err != nil {
return err
}
repeatedResult, err := measuredReconciliation(ctx, apply)
if err != nil {
return err
}
result := consistencyResult{
PageSize: *pageSize, SeededConsistent: seededConsistent,
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
TotalDurationMillis: time.Since(started).Milliseconds(),
}
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
return fmt.Errorf("unexpected reconciliation result: %+v", result)
}
return writeJSON(output, result)
}
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
started := time.Now()
report, err := reconciler.Run(ctx)
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
}
func hasAnomalies(report reconcile.Report) bool {
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
}
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
if err != nil {
return nil, fmt.Errorf("query attachment keys: %w", err)
}
defer rows.Close()
keys := make(map[string]string, len(ids))
for rows.Next() {
var id string
var key string
if err := rows.Scan(&id, &key); err != nil {
return nil, fmt.Errorf("scan attachment key: %w", err)
}
keys[id] = key
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate attachment keys: %w", err)
}
if len(keys) != len(ids) {
return nil, errors.New("not all attachment keys were found")
}
return keys, nil
}
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
var exists bool
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
}
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
_, err := objects.Stat(ctx, key)
return err == nil
}
func prepare(ctx context.Context, baseURL string) (setup, error) {
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
var user struct {
ID string `json:"id"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
}, &user, ""); err != nil {
return setup{}, err
}
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
var session struct {
Token string `json:"token"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
return setup{}, err
}
client.token = session.Token
var chat struct {
ID string `json:"id"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
}, &chat, "Bearer "+client.token); err != nil {
return setup{}, err
}
return setup{client: client, chatID: chat.ID}, nil
}
func (c *apiClient) upload(
ctx context.Context,
chatID string,
filename string,
body io.Reader,
contentLength int64,
) (attachment, error) {
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
if err != nil {
return attachment{}, err
}
request.ContentLength = contentLength
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "image/jpeg")
response, err := c.client.Do(request)
if err != nil {
return attachment{}, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
}
var result attachment
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
return attachment{}, err
}
return result, nil
}
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
"text": "", "attachment_ids": []string{attachmentID},
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
}
func (c *apiClient) jsonRequest(
ctx context.Context,
method string,
path string,
payload any,
result any,
authorization string,
extraHeader ...string,
) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
body = bytes.NewReader(encoded)
}
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
if err != nil {
return err
}
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if authorization != "" {
request.Header.Set("Authorization", authorization)
}
for index := 0; index+1 < len(extraHeader); index += 2 {
request.Header.Set(extraHeader[index], extraHeader[index+1])
}
response, err := c.client.Do(request)
if err != nil {
return err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode < 200 || response.StatusCode >= 300 {
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
}
if result == nil {
return nil
}
return json.NewDecoder(response.Body).Decode(result)
}
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
if err == nil && count == int64(expected) {
return nil
}
select {
case <-ctx.Done():
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
case <-ticker.C:
}
}
}
func activeReplicasQuery() string {
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
}
func freshMetricQuery(metric string) string {
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
}
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
if err != nil {
return 0, err
}
values := request.URL.Query()
values.Set("query", query)
request.URL.RawQuery = values.Encode()
response, err := http.DefaultClient.Do(request)
if err != nil {
return 0, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
}
var payload struct {
Status string `json:"status"`
Data struct {
Result []struct {
Value []json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
return 0, err
}
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
return 0, errors.New("prometheus query returned no scalar value")
}
var raw string
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
return 0, err
}
value, err := strconv.ParseFloat(raw, 64)
return int64(value), err
}
func jpegFixture(width, height int) ([]byte, error) {
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
state := uint64(0x9e3779b97f4a7c15)
for index := range pixels.Pix {
state ^= state << 13
state ^= state >> 7
state ^= state << 17
pixels.Pix[index] = byte(state >> 24)
}
var encoded bytes.Buffer
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
return nil, err
}
return encoded.Bytes(), nil
}
type rateReader struct {
reader *bytes.Reader
rate int64
started time.Time
read int64
}
func newRateReader(data []byte, rate int64) *rateReader {
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
}
func (r *rateReader) Read(buffer []byte) (int, error) {
for {
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
if allowed > 0 {
if int64(len(buffer)) > allowed {
buffer = buffer[:allowed]
}
count, err := r.reader.Read(buffer)
r.read += int64(count)
return count, err
}
time.Sleep(10 * time.Millisecond)
}
}
func writeJSON(output io.Writer, value any) error {
encoder := json.NewEncoder(output)
encoder.SetIndent("", " ")
return encoder.Encode(value)
}
+31
View File
@@ -0,0 +1,31 @@
package main
import (
"errors"
"log/slog"
"os"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
)
func main() {
appLogger := logger.New()
if err := run(appLogger, os.Args[1:]); err != nil {
appLogger.Error("application stopped", "error", err)
os.Exit(1)
}
}
func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 0 {
return bootstrap.RunServer(logger)
}
if len(arguments) == 1 && arguments[0] == "migrate" {
return bootstrap.RunMigrations()
}
if arguments[0] == "reconcile-attachments" {
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
}
return errors.New("usage: messenger [migrate|reconcile-attachments]")
}
+328
View File
@@ -0,0 +1,328 @@
services:
postgres:
image: postgres:17.6-alpine3.22
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${POSTGRES_CPUS}
mem_limit: ${POSTGRES_MEMORY}
restart: unless-stopped
networks: [data]
valkey:
image: valkey/valkey:8.1.4-alpine
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"]
environment:
VALKEY_PASSWORD: ${VALKEY_PASSWORD}
volumes:
- valkey_data:/data
healthcheck:
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${VALKEY_CPUS}
mem_limit: ${VALKEY_MEMORY}
restart: unless-stopped
networks: [data]
s3:
build:
context: .
dockerfile: infra/minio/server.Dockerfile
args:
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
command: ["server", "/data", "--address", ":9000"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
MINIO_BROWSER: "off"
MINIO_PROMETHEUS_AUTH_TYPE: public
volumes:
- s3_data:/data
expose:
- "9000"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${S3_CPUS}
mem_limit: ${S3_MEMORY}
restart: unless-stopped
networks: [data]
s3-bootstrap:
build:
context: .
dockerfile: infra/minio/client.Dockerfile
args:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z
entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
depends_on:
s3:
condition: service_healthy
healthcheck:
disable: true
cpus: ${S3_BOOTSTRAP_CPUS}
mem_limit: ${S3_BOOTSTRAP_MEMORY}
restart: "no"
networks: [data]
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["/app/messenger", "migrate"]
environment:
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
cpus: ${MIGRATE_CPUS}
mem_limit: ${MIGRATE_MEMORY}
restart: "no"
networks: [data]
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab4}
environment:
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
DATABASE_URL: ${DATABASE_URL}
CACHE_URL: ${CACHE_URL}
SESSION_STORE: ${SESSION_STORE}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
SWAGGER_PORT: ${SWAGGER_PORT}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
S3_USE_TLS: ${S3_USE_TLS}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
expose:
- "8080"
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
interval: 2s
timeout: 1s
start_period: 5s
retries: 2
deploy:
replicas: ${API_REPLICAS:-2}
cpus: ${API_CPUS}
mem_limit: ${API_MEMORY}
stop_grace_period: 8s
restart: unless-stopped
networks: [application, data]
nginx:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab4}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"]
interval: 2s
timeout: 1s
retries: 10
cpus: ${NGINX_CPUS}
mem_limit: ${NGINX_MEMORY}
restart: unless-stopped
networks: [edge, application, monitoring]
swagger:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab4}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
nginx:
condition: service_healthy
cpus: ${SWAGGER_CPUS}
mem_limit: ${SWAGGER_MEMORY}
restart: unless-stopped
networks: [edge]
tests:
profiles: ["tests"]
build:
context: .
target: test
environment:
TEST_DATABASE_URL: ${DATABASE_URL}
TEST_CACHE_URL: ${CACHE_URL}
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
cpus: ${TEST_CPUS}
mem_limit: ${TEST_MEMORY}
networks: [data]
postgres-exporter:
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
environment:
DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable
DATA_SOURCE_USER: ${POSTGRES_USER}
DATA_SOURCE_PASS: ${POSTGRES_PASSWORD}
PG_EXPORTER_COLLECTION_TIMEOUT: 5s
depends_on:
postgres:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
networks: [data, monitoring]
valkey-exporter:
image: oliver006/redis_exporter:v1.89.0-alpine
environment:
REDIS_ADDR: redis://valkey:6379
REDIS_USER: default
REDIS_PASSWORD: ${VALKEY_PASSWORD}
depends_on:
valkey:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
networks: [data, monitoring]
nginx-exporter:
image: nginx/nginx-prometheus-exporter:1.5.1
command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"]
depends_on:
nginx:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
networks: [monitoring]
prometheus:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab4}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
volumes:
- prometheus_data:/prometheus
ports:
- "${PROMETHEUS_BIND_ADDRESS}:9090"
healthcheck:
test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"]
interval: 5s
timeout: 3s
retries: 20
depends_on:
- api
- nginx-exporter
- postgres-exporter
- valkey-exporter
- s3
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
networks: [edge, application, data, monitoring]
grafana:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab4}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
volumes:
- grafana_data:/var/lib/grafana
ports:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
volumes:
postgres_data:
valkey_data:
prometheus_data:
grafana_data:
s3_data:
networks:
edge:
application:
internal: true
data:
internal: true
monitoring:
internal: true
+24
View File
@@ -0,0 +1,24 @@
services:
api:
image: ${API_IMAGE:?API_IMAGE is required}
migrate:
image: ${API_IMAGE:?API_IMAGE is required}
nginx:
image: ${NGINX_IMAGE:?NGINX_IMAGE is required}
swagger:
image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required}
prometheus:
image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required}
grafana:
image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required}
s3:
image: ${S3_IMAGE:?S3_IMAGE is required}
s3-bootstrap:
image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required}
+4
View File
@@ -0,0 +1,4 @@
services:
nginx:
volumes:
- ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro
+94
View File
@@ -0,0 +1,94 @@
{
"page_size": 10,
"seeded_consistent": 11,
"fresh_apply": {
"duration_ms": 4,
"report": {
"metadata_scanned": 14,
"objects_scanned": 13,
"consistent": 12,
"fresh_pending_attachments": 1,
"old_pending_attachments": 0,
"deleting_attachments": 0,
"fresh_unreferenced_attachments": 11,
"old_unreferenced_attachments": 0,
"deleted_attachment_objects": 0,
"fresh_orphan_objects": 1,
"old_orphan_objects": 0,
"deleted_orphan_objects": 0,
"fresh_missing_objects": 2,
"referenced_missing_objects": 0,
"old_unreferenced_metadata": 0,
"deleted_unreferenced_rows": 0
}
},
"dry_run": {
"duration_ms": 5,
"report": {
"metadata_scanned": 14,
"objects_scanned": 13,
"consistent": 11,
"fresh_pending_attachments": 0,
"old_pending_attachments": 1,
"deleting_attachments": 0,
"fresh_unreferenced_attachments": 11,
"old_unreferenced_attachments": 0,
"deleted_attachment_objects": 0,
"fresh_orphan_objects": 0,
"old_orphan_objects": 1,
"deleted_orphan_objects": 0,
"fresh_missing_objects": 0,
"referenced_missing_objects": 1,
"old_unreferenced_metadata": 1,
"deleted_unreferenced_rows": 0
}
},
"apply": {
"duration_ms": 13,
"report": {
"metadata_scanned": 14,
"objects_scanned": 13,
"consistent": 11,
"fresh_pending_attachments": 0,
"old_pending_attachments": 1,
"deleting_attachments": 0,
"fresh_unreferenced_attachments": 11,
"old_unreferenced_attachments": 0,
"deleted_attachment_objects": 1,
"fresh_orphan_objects": 0,
"old_orphan_objects": 1,
"deleted_orphan_objects": 1,
"fresh_missing_objects": 0,
"referenced_missing_objects": 1,
"old_unreferenced_metadata": 1,
"deleted_unreferenced_rows": 2
}
},
"repeated_apply": {
"duration_ms": 1,
"report": {
"metadata_scanned": 12,
"objects_scanned": 11,
"consistent": 11,
"fresh_pending_attachments": 0,
"old_pending_attachments": 0,
"deleting_attachments": 0,
"fresh_unreferenced_attachments": 11,
"old_unreferenced_attachments": 0,
"deleted_attachment_objects": 0,
"fresh_orphan_objects": 0,
"old_orphan_objects": 0,
"deleted_orphan_objects": 0,
"fresh_missing_objects": 0,
"referenced_missing_objects": 1,
"old_unreferenced_metadata": 0,
"deleted_unreferenced_rows": 0
}
},
"fresh_preserved": true,
"orphan_deleted": true,
"unused_metadata_deleted": true,
"in_use_metadata_preserved": true,
"abandoned_attachment_deleted": true,
"total_duration_ms": 328
}
+17
View File
@@ -0,0 +1,17 @@
{
"replicas": 2,
"concurrency": 12,
"rounds": 4,
"settle_time_ms": 3000,
"image_bytes": 3759914,
"upload_rate_bytes_per_second": 1572864,
"successful_uploads": 48,
"failed_uploads": 0,
"duration_ms": 12974,
"heap_baseline_bytes": 153985824,
"heap_peak_bytes": 202505208,
"heap_delta_bytes": 48519384,
"rss_baseline_bytes": 335298560,
"rss_peak_bytes": 345976832,
"rss_delta_bytes": 10678272
}
+30
View File
@@ -0,0 +1,30 @@
Date: 2026-10-09
Source commit: 37d93c340e394ed4a923972accc6c872abb96076
Result collection state: application, migrations, Compose and Nginx configuration matched the source commit; REPORT.md and evidence/ were refreshed afterward
Environment: Docker 29.2.1, linux/arm64, 2 CPUs, 2054299648 bytes RAM
Application topology: 2 API replicas, 512 MiB limit per replica
PASS gofmt for changed Go packages
PASS go test ./internal/...
PASS go build ./cmd/messenger and ./cmd/lab4experiment in Docker build
PASS make check
PASS docker compose config --quiet
PASS sh -n scripts/rolling-update.sh
PASS make test BASE_URL=http://127.0.0.1:28080
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
PASS PostgreSQL and Valkey integration suites
PASS MinIO bootstrap completed twice without an error
PASS anonymous bucket request returned HTTP 403
PASS real pending -> deleting -> delete on PostgreSQL
PASS stale pending candidate cannot claim an attachment already changed to ready
PASS 48 of 48 uploads succeeded in 4 waves with concurrency 12
PASS reconciler preserved fresh and referenced anomalies
PASS reconciler deleted an old orphan, old unreferenced missing metadata, and old pending data
PASS repeated reconciliation apply made no further changes
PASS Nginx accepted the candidate upstream with nginx -t and atomic reload
PASS 20 of 20 health requests returned HTTP 200 from only the two candidate API instances
CI additionally runs full OpenAPI validation, race tests, golangci-lint,
persistence and API-replica failover on the exact pushed commit. Production
deployment publishes verified images and passes immutable repository@sha256
references to the manually approved environment job.
+55
View File
@@ -0,0 +1,55 @@
module gitea.privcloud.tech/dizi/mtusi2026lab4
go 1.26.0
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.0.99
github.com/prometheus/client_golang v1.23.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+117
View File
@@ -0,0 +1,117 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
set -eu
alias_name=bootstrap
policy_name=messenger-attachments
policy_file=/tmp/messenger-attachments-policy.json
mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing "$alias_name/$S3_BUCKET"
mc anonymous set none "$alias_name/$S3_BUCKET"
cat >"$policy_file" <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::$S3_BUCKET"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]
}
]
}
EOF
mc admin policy create "$alias_name" "$policy_name" "$policy_file"
mc admin user add "$alias_name" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
mc admin policy attach "$alias_name" "$policy_name" --user "$S3_ACCESS_KEY"
mc alias set application http://s3:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
mc ls "application/$S3_BUCKET" >/dev/null
+16
View File
@@ -0,0 +1,16 @@
FROM golang:1.24.8-alpine3.22 AS build
ARG MC_VERSION
RUN apk add --no-cache git
RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc
WORKDIR /src/mc
RUN go mod download
RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc .
FROM alpine:3.22.2
COPY --from=build /out/mc /usr/local/bin/mc
COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh
ENTRYPOINT ["mc"]
+18
View File
@@ -0,0 +1,18 @@
FROM golang:1.24.8-alpine3.22 AS build
ARG MINIO_VERSION
RUN apk add --no-cache git
RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio
WORKDIR /src/minio
RUN go mod download
RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio .
FROM alpine:3.22.2
RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data
COPY --from=build /out/minio /usr/local/bin/minio
USER minio
VOLUME ["/data"]
ENTRYPOINT ["minio"]
+25
View File
@@ -0,0 +1,25 @@
package bootstrap
import (
"context"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
)
func RunMigrations() error {
settings, err := config.LoadMigration()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
+85
View File
@@ -0,0 +1,85 @@
package bootstrap
import (
"context"
"encoding/json"
"errors"
"flag"
"io"
"os/signal"
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
)
type reconciliationOutput struct {
Mode string `json:"mode"`
GracePeriod string `json:"grace_period"`
UnreferencedRetention string `json:"unreferenced_retention"`
PageSize int `json:"page_size"`
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func RunReconciliation(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
flags.SetOutput(io.Discard)
apply := flags.Bool("apply", false, "apply safe repairs")
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments")
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]")
}
if *apply && *gracePeriod <= 0 {
return errors.New("grace-period must be positive in apply mode")
}
settings, err := config.LoadReconciliation()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
UnreferencedRetention: *retention, Apply: *apply,
})
if err != nil {
return err
}
started := time.Now()
report, err := reconciler.Run(ctx)
if err != nil {
return err
}
mode := "dry-run"
if *apply {
mode = "apply"
}
return json.NewEncoder(output).Encode(reconciliationOutput{
Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(),
PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report,
})
}
+13
View File
@@ -0,0 +1,13 @@
package bootstrap
import (
"io"
"testing"
"github.com/stretchr/testify/require"
)
func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) {
err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard)
require.EqualError(t, err, "grace-period must be positive in apply mode")
}
+113
View File
@@ -0,0 +1,113 @@
package bootstrap
import (
"context"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
func RunServer(logger *slog.Logger) error {
settings, err := config.LoadServer()
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), databaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
sessions, err := session.New(settings.CacheURL)
if err != nil {
return err
}
defer func() { _ = sessions.Close() }()
passwords, err := password.New(password.Config{
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
KeyLength: settings.PasswordKeyLength,
})
if err != nil {
return err
}
objects, err := objectstore.New(objectStoreConfig(settings))
if err != nil {
return err
}
repository := postgres.New(pool)
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
CleanupTimeout: settings.RequestTimeout,
})
if err != nil {
return err
}
app := httpapi.NewServer(logger, httpapi.Dependencies{
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
}, httpapi.Config{
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
})
server := &http.Server{
Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout,
IdleTimeout: settings.HTTPIdleTimeout,
}
listener, err := net.Listen("tcp", server.Addr)
if err != nil {
return err
}
serveResult := make(chan error, 1)
go func() { serveResult <- server.Serve(listener) }()
logger.Info("server started", "address", server.Addr)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
select {
case err := <-serveResult:
return err
case <-ctx.Done():
logger.Info("shutdown requested")
app.SetReady(false)
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
_ = server.Close()
return err
}
return nil
}
}
func databaseConfig(settings config.ServerConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}
func objectStoreConfig(settings config.ServerConfig) objectstore.Config {
return objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
}
}
+400
View File
@@ -0,0 +1,400 @@
package config
import (
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
"github.com/spf13/viper"
)
type ServerConfig struct {
Address string
DatabaseURL string
CacheURL string
SessionTTL time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
RequestTimeout time.Duration
HTTPReadHeaderTimeout time.Duration
HTTPReadTimeout time.Duration
HTTPWriteTimeout time.Duration
HTTPIdleTimeout time.Duration
HTTPShutdownTimeout time.Duration
PasswordMemoryKiB uint32
PasswordIterations uint32
PasswordParallelism uint8
PasswordSaltLength uint32
PasswordKeyLength uint32
SwaggerPort string
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
AttachmentMaxBytes int64
AttachmentMaxPixels int64
AttachmentBodyMemory int64
AttachmentDecodeMemory int64
}
type MigrationConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
MigrationTimeout time.Duration
}
type ReconciliationConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
}
type databaseConfig struct {
url string
maxConns int
minConns int
maxConnectionAge time.Duration
maxConnectionIdle time.Duration
healthCheckPeriod time.Duration
dependencyTimeout time.Duration
}
type objectStorageConfig struct {
endpoint string
region string
bucket string
accessKey string
secretKey string
useTLS bool
timeout time.Duration
}
func LoadServer() (ServerConfig, error) {
values := viper.New()
values.AutomaticEnv()
host, err := requiredString(values, "APP_HOST")
if err != nil {
return ServerConfig{}, err
}
port, err := requiredPort(values, "APP_PORT")
if err != nil {
return ServerConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return ServerConfig{}, err
}
cacheURL, err := requiredString(values, "CACHE_URL")
if err != nil {
return ServerConfig{}, err
}
store, err := requiredString(values, "SESSION_STORE")
if err != nil {
return ServerConfig{}, err
}
if store != "valkey" {
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
}
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
if err != nil {
return ServerConfig{}, err
}
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
if err != nil {
return ServerConfig{}, err
}
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return ServerConfig{}, err
}
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return ServerConfig{}, err
}
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return ServerConfig{}, err
}
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return ServerConfig{}, err
}
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
if err != nil {
return ServerConfig{}, err
}
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return ServerConfig{}, err
}
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
if err != nil {
return ServerConfig{}, err
}
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
if err != nil {
return ServerConfig{}, err
}
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
if err != nil {
return ServerConfig{}, err
}
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
if err != nil {
return ServerConfig{}, err
}
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
if err != nil {
return ServerConfig{}, err
}
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
if err != nil {
return ServerConfig{}, err
}
storage, err := loadObjectStorageConfig(values)
if err != nil {
return ServerConfig{}, err
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
if err != nil {
return ServerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
if err != nil {
return ServerConfig{}, err
}
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
if err != nil {
return ServerConfig{}, err
}
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
}
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
if err != nil {
return ServerConfig{}, err
}
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
}
return ServerConfig{
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
RequestTimeout: requestTimeout,
HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout,
HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout,
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
AttachmentBodyMemory: int64(attachmentBodyMemory),
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
}, nil
}
func LoadMigration() (MigrationConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return MigrationConfig{}, err
}
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
if err != nil {
return MigrationConfig{}, err
}
return MigrationConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
MigrationTimeout: migrationTimeout,
}, nil
}
func LoadReconciliation() (ReconciliationConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return ReconciliationConfig{}, err
}
storage, err := loadObjectStorageConfig(values)
if err != nil {
return ReconciliationConfig{}, err
}
return ReconciliationConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
}, nil
}
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
endpoint, err := requiredString(values, "S3_ENDPOINT")
if err != nil {
return objectStorageConfig{}, err
}
region, err := requiredString(values, "S3_REGION")
if err != nil {
return objectStorageConfig{}, err
}
bucket, err := requiredString(values, "S3_BUCKET")
if err != nil {
return objectStorageConfig{}, err
}
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
if err != nil {
return objectStorageConfig{}, err
}
secretKey, err := requiredString(values, "S3_SECRET_KEY")
if err != nil {
return objectStorageConfig{}, err
}
useTLS, err := boolean(values, "S3_USE_TLS")
if err != nil {
return objectStorageConfig{}, err
}
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return objectStorageConfig{}, err
}
return objectStorageConfig{
endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey,
secretKey: secretKey, useTLS: useTLS, timeout: timeout,
}, nil
}
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
databaseURL, err := requiredString(values, "DATABASE_URL")
if err != nil {
return databaseConfig{}, err
}
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
if err != nil {
return databaseConfig{}, err
}
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
if err != nil {
return databaseConfig{}, err
}
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
if err != nil {
return databaseConfig{}, err
}
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
if err != nil {
return databaseConfig{}, err
}
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
if err != nil {
return databaseConfig{}, err
}
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
if err != nil {
return databaseConfig{}, err
}
return databaseConfig{
url: databaseURL, maxConns: maxConns, minConns: minConns,
maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle,
healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout,
}, nil
}
func requiredString(values *viper.Viper, name string) (string, error) {
value := strings.TrimSpace(values.GetString(name))
if value == "" {
return "", fmt.Errorf("%s is required", name)
}
return value, nil
}
func requiredPort(values *viper.Viper, name string) (string, error) {
value, err := requiredString(values, name)
if err != nil {
return "", err
}
if _, err := integerValue(value, 1, 65535); err != nil {
return "", fmt.Errorf("%s: %w", name, err)
}
return value, nil
}
func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) {
value, err := integer(values, name, minimum, maximum)
if err != nil {
return 0, err
}
return time.Duration(value) * time.Second, nil
}
func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) {
value, err := requiredString(values, name)
if err != nil {
return 0, err
}
parsed, err := integerValue(value, minimum, maximum)
if err != nil {
return 0, fmt.Errorf("%s: %w", name, err)
}
return parsed, nil
}
func integerValue(value string, minimum, maximum int) (int, error) {
parsed, err := strconv.Atoi(value)
if err != nil || parsed < minimum || parsed > maximum {
return 0, fmt.Errorf("must be between %d and %d", minimum, maximum)
}
return parsed, nil
}
func boolean(values *viper.Viper, name string) (bool, error) {
value, err := requiredString(values, name)
if err != nil {
return false, err
}
parsed, err := strconv.ParseBool(value)
if err != nil {
return false, fmt.Errorf("%s: must be true or false", name)
}
return parsed, nil
}
+29
View File
@@ -0,0 +1,29 @@
package config
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
values := map[string]string{
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
}
for name, value := range values {
t.Setenv(name, value)
}
t.Setenv("APP_HOST", "")
t.Setenv("CACHE_URL", "")
t.Setenv("SWAGGER_PORT", "")
settings, err := LoadReconciliation()
require.NoError(t, err)
require.Equal(t, "attachments", settings.S3Bucket)
}
+93
View File
@@ -0,0 +1,93 @@
package database
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab4/migrations"
)
var (
errURLRequired = errors.New("database URL is required")
errMinConnections = errors.New("minimum database connections must not be negative")
errMaxConnections = errors.New("maximum database connections must be positive")
errConnectionRange = errors.New("minimum database connections exceed maximum")
errMaxConnectionAge = errors.New("maximum database connection age must be positive")
errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive")
errHealthCheckPeriod = errors.New("database health check period must be positive")
errConnectionTimeout = errors.New("database connection timeout must be positive")
)
type Config struct {
URL string
MinConnections int32
MaxConnections int32
MaxConnectionAge time.Duration
MaxConnectionIdle time.Duration
HealthCheckPeriod time.Duration
ConnectTimeout time.Duration
}
func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
if err := validateConfig(config); err != nil {
return nil, err
}
poolConfig, err := pgxpool.ParseConfig(config.URL)
if err != nil {
return nil, fmt.Errorf("parse database configuration: %w", err)
}
poolConfig.MinConns = config.MinConnections
poolConfig.MaxConns = config.MaxConnections
poolConfig.MaxConnLifetime = config.MaxConnectionAge
poolConfig.MaxConnIdleTime = config.MaxConnectionIdle
poolConfig.HealthCheckPeriod = config.HealthCheckPeriod
connectCtx, cancel := context.WithTimeout(ctx, config.ConnectTimeout)
defer cancel()
pool, err := pgxpool.NewWithConfig(connectCtx, poolConfig)
if err != nil {
return nil, fmt.Errorf("create database pool: %w", err)
}
if err := pool.Ping(connectCtx); err != nil {
pool.Close()
return nil, fmt.Errorf("ping database: %w", err)
}
return pool, nil
}
func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error {
if timeout <= 0 {
return errors.New("database migration timeout must be positive")
}
migrationCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
return applyMigrations(migrationCtx, pool, migrations.FS)
}
func validateConfig(config Config) error {
switch {
case config.URL == "":
return errURLRequired
case config.MinConnections < 0:
return errMinConnections
case config.MaxConnections < 1:
return errMaxConnections
case config.MinConnections > config.MaxConnections:
return errConnectionRange
case config.MaxConnectionAge <= 0:
return errMaxConnectionAge
case config.MaxConnectionIdle <= 0:
return errMaxConnectionIdle
case config.HealthCheckPeriod <= 0:
return errHealthCheckPeriod
case config.ConnectTimeout <= 0:
return errConnectionTimeout
default:
return nil
}
}
+150
View File
@@ -0,0 +1,150 @@
package database
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"path/filepath"
"regexp"
"sort"
"strconv"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const (
migrationLockID int64 = 0x4d54555349324c42
migrationUnlockTimeout = 5 * time.Second
)
var migrationNamePattern = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.sql$`)
type migration struct {
version int64
name string
checksum string
sql string
}
func applyMigrations(ctx context.Context, pool *pgxpool.Pool, migrationFS fs.FS) (returnErr error) {
migrations, err := readMigrations(migrationFS)
if err != nil {
return err
}
connection, err := pool.Acquire(ctx)
if err != nil {
return fmt.Errorf("acquire connection for migrations: %w", err)
}
defer connection.Release()
if _, err := connection.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil {
return fmt.Errorf("lock migrations: %w", err)
}
defer func() {
unlockCtx, cancel := context.WithTimeout(context.Background(), migrationUnlockTimeout)
defer cancel()
if _, err := connection.Exec(unlockCtx, "SELECT pg_advisory_unlock($1)", migrationLockID); err != nil {
_ = connection.Conn().Close(unlockCtx)
if returnErr == nil {
returnErr = fmt.Errorf("unlock migrations: %w", err)
}
}
}()
if _, err := connection.Exec(ctx, `
CREATE TABLE IF NOT EXISTS schema_migrations (
version BIGINT PRIMARY KEY,
name TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)`); err != nil {
return fmt.Errorf("create migration history: %w", err)
}
for _, migration := range migrations {
if err := applyMigration(ctx, connection, migration); err != nil {
return err
}
}
return nil
}
func readMigrations(migrationFS fs.FS) ([]migration, error) {
entries, err := fs.ReadDir(migrationFS, ".")
if err != nil {
return nil, fmt.Errorf("read embedded migrations: %w", err)
}
migrations := make([]migration, 0, len(entries))
versions := make(map[int64]string)
for _, entry := range entries {
if entry.IsDir() || filepath.Ext(entry.Name()) != ".sql" {
continue
}
matches := migrationNamePattern.FindStringSubmatch(entry.Name())
if matches == nil {
return nil, fmt.Errorf("invalid migration filename %q", entry.Name())
}
version, err := strconv.ParseInt(matches[1], 10, 64)
if err != nil || version < 1 {
return nil, fmt.Errorf("invalid migration version in %q", entry.Name())
}
if previous, exists := versions[version]; exists {
return nil, fmt.Errorf("migration version %d is used by %q and %q", version, previous, entry.Name())
}
data, err := fs.ReadFile(migrationFS, entry.Name())
if err != nil {
return nil, fmt.Errorf("read migration %q: %w", entry.Name(), err)
}
hash := sha256.Sum256(data)
versions[version] = entry.Name()
migrations = append(migrations, migration{
version: version,
name: entry.Name(),
checksum: hex.EncodeToString(hash[:]),
sql: string(data),
})
}
if len(migrations) == 0 {
return nil, errors.New("no database migrations found")
}
sort.Slice(migrations, func(i, j int) bool {
return migrations[i].version < migrations[j].version
})
return migrations, nil
}
func applyMigration(ctx context.Context, connection *pgxpool.Conn, migration migration) error {
var checksum string
err := connection.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version = $1", migration.version).Scan(&checksum)
switch {
case err == nil && checksum != migration.checksum:
return fmt.Errorf("migration %q checksum does not match applied version", migration.name)
case err == nil:
return nil
case !errors.Is(err, pgx.ErrNoRows):
return fmt.Errorf("read migration %q state: %w", migration.name, err)
}
transaction, err := connection.Begin(ctx)
if err != nil {
return fmt.Errorf("begin migration %q: %w", migration.name, err)
}
defer func() {
_ = transaction.Rollback(context.WithoutCancel(ctx))
}()
if _, err := transaction.Exec(ctx, migration.sql, pgx.QueryExecModeSimpleProtocol); err != nil {
return fmt.Errorf("apply migration %q: %w", migration.name, err)
}
if _, err := transaction.Exec(ctx, `
INSERT INTO schema_migrations (version, name, checksum)
VALUES ($1, $2, $3)`, migration.version, migration.name, migration.checksum); err != nil {
return fmt.Errorf("record migration %q: %w", migration.name, err)
}
if err := transaction.Commit(ctx); err != nil {
return fmt.Errorf("commit migration %q: %w", migration.name, err)
}
return nil
}
+10
View File
@@ -0,0 +1,10 @@
package logger
import (
"log/slog"
"os"
)
func New() *slog.Logger {
return slog.New(slog.NewJSONHandler(os.Stdout, nil))
}
+253
View File
@@ -0,0 +1,253 @@
package attachment
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"math"
"mime"
"net/http"
"time"
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type Repository interface {
ChatForUser(context.Context, string, string) (model.Chat, error)
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
DeletePendingAttachment(context.Context, string, string) error
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
}
type ObjectStore interface {
Put(context.Context, string, string, []byte) error
Get(context.Context, string) (io.ReadCloser, error)
Delete(context.Context, string) error
}
type Config struct {
MaxBytes int64
MaxPixels int64
BodyMemoryBytes int64
DecodeMemoryBytes int64
CleanupTimeout time.Duration
}
type Upload struct {
ChatID string
OwnerID string
Filename string
ContentType string
ContentLength int64
Body io.Reader
}
type Content struct {
Attachment model.Attachment
Body io.ReadCloser
}
type Service struct {
repository Repository
objects ObjectStore
config Config
bodyMemory *semaphore.Weighted
decodeMemory *semaphore.Weighted
logger *slog.Logger
}
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
return nil, errors.New("invalid attachment service configuration")
}
if config.BodyMemoryBytes < config.MaxBytes {
return nil, errors.New("attachment body memory must fit one maximum upload")
}
if config.DecodeMemoryBytes < config.MaxPixels*8 {
return nil, errors.New("attachment decode memory must fit one maximum image")
}
return &Service{
repository: repository, objects: objects, config: config,
bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes),
decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger,
}, nil
}
func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) {
if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil {
return model.Attachment{}, mapDependencyError(err)
}
data, contentType, release, err := s.readImage(input)
if err != nil {
return model.Attachment{}, err
}
defer release()
attachmentID := uuid.NewString()
objectKey := "attachments/" + uuid.NewString() + "/original"
digest := sha256.Sum256(data)
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
ObjectKey: objectKey, Status: model.AttachmentPending,
})
if err != nil {
return model.Attachment{}, mapDependencyError(err)
}
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
s.deletePending(pending.ID, objectKey)
return model.Attachment{}, model.ErrUnavailable
}
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
if err == nil {
return attachment, nil
}
return s.resolveReadyOutcome(pending.ID, objectKey)
}
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
if err == nil && stored.Status == model.AttachmentReady {
return stored, nil
}
if err == nil && stored.Status == model.AttachmentPending {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
return model.Attachment{}, model.ErrUnavailable
}
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
if errors.Is(err, model.ErrNotFound) {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
return model.Attachment{}, model.ErrUnavailable
}
func (s *Service) deletePending(attachmentID, objectKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
}
}
func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID)
if err != nil {
return model.Attachment{}, mapDependencyError(err)
}
return attachment, nil
}
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
attachment, err := s.Metadata(ctx, attachmentID, userID)
if err != nil {
return Content{}, err
}
body, err := s.objects.Get(ctx, attachment.ObjectKey)
if err != nil {
return Content{}, model.ErrUnavailable
}
return Content{Attachment: attachment, Body: body}, nil
}
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
mediaType, parameters, err := mime.ParseMediaType(input.ContentType)
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
return nil, "", nil, model.ErrUnsupportedMedia
}
if input.ContentLength > s.config.MaxBytes {
return nil, "", nil, model.ErrPayloadTooLarge
}
reservation := s.config.MaxBytes
if !s.bodyMemory.TryAcquire(reservation) {
return nil, "", nil, model.ErrCapacity
}
releaseBody := func() { s.bodyMemory.Release(reservation) }
data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1))
if err != nil {
releaseBody()
return nil, "", nil, model.ErrInvalidInput
}
if int64(len(data)) > s.config.MaxBytes {
releaseBody()
return nil, "", nil, model.ErrPayloadTooLarge
}
if len(data) == 0 {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
detected := http.DetectContentType(data)
if detected != mediaType {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || imageContentType(format) != mediaType {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) {
releaseBody()
return nil, "", nil, model.ErrPayloadTooLarge
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
decodeBytes := pixels * 8
if !s.decodeMemory.TryAcquire(decodeBytes) {
releaseBody()
return nil, "", nil, model.ErrCapacity
}
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
s.decodeMemory.Release(decodeBytes)
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
release := func() {
s.decodeMemory.Release(decodeBytes)
releaseBody()
}
return data, mediaType, release, nil
}
func imageContentType(format string) string {
switch format {
case "png":
return "image/png"
case "jpeg":
return "image/jpeg"
default:
return ""
}
}
func mapDependencyError(err error) error {
switch {
case errors.Is(err, model.ErrNotFound):
return model.ErrNotFound
case errors.Is(err, model.ErrConflict):
return model.ErrConflict
default:
return model.ErrUnavailable
}
}
@@ -0,0 +1,218 @@
package attachment
import (
"bytes"
"context"
"errors"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"math"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func TestUploadLifecycle(t *testing.T) {
data := pngFixture(t, 2, 2)
events := make([]string, 0, 3)
repository := &repositoryStub{events: &events}
objects := &objectStub{events: &events}
service := newTestService(t, repository, objects, 1024, 4)
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
require.Equal(t, []string{"pending", "put", "ready"}, events)
}
func TestUploadValidation(t *testing.T) {
data := pngFixture(t, 2, 2)
tests := []struct {
name string
contentType string
data []byte
maxBytes int64
maxPixels int64
want error
}{
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge},
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge},
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels)
input := upload(test.data, test.contentType)
_, err := service.Upload(context.Background(), input)
require.ErrorIs(t, err, test.want)
})
}
}
func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
data := pngFixture(t, 2, 2)
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
require.True(t, service.bodyMemory.TryAcquire(1024))
_, err := service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrCapacity)
service.bodyMemory.Release(1024)
require.True(t, service.decodeMemory.TryAcquire(32))
_, err = service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrCapacity)
service.decodeMemory.Release(32)
}
func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) {
oversized := pngFixture(t, 3, 2)
valid := pngFixture(t, 2, 2)
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
_, err := service.Upload(context.Background(), upload(oversized, "image/png"))
require.ErrorIs(t, err, model.ErrPayloadTooLarge)
_, err = service.Upload(context.Background(), upload(valid, "image/png"))
require.NoError(t, err)
}
func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
_, err := NewService(
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
Config{
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
},
)
require.Error(t, err)
}
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{}
objects := &objectStub{putErr: errors.New("s3 unavailable")}
service := newTestService(t, repository, objects, 1024, 4)
_, err := service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrUnavailable)
require.True(t, repository.pendingDeleted)
}
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
objects := &objectStub{}
service := newTestService(t, repository, objects, 1024, 4)
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
require.False(t, objects.deleted)
}
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
t.Helper()
service, err := NewService(
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
Config{
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
},
)
require.NoError(t, err)
return service
}
func upload(data []byte, contentType string) Upload {
return Upload{
ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType,
ContentLength: int64(len(data)), Body: bytes.NewReader(data),
}
}
type repositoryStub struct {
events *[]string
markErr error
lookupReady bool
pendingDeleted bool
pending model.Attachment
}
func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) {
return model.Chat{}, nil
}
func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) {
attachment.Status = model.AttachmentPending
s.pending = attachment
s.record("pending")
return attachment, nil
}
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
if s.markErr != nil {
return model.Attachment{}, s.markErr
}
s.pending.Status = model.AttachmentReady
s.record("ready")
return s.pending, nil
}
func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error {
s.pendingDeleted = true
return nil
}
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
if s.lookupReady {
s.pending.Status = model.AttachmentReady
return s.pending, nil
}
return model.Attachment{}, model.ErrNotFound
}
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
return s.pending, nil
}
func (s *repositoryStub) record(event string) {
if s.events != nil {
*s.events = append(*s.events, event)
}
}
type objectStub struct {
events *[]string
putErr error
deleted bool
}
func (s *objectStub) Put(context.Context, string, string, []byte) error {
if s.events != nil {
*s.events = append(*s.events, "put")
}
return s.putErr
}
func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(nil)), nil
}
func (s *objectStub) Delete(context.Context, string) error {
s.deleted = true
return nil
}
func pngFixture(t *testing.T, width, height int) []byte {
t.Helper()
var data bytes.Buffer
value := image.NewRGBA(image.Rect(0, 0, width, height))
value.Set(0, 0, color.White)
require.NoError(t, png.Encode(&data, value))
return data.Bytes()
}
+164
View File
@@ -0,0 +1,164 @@
package httpapi
import (
"errors"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
}()
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return invalidRequest()
}
filename, valid := attachmentFilename(query)
if !valid {
return invalidRequest()
}
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
})
if err != nil {
if errors.Is(err, model.ErrCapacity) {
result = "capacity_rejected"
w.Header().Set("Retry-After", "1")
} else if errors.Is(err, model.ErrUnavailable) {
result = "dependency_error"
}
return attachmentError(err)
}
result = "success"
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
writeJSON(w, http.StatusCreated, uploaded)
return nil
}
func attachmentFilename(query url.Values) (string, bool) {
if len(query) != 1 || len(query["filename"]) != 1 {
return "", false
}
filename := query["filename"][0]
length := utf8.RuneCountInString(filename)
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
return "", false
}
for _, char := range filename {
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
return "", false
}
}
return filename, true
}
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
attachmentID, apiErr := attachmentIDFromRequest(r)
if apiErr != nil {
return apiErr
}
value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID)
if err != nil {
return attachmentError(err)
}
writeJSON(w, http.StatusOK, value)
return nil
}
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || !originalVariant(query) {
return invalidRequest()
}
attachmentID, apiErr := attachmentIDFromRequest(r)
if apiErr != nil {
return apiErr
}
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
if err != nil {
return attachmentError(err)
}
defer func() { _ = content.Body.Close() }()
w.Header().Set("Content-Type", content.Attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content.Body); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
}
return nil
}
func originalVariant(query url.Values) bool {
if len(query) == 0 {
return true
}
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
}
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
attachmentID, valid := canonicalID(parts[3])
if !valid {
return "", invalidRequest()
}
return attachmentID, nil
}
func attachmentError(err error) *apiError {
switch {
case errors.Is(err, model.ErrNotFound):
return notFound()
case errors.Is(err, model.ErrInvalidInput):
return invalidRequest()
case errors.Is(err, model.ErrUnsupportedMedia):
return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
case errors.Is(err, model.ErrPayloadTooLarge):
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
case errors.Is(err, model.ErrCapacity):
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
default:
return unavailable()
}
}
+111
View File
@@ -0,0 +1,111 @@
package httpapi
import (
"errors"
"io"
"net/http"
"strings"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
body, err := io.ReadAll(io.LimitReader(r.Body, 1))
if err != nil || len(body) != 0 {
return invalidRequest()
}
username, passwordValue, ok := singleBasicAuth(r)
if !ok {
return unauthorized()
}
credentials, err := s.userRepository.UserByUsername(r.Context(), username)
if errors.Is(err, model.ErrNotFound) {
if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil {
return internalError()
}
return unauthorized()
}
if err != nil {
return unavailable()
}
verified, err := s.passwords.Verify(passwordValue, credentials.PasswordHash)
if err != nil {
return internalError()
}
if !verified {
return unauthorized()
}
token, expiresAt, err := s.sessions.Create(r.Context(), credentials.ID, s.sessionTTL)
if err != nil {
return unavailable()
}
writeJSON(w, http.StatusCreated, struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt string `json:"expires_at"`
User User `json:"user"`
}{token, "Bearer", model.FormatTime(expiresAt), credentials.User})
return nil
}
func singleBasicAuth(r *http.Request) (string, string, bool) {
if len(r.Header.Values("Authorization")) != 1 {
return "", "", false
}
username, passwordValue, ok := r.BasicAuth()
return username, passwordValue, ok && username != ""
}
func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodDelete {
return methodNotAllowed(w, http.MethodDelete)
}
_, token, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) {
return unauthorized()
} else if err != nil {
return unavailable()
}
w.WriteHeader(http.StatusNoContent)
return nil
}
func (s *Server) authenticate(r *http.Request) (User, string, *apiError) {
values := r.Header.Values("Authorization")
if len(values) != 1 {
return User{}, "", unauthorized()
}
scheme, token, found := strings.Cut(values[0], " ")
if !found || !strings.EqualFold(scheme, "Bearer") || token == "" || strings.ContainsAny(token, " \t\r\n") {
return User{}, "", unauthorized()
}
userID, err := s.sessions.UserID(r.Context(), token)
if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) {
return User{}, "", unauthorized()
}
if err != nil {
return User{}, "", unavailable()
}
user, err := s.userRepository.UserByID(r.Context(), userID)
if errors.Is(err, model.ErrNotFound) {
_ = s.sessions.Delete(r.Context(), token)
return User{}, "", unauthorized()
}
if err != nil {
return User{}, "", unavailable()
}
return user, token, nil
}
func authenticationChallenge(r *http.Request) string {
if r.URL.Path == "/api/v1/auth/sessions" {
return `Basic realm="messenger", charset="UTF-8"`
}
return `Bearer realm="messenger"`
}
+61
View File
@@ -0,0 +1,61 @@
package httpapi
import (
"encoding/json"
"net/http"
)
const (
errorCodeInvalidRequest = "invalid_request"
errorCodeUnauthorized = "unauthorized"
errorCodeNotFound = "not_found"
errorCodeInternal = "internal_error"
errorCodeUnsupportedMediaType = "unsupported_media_type"
errorCodePayloadTooLarge = "payload_too_large"
errorCodeUnavailable = "unavailable"
errorCodeConflict = "conflict"
errorCodeRateLimited = "rate_limited"
)
type apiError struct {
status int
code string
message string
}
func methodNotAllowed(w http.ResponseWriter, allowed string) *apiError {
w.Header().Set("Allow", allowed)
return &apiError{http.StatusMethodNotAllowed, errorCodeInvalidRequest, "method not allowed"}
}
func invalidRequest() *apiError {
return &apiError{http.StatusBadRequest, errorCodeInvalidRequest, "invalid request"}
}
func unauthorized() *apiError {
return &apiError{http.StatusUnauthorized, errorCodeUnauthorized, "authentication required"}
}
func notFound() *apiError {
return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"}
}
func internalError() *apiError {
return &apiError{http.StatusInternalServerError, errorCodeInternal, "internal server error"}
}
func unavailable() *apiError {
return &apiError{http.StatusServiceUnavailable, errorCodeUnavailable, "service unavailable"}
}
func writeError(w http.ResponseWriter, requestID string, err *apiError) {
writeJSON(w, err.status, map[string]any{"error": map[string]string{
"code": err.code, "message": err.message, "request_id": requestID,
}})
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(value)
}
+300
View File
@@ -0,0 +1,300 @@
package httpapi
import (
"context"
"errors"
"net/http"
"regexp"
"strings"
"time"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
func (s *Server) live(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
return nil
}
func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
if !s.ready.Load() {
return unavailable()
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
if err := s.database.Ping(ctx); err != nil {
return unavailable()
}
if err := s.sessions.Ping(ctx); err != nil {
return unavailable()
}
if err := s.objects.Ping(ctx); err != nil {
return unavailable()
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
return nil
}
func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
var input struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
if err := readJSON(w, r, &input); err != nil {
return err
}
if !usernamePattern.MatchString(input.Username) || !validText(input.DisplayName, 100) || !validPassword(input.Password) {
return invalidRequest()
}
passwordHash, err := s.passwords.Hash(input.Password)
if err != nil {
return internalError()
}
user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
if errors.Is(err, model.ErrConflict) {
return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"}
}
if err != nil {
return unavailable()
}
writeJSON(w, http.StatusCreated, user)
return nil
}
func validPassword(value string) bool {
length := utf8.RuneCountInString(value)
return utf8.ValidString(value) && strings.TrimSpace(value) != "" && length >= 12 && length <= 128
}
func (s *Server) me(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, err := s.authenticate(r)
if err != nil {
return err
}
writeJSON(w, http.StatusOK, user)
return nil
}
func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodPost && r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
if r.Method == http.MethodGet {
return s.listChats(w, r, user)
}
var input struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
if err := readJSON(w, r, &input); err != nil {
return err
}
if !validText(input.Title, 100) || len(input.MemberIDs) < 1 || len(input.MemberIDs) > 100 {
return invalidRequest()
}
members := make(map[string]struct{}, len(input.MemberIDs))
for i, memberID := range input.MemberIDs {
canonical, valid := canonicalID(memberID)
if !valid {
return invalidRequest()
}
if _, duplicate := members[canonical]; duplicate {
return invalidRequest()
}
members[canonical] = struct{}{}
input.MemberIDs[i] = canonical
}
if _, included := members[user.ID]; !included {
return invalidRequest()
}
chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
if errors.Is(err, model.ErrInvalidMembers) {
return invalidRequest()
}
if err != nil {
return unavailable()
}
writeJSON(w, http.StatusCreated, chat)
return nil
}
func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *apiError {
limit, cursor, err := pageParams(r, "chat", user.ID)
if err != nil {
return err
}
items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, model.ErrInvalidCursor) {
return invalidRequest()
}
if listErr != nil {
return unavailable()
}
if items == nil {
items = []Chat{}
}
var next *string
if hasNext {
last := items[len(items)-1]
value := encodeCursor("chat", user.ID, last.CreatedAt, last.ID)
next = &value
}
writeJSON(w, http.StatusOK, struct {
Items []Chat `json:"items"`
NextCursor *string `json:"next_cursor"`
}{items, next})
return nil
}
func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID)
if errors.Is(err, model.ErrNotFound) {
return notFound()
}
if err != nil {
return unavailable()
}
writeJSON(w, http.StatusOK, chat)
return nil
}
func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodPost && r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
if r.Method == http.MethodGet {
return s.listMessages(w, r, user, chatID)
}
key, valid := idempotencyKey(r)
if !valid {
return invalidRequest()
}
var input createMessageRequest
if err := readJSON(w, r, &input); err != nil {
return err
}
text, attachmentIDs, valid := input.values()
if !valid || !validMessage(text, attachmentIDs) || !canonicalAttachmentIDs(attachmentIDs) {
return invalidRequest()
}
message, replay, err := s.messageRepository.CreateMessage(
r.Context(), chatID, user.ID, text, attachmentIDs, key, s.idempotencyTTL,
)
if errors.Is(err, model.ErrNotFound) {
return notFound()
}
if errors.Is(err, model.ErrIdempotencyConflict) {
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
}
if err != nil {
return unavailable()
}
if replay {
s.metrics.replays.Inc()
}
writeJSON(w, http.StatusCreated, message)
return nil
}
func validMessage(text string, attachmentIDs []string) bool {
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 {
return false
}
return len(attachmentIDs) > 0 || strings.TrimSpace(text) != ""
}
func canonicalAttachmentIDs(attachmentIDs []string) bool {
if len(attachmentIDs) > 10 {
return false
}
seen := make(map[string]struct{}, len(attachmentIDs))
for index, attachmentID := range attachmentIDs {
canonical, valid := canonicalID(attachmentID)
if !valid {
return false
}
if _, duplicate := seen[canonical]; duplicate {
return false
}
seen[canonical] = struct{}{}
attachmentIDs[index] = canonical
}
return true
}
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
limit, cursor, err := pageParams(r, "message", chatID)
if err != nil {
return err
}
items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, model.ErrNotFound) {
return notFound()
}
if errors.Is(listErr, model.ErrInvalidCursor) {
return invalidRequest()
}
if listErr != nil {
return unavailable()
}
if items == nil {
items = []Message{}
}
var next *string
if hasNext {
last := items[len(items)-1]
value := encodeCursor("message", chatID, last.CreatedAt, last.ID)
next = &value
}
writeJSON(w, http.StatusOK, struct {
Items []Message `json:"items"`
NextCursor *string `json:"next_cursor"`
}{items, next})
return nil
}
func cursorPosition(cursor *pageCursor) *model.Position {
if cursor == nil {
return nil
}
return &model.Position{CreatedAt: cursor.Time, ID: cursor.ID}
}
@@ -0,0 +1,49 @@
package httpapi
import (
"context"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type UserRepository interface {
CreateUser(context.Context, string, string, string) (model.User, error)
UserByID(context.Context, string) (model.User, error)
UserByUsername(context.Context, string) (model.UserCredentials, error)
}
type ChatRepository interface {
CreateChat(context.Context, string, []string, string) (model.Chat, error)
ChatForUser(context.Context, string, string) (model.Chat, error)
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
}
type MessageRepository interface {
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
}
type AttachmentService interface {
Upload(context.Context, attachment.Upload) (model.Attachment, error)
Metadata(context.Context, string, string) (model.Attachment, error)
Content(context.Context, string, string) (attachment.Content, error)
}
type HealthChecker interface {
Ping(context.Context) error
}
type SessionStore interface {
HealthChecker
Create(context.Context, string, time.Duration) (string, time.Time, error)
UserID(context.Context, string) (string, error)
Delete(context.Context, string) error
}
type PasswordService interface {
Hash(string) (string, error)
Verify(string, string) (bool, error)
DummyHash() string
}
+114
View File
@@ -0,0 +1,114 @@
package httpapi
import (
"net/http"
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
attachmentUploads *prometheus.CounterVec
attachmentDuration prometheus.Histogram
attachmentBytes prometheus.Counter
handler http.Handler
}
func newMetrics() *metrics {
registry := prometheus.NewRegistry()
result := &metrics{
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_total",
Help: "Completed HTTP requests.",
}, []string{"method", "route", "status_class"}),
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "request_duration_seconds",
Help: "HTTP request duration in seconds.",
Buckets: prometheus.DefBuckets,
}, []string{"method", "route"}),
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_in_flight",
Help: "HTTP requests currently being served.",
}),
replays: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "idempotency_replays_total",
Help: "Message requests served from an idempotency record.",
}),
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_uploads_total",
Help: "Completed attachment uploads by result.",
}, []string{"result"}),
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "messenger",
Name: "attachment_upload_duration_seconds",
Help: "Attachment upload duration in seconds.",
Buckets: prometheus.DefBuckets,
}),
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_upload_bytes_total",
Help: "Bytes accepted by successful attachment uploads.",
}),
}
registry.MustRegister(
result.requests,
result.duration,
result.inFlight,
result.replays,
result.attachmentUploads,
result.attachmentDuration,
result.attachmentBytes,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return result
}
func (m *metrics) observe(r *http.Request, status int, started time.Time) {
route := routeTemplate(r.URL.Path)
m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc()
m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds())
}
func routeTemplate(path string) string {
switch path {
case "/health/live", "/health/ready", "/metrics", "/api/v1/users", "/api/v1/users/me",
"/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats":
return path
}
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" {
return "/api/v1/chats/{chat_id}"
}
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
return "/api/v1/chats/{chat_id}/messages"
}
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
return "/api/v1/chats/{chat_id}/attachments"
}
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
return "/api/v1/attachments/{attachment_id}"
}
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
return "/api/v1/attachments/{attachment_id}/content"
}
return "unknown"
}
+30
View File
@@ -0,0 +1,30 @@
package httpapi
import (
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"github.com/google/uuid"
)
type User = model.User
type Chat = model.Chat
type Message = model.Message
type Attachment = model.Attachment
func newID() (string, error) {
id, err := uuid.NewRandom()
if err != nil {
return "", err
}
return id.String(), nil
}
func canonicalID(value string) (string, bool) {
if len(value) != 36 {
return "", false
}
id, err := uuid.Parse(value)
if err != nil {
return "", false
}
return id.String(), true
}
+147
View File
@@ -0,0 +1,147 @@
package httpapi
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"io"
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"unicode/utf8"
)
const maxRequestBytes = 1 << 20
var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
type createMessageRequest struct {
Text json.RawMessage `json:"text"`
AttachmentIDs json.RawMessage `json:"attachment_ids"`
}
func (r createMessageRequest) values() (string, []string, bool) {
var text *string
if len(r.Text) == 0 || json.Unmarshal(r.Text, &text) != nil || text == nil {
return "", nil, false
}
if len(r.AttachmentIDs) == 0 {
return *text, nil, true
}
var attachmentIDs *[]string
if json.Unmarshal(r.AttachmentIDs, &attachmentIDs) != nil || attachmentIDs == nil {
return "", nil, false
}
return *text, *attachmentIDs, true
}
func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError {
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "expected application/json"}
}
r.Body = http.MaxBytesReader(w, r.Body, maxRequestBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "request body too large"}
}
return invalidRequest()
}
if !utf8.Valid(data) {
return invalidRequest()
}
var object map[string]json.RawMessage
if err := json.Unmarshal(data, &object); err != nil || object == nil {
return invalidRequest()
}
normalized, err := json.Marshal(object)
if err != nil {
return invalidRequest()
}
decoder := json.NewDecoder(bytes.NewReader(normalized))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return invalidRequest()
}
return nil
}
func validText(value string, maxRunes int) bool {
return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes
}
func idempotencyKey(r *http.Request) (string, bool) {
values := r.Header.Values("Idempotency-Key")
if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) {
return "", false
}
return values[0], true
}
type pageCursor struct {
Kind string `json:"kind"`
Resource string `json:"resource"`
Time time.Time `json:"time"`
ID string `json:"id"`
}
func pageParams(r *http.Request, kind, resource string) (int, *pageCursor, *apiError) {
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return 0, nil, invalidRequest()
}
for key := range query {
if key != "limit" && key != "cursor" {
return 0, nil, invalidRequest()
}
if len(query[key]) != 1 {
return 0, nil, invalidRequest()
}
}
limit := 50
if value, present := query["limit"]; present {
parsed, err := strconv.Atoi(value[0])
if err != nil || parsed < 1 || parsed > 100 {
return 0, nil, invalidRequest()
}
limit = parsed
}
if value, present := query["cursor"]; present {
cursor, valid := decodeCursor(value[0], kind, resource)
if !valid {
return 0, nil, invalidRequest()
}
return limit, cursor, nil
}
return limit, nil, nil
}
func encodeCursor(kind, resource string, createdAt time.Time, id string) string {
data, _ := json.Marshal(pageCursor{kind, resource, createdAt, id})
return base64.RawURLEncoding.EncodeToString(data)
}
func decodeCursor(value, kind, resource string) (*pageCursor, bool) {
data, err := base64.RawURLEncoding.DecodeString(value)
if err != nil {
return nil, false
}
var cursor pageCursor
if err := json.Unmarshal(data, &cursor); err != nil || cursor.Kind != kind || cursor.Resource != resource {
return nil, false
}
if cursor.Time.IsZero() {
return nil, false
}
if _, valid := canonicalID(cursor.ID); !valid {
return nil, false
}
return &cursor, true
}
+47
View File
@@ -0,0 +1,47 @@
package httpapi
import (
"net/http"
"strings"
)
type requestHandler func(http.ResponseWriter, *http.Request) *apiError
func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError {
var handler requestHandler
switch r.URL.Path {
case "/health/live":
handler = s.live
case "/health/ready":
handler = s.readiness
case "/metrics":
s.metrics.handler.ServeHTTP(w, r)
return nil
case "/api/v1/users":
handler = s.users
case "/api/v1/users/me":
handler = s.me
case "/api/v1/auth/sessions":
handler = s.createSession
case "/api/v1/auth/sessions/current":
handler = s.revokeSession
case "/api/v1/chats":
handler = s.chats
default:
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" {
handler = s.chat
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
handler = s.messages
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
handler = s.uploadAttachment
} else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
handler = s.attachmentMetadata
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
handler = s.attachmentContent
} else {
return notFound()
}
}
return handler(w, r)
}
+133
View File
@@ -0,0 +1,133 @@
package httpapi
import (
"context"
"log/slog"
"net/http"
"sync/atomic"
"time"
)
type Dependencies struct {
Users UserRepository
Chats ChatRepository
Messages MessageRepository
Attachments AttachmentService
Sessions SessionStore
Passwords PasswordService
Database HealthChecker
Objects HealthChecker
}
type Config struct {
SessionTTL time.Duration
IdempotencyTTL time.Duration
RequestTTL time.Duration
InstanceID string
SwaggerPort string
}
type Server struct {
userRepository UserRepository
chatRepository ChatRepository
messageRepository MessageRepository
attachments AttachmentService
sessions SessionStore
passwords PasswordService
database HealthChecker
objects HealthChecker
sessionTTL time.Duration
idempotencyTTL time.Duration
requestTTL time.Duration
instanceID string
logger *slog.Logger
swaggerPort string
metrics *metrics
ready atomic.Bool
}
type statusWriter struct {
http.ResponseWriter
status int
}
func (w *statusWriter) WriteHeader(status int) {
w.status = status
w.ResponseWriter.WriteHeader(status)
}
func NewServer(
logger *slog.Logger,
dependencies Dependencies,
config Config,
) *Server {
s := &Server{
userRepository: dependencies.Users, chatRepository: dependencies.Chats,
messageRepository: dependencies.Messages,
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
}
s.ready.Store(true)
return s
}
func (s *Server) SetReady(ready bool) {
s.ready.Store(ready)
}
func (s *Server) Handler() http.Handler {
return http.HandlerFunc(s.serve)
}
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", s.instanceID)
requestID, err := newID()
if err != nil {
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
return
}
w.Header().Set("X-Request-Id", requestID)
w.Header().Set("Cache-Control", "no-store")
wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK}
started := time.Now()
s.metrics.inFlight.Inc()
defer func() {
s.metrics.inFlight.Dec()
if recovered := recover(); recovered != nil {
s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path)
if wrapped.status == http.StatusOK {
writeError(wrapped, requestID, internalError())
}
}
s.metrics.observe(r, wrapped.status, started)
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
}()
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key")
wrapped.WriteHeader(http.StatusNoContent)
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.requestTTL)
defer cancel()
r = r.WithContext(ctx)
if apiErr := s.route(wrapped, r); apiErr != nil {
if apiErr.status == http.StatusUnauthorized {
wrapped.Header().Set("WWW-Authenticate", authenticationChallenge(r))
}
writeError(wrapped, requestID, apiErr)
}
}
func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool {
origin := r.Header.Get("Origin")
if origin != "http://localhost:"+s.swaggerPort && origin != "http://127.0.0.1:"+s.swaggerPort {
return false
}
w.Header().Set("Access-Control-Allow-Origin", origin)
w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id")
w.Header().Add("Vary", "Origin")
return true
}
+374
View File
@@ -0,0 +1,374 @@
package httpapi
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
var errDependency = errors.New("dependency unavailable")
type MessengerSuite struct {
suite.Suite
logger *slog.Logger
}
func TestMessengerSuite(t *testing.T) {
suite.Run(t, new(MessengerSuite))
}
func (s *MessengerSuite) SetupTest() {
s.logger = slog.New(slog.NewTextHandler(io.Discard, nil))
}
type messengerStoreStub struct {
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
}
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
func (messengerStoreStub) CreateUser(context.Context, string, string, string) (model.User, error) {
return model.User{}, nil
}
func (s messengerStoreStub) UserByID(context.Context, string) (model.User, error) {
return s.userByID, s.userByIDErr
}
func (s messengerStoreStub) UserByUsername(context.Context, string) (model.UserCredentials, error) {
return s.credentials, s.credentialsErr
}
func (messengerStoreStub) CreateChat(context.Context, string, []string, string) (model.Chat, error) {
return model.Chat{}, nil
}
func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Chat, error) {
return model.Chat{}, nil
}
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
return nil, false, nil
}
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) {
return s.message, s.messageReplay, s.messageErr
}
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
return nil, false, nil
}
type objectStoreStub struct {
pingErr error
}
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
type attachmentServiceStub struct {
uploaded model.Attachment
err error
}
func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) {
return s.uploaded, s.err
}
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
return s.uploaded, s.err
}
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
}
type sessionStub struct {
pingErr error
userID string
userIDErr error
}
func (s sessionStub) Ping(context.Context) error { return s.pingErr }
func (sessionStub) Create(context.Context, string, time.Duration) (string, time.Time, error) {
return "token", time.Now().Add(time.Hour), nil
}
func (s sessionStub) UserID(context.Context, string) (string, error) { return s.userID, s.userIDErr }
func (sessionStub) Delete(context.Context, string) error { return nil }
type passwordsStub struct {
verified bool
err error
}
func (passwordsStub) Hash(string) (string, error) { return "hash", nil }
func (passwordsStub) DummyHash() string { return "hash" }
func (s passwordsStub) Verify(string, string) (bool, error) {
return s.verified, s.err
}
func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
t := s.T()
tests := []struct {
name string
store messengerStoreStub
sessions sessionStub
objects objectStoreStub
}{
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
{name: "s3", objects: objectStoreStub{pingErr: errDependency}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{})
response := serveRequest(server, http.MethodGet, "/health/ready", "")
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
}
})
}
}
func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() {
t := s.T()
tests := []struct {
name string
store messengerStoreStub
sessions sessionStub
}{
{name: "valkey", sessions: sessionStub{userIDErr: errDependency}},
{name: "postgres", store: messengerStoreStub{userByIDErr: errDependency}, sessions: sessionStub{userID: "user-id"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)
request.Header.Set("Authorization", "Bearer valid-token")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
}
})
}
}
func (s *MessengerSuite) TestAuthenticationChallenges() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "")
if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` {
t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate"))
}
store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}}
server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false})
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil)
request.SetBasicAuth("user", "wrong-password")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusUnauthorized || response.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` {
t.Fatalf("session challenge = %d %q", response.Code, response.Header().Get("WWW-Authenticate"))
}
}
func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() {
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
response := serveRequest(
server,
http.MethodGet,
"/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid",
"",
)
s.Equal(http.StatusUnauthorized, response.Code)
s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate"))
}
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}"))
request.ContentLength = -1
request.SetBasicAuth("user", "password")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
}
}
func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
}
}
func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
for _, response := range []*httptest.ResponseRecorder{
serveRequest(server, http.MethodGet, "/health/live", ""),
serveRequest(server, http.MethodGet, "/api/v1/users/me", ""),
} {
if value := response.Header().Get("X-Instance-Id"); value != "test-instance" {
t.Fatalf("X-Instance-Id = %q, want test-instance", value)
}
}
}
func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() {
t := s.T()
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
tests := []struct {
name string
key string
}{
{name: "missing"},
{name: "invalid", key: "invalid key"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/json")
if tt.key != "" {
request.Header.Set("Idempotency-Key", tt.key)
}
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest)
}
})
}
}
func (s *MessengerSuite) TestMessageCreationValidatesFieldPresenceAndTypes() {
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
attachmentID := "00000000-0000-0000-0000-000000000002"
invalidBodies := []string{
`{"attachment_ids":["` + attachmentID + `"]}`,
`{"text":null,"attachment_ids":["` + attachmentID + `"]}`,
`{"text":"x","attachment_ids":null}`,
`{"text":1}`,
`{"text":"x","attachment_ids":{}}`,
}
for _, body := range invalidBodies {
response := createMessageRequestResponse(server, body)
s.Equal(http.StatusBadRequest, response.Code, body)
}
validBodies := []string{
`{"text":"message"}`,
`{"text":"message","attachment_ids":[]}`,
`{"text":"","attachment_ids":["` + attachmentID + `"]}`,
}
for _, body := range validBodies {
response := createMessageRequestResponse(server, body)
s.Equal(http.StatusCreated, response.Code, body)
}
}
func createMessageRequestResponse(server *Server, body string) *httptest.ResponseRecorder {
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000001/messages",
strings.NewReader(body),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", "message-key")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
return response
}
func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() {
t := s.T()
store := messengerStoreStub{
userByID: model.User{ID: "user-id"},
messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict),
}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", "same-key")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
if response.Code != http.StatusConflict {
t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict)
}
}
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
s.False(valid, filename)
}
filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}})
s.True(valid)
s.Equal("фото.png", filename)
}
func (s *MessengerSuite) TestUploadRejectsMalformedQuery() {
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG",
strings.NewReader("image"),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
s.Equal(http.StatusBadRequest, response.Code)
}
func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server {
return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{})
}
func (s *MessengerSuite) server(
store messengerStoreStub,
sessions SessionStore,
passwords PasswordService,
objects objectStoreStub,
attachments AttachmentService,
) *Server {
return NewServer(s.logger, Dependencies{
Users: store, Chats: store, Messages: store, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: store, Objects: objects,
}, Config{
SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second,
InstanceID: "test-instance", SwaggerPort: "8081",
})
}
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
request := httptest.NewRequest(method, path, strings.NewReader(body))
if body != "" {
request.Header.Set("Content-Type", "application/json")
}
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
return response
}
+18
View File
@@ -0,0 +1,18 @@
package model
import "errors"
var (
ErrNotFound = errors.New("not found")
ErrConflict = errors.New("conflict")
ErrInvalidMembers = errors.New("invalid members")
ErrInvalidCursor = errors.New("invalid cursor")
ErrIdempotencyConflict = errors.New("idempotency conflict")
ErrInvalidToken = errors.New("invalid token")
ErrSessionNotFound = errors.New("session not found")
ErrInvalidInput = errors.New("invalid input")
ErrUnsupportedMedia = errors.New("unsupported media")
ErrPayloadTooLarge = errors.New("payload too large")
ErrCapacity = errors.New("capacity exhausted")
ErrUnavailable = errors.New("dependency unavailable")
)
+153
View File
@@ -0,0 +1,153 @@
package model
import (
"encoding/json"
"time"
)
const (
AttachmentPending = "pending"
AttachmentReady = "ready"
AttachmentDeleting = "deleting"
)
const timestampLayout = "2006-01-02T15:04:05.000000Z"
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt time.Time `json:"created_at"`
}
type UserCredentials struct {
User
PasswordHash string `json:"-"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt time.Time `json:"created_at"`
}
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
ReadyAt *time.Time `json:"-"`
Referenced bool `json:"-"`
}
type StoredObject struct {
Key string
Size int64
LastModified time.Time
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Width int `json:"width"`
Height int `json:"height"`
}
type Position struct {
CreatedAt time.Time
ID string
}
type userJSON struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt string `json:"created_at"`
}
type chatJSON struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt string `json:"created_at"`
}
type messageJSON struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt string `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type attachmentJSON struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
func (u User) MarshalJSON() ([]byte, error) {
return json.Marshal(userJSON{u.ID, u.Username, u.DisplayName, formatTime(u.CreatedAt)})
}
func (c Chat) MarshalJSON() ([]byte, error) {
return json.Marshal(chatJSON{c.ID, c.Title, c.MemberIDs, formatTime(c.CreatedAt)})
}
func (m Message) MarshalJSON() ([]byte, error) {
attachments := m.AttachmentIDs
if attachments == nil {
attachments = []string{}
}
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments})
}
func (a Attachment) MarshalJSON() ([]byte, error) {
variants := a.Variants
if variants == nil {
variants = []Variant{}
}
return json.Marshal(attachmentJSON{
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt),
})
}
func FormatTime(value time.Time) string {
return formatTime(value)
}
func formatTime(value time.Time) string {
return value.UTC().Format(timestampLayout)
}
+24
View File
@@ -0,0 +1,24 @@
package model
import (
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type ModelSuite struct {
suite.Suite
}
func TestModelSuite(t *testing.T) {
suite.Run(t, new(ModelSuite))
}
func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() {
createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC)
data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt})
s.Require().NoError(err)
s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data))
}
+176
View File
@@ -0,0 +1,176 @@
package objectstore
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
var ErrNotFound = errors.New("object not found")
type Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
UseTLS bool
Timeout time.Duration
}
type Store struct {
client *minio.Client
bucket string
timeout time.Duration
}
func New(config Config) (*Store, error) {
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
if err != nil {
return nil, err
}
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
return nil, errors.New("invalid object store configuration")
}
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
Secure: config.UseTLS,
Region: config.Region,
})
if err != nil {
return nil, fmt.Errorf("create S3 client: %w", err)
}
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
}
func endpointHost(value string, useTLS bool) (string, error) {
value = strings.TrimSpace(value)
if !strings.Contains(value, "://") {
if value == "" {
return "", errors.New("S3 endpoint is required")
}
return value, nil
}
parsed, err := url.Parse(value)
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
return "", errors.New("invalid S3 endpoint")
}
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
}
return parsed.Host, nil
}
func (s *Store) Ping(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
exists, err := s.client.BucketExists(ctx, s.bucket)
if err != nil {
return fmt.Errorf("check S3 bucket: %w", err)
}
if !exists {
return errors.New("S3 bucket does not exist")
}
return nil
}
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
_, err := s.client.PutObject(
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
)
if err != nil {
return fmt.Errorf("put S3 object: %w", err)
}
return nil
}
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
cancel()
return nil, objectError("get S3 object", err)
}
if _, err := object.Stat(); err != nil {
cancel()
_ = object.Close()
return nil, objectError("stat S3 object", err)
}
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
}
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return model.StoredObject{}, objectError("stat S3 object", err)
}
return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
}
func (s *Store) Delete(ctx context.Context, key string) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
return objectError("delete S3 object", err)
}
return nil
}
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", errors.New("object list limit must be between 1 and 1000")
}
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
objects := make([]model.StoredObject, 0, limit+1)
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
for info := range s.client.ListObjects(ctx, s.bucket, options) {
if info.Err != nil {
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
}
objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
if len(objects) == limit+1 {
cancel()
break
}
}
var next string
if len(objects) > limit {
objects = objects[:limit]
next = objects[len(objects)-1].Key
}
return objects, next, nil
}
type timeoutReader struct {
io.ReadCloser
cancel context.CancelFunc
}
func (r *timeoutReader) Close() error {
err := r.ReadCloser.Close()
r.cancel()
return err
}
func objectError(action string, err error) error {
response := minio.ToErrorResponse(err)
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
return ErrNotFound
}
return fmt.Errorf("%s: %w", action, err)
}
+133
View File
@@ -0,0 +1,133 @@
package password
import (
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"errors"
"fmt"
"strconv"
"strings"
"golang.org/x/crypto/argon2"
)
var (
ErrInvalidConfig = errors.New("invalid password configuration")
ErrInvalidHash = errors.New("invalid password hash")
)
type Config struct {
MemoryKiB uint32
Iterations uint32
Parallelism uint8
SaltLength uint32
KeyLength uint32
}
type Hasher struct {
config Config
dummyHash string
}
func New(config Config) (*Hasher, error) {
if err := validateConfig(config); err != nil {
return nil, err
}
salt := make([]byte, config.SaltLength)
digest := argon2.IDKey([]byte("invalid-user-password"), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength)
return &Hasher{config: config, dummyHash: encode(config, salt, digest)}, nil
}
func validateConfig(config Config) error {
if config.MemoryKiB < 8*1024 || config.MemoryKiB > 1024*1024 ||
config.Iterations < 1 || config.Iterations > 20 ||
config.Parallelism < 1 ||
config.SaltLength < 8 || config.SaltLength > 1024 ||
config.KeyLength < 16 || config.KeyLength > 1024 {
return ErrInvalidConfig
}
return nil
}
func (h *Hasher) Hash(value string) (string, error) {
salt := make([]byte, h.config.SaltLength)
if _, err := rand.Read(salt); err != nil {
return "", fmt.Errorf("generate password salt: %w", err)
}
digest := argon2.IDKey([]byte(value), salt, h.config.Iterations, h.config.MemoryKiB, h.config.Parallelism, h.config.KeyLength)
return encode(h.config, salt, digest), nil
}
func (h *Hasher) Verify(value, encodedHash string) (bool, error) {
config, salt, digest, err := decode(encodedHash)
if err != nil {
return false, err
}
actual := argon2.IDKey([]byte(value), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength)
return subtle.ConstantTimeCompare(actual, digest) == 1, nil
}
func (h *Hasher) DummyHash() string {
return h.dummyHash
}
func encode(config Config, salt, digest []byte) string {
return fmt.Sprintf(
"$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
argon2.Version,
config.MemoryKiB,
config.Iterations,
config.Parallelism,
base64.RawStdEncoding.EncodeToString(salt),
base64.RawStdEncoding.EncodeToString(digest),
)
}
func decode(encodedHash string) (Config, []byte, []byte, error) {
parts := strings.Split(encodedHash, "$")
if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" || parts[2] != fmt.Sprintf("v=%d", argon2.Version) {
return Config{}, nil, nil, ErrInvalidHash
}
parameters := strings.Split(parts[3], ",")
if len(parameters) != 3 {
return Config{}, nil, nil, ErrInvalidHash
}
memory, err := parameter(parameters[0], "m=", 32)
if err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
iterations, err := parameter(parameters[1], "t=", 32)
if err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
parallelism, err := parameter(parameters[2], "p=", 8)
if err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
salt, err := base64.RawStdEncoding.Strict().DecodeString(parts[4])
if err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
digest, err := base64.RawStdEncoding.Strict().DecodeString(parts[5])
if err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
config := Config{
MemoryKiB: uint32(memory), Iterations: uint32(iterations), Parallelism: uint8(parallelism),
SaltLength: uint32(len(salt)), KeyLength: uint32(len(digest)),
}
if err := validateConfig(config); err != nil {
return Config{}, nil, nil, ErrInvalidHash
}
return config, salt, digest, nil
}
func parameter(value, prefix string, bitSize int) (uint64, error) {
if !strings.HasPrefix(value, prefix) {
return 0, ErrInvalidHash
}
return strconv.ParseUint(strings.TrimPrefix(value, prefix), 10, bitSize)
}
+44
View File
@@ -0,0 +1,44 @@
package password
import (
"testing"
"github.com/stretchr/testify/suite"
)
type PasswordSuite struct {
suite.Suite
hasher *Hasher
}
func TestPasswordSuite(t *testing.T) {
suite.Run(t, new(PasswordSuite))
}
func (s *PasswordSuite) SetupTest() {
hasher, err := New(testConfig())
s.Require().NoError(err)
s.hasher = hasher
}
func (s *PasswordSuite) TestHasher() {
hash, err := s.hasher.Hash("T3st-password-123")
s.Require().NoError(err)
verified, err := s.hasher.Verify("T3st-password-123", hash)
s.Require().NoError(err)
s.True(verified)
verified, err = s.hasher.Verify("wrong-password", hash)
s.Require().NoError(err)
s.False(verified)
}
func (s *PasswordSuite) TestVerifyRejectsMalformedHash() {
_, err := s.hasher.Verify("password", "invalid")
s.Error(err)
}
func testConfig() Config {
return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32}
}
+254
View File
@@ -0,0 +1,254 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
const attachmentColumns = `
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, created_at, updated_at, ready_at`
type rowScanner interface {
Scan(...any) error
}
func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
query := `
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at
)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL
WHERE EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
)
RETURNING ` + attachmentColumns
err := scanAttachment(s.pool.QueryRow(
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
query := `
UPDATE attachments
SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE id = $1 AND object_key = $2 AND status = 'pending'
RETURNING ` + attachmentColumns
var attachment model.Attachment
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error {
const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'`
if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil {
return fmt.Errorf("delete pending attachment: %w", err)
}
return nil
}
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
query := `
SELECT ` + attachmentColumnsWithAlias("a") + `
FROM attachments a
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
WHERE a.id = $1 AND a.status = 'ready'`
var attachment model.Attachment
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
var attachment model.Attachment
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
var exists bool
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
return false, fmt.Errorf("check attachment by object key: %w", err)
}
return exists, nil
}
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
}
query := `
SELECT ` + attachmentColumnsWithAlias("a") + `,
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
FROM attachments a
WHERE a.object_key > $1
ORDER BY a.object_key
LIMIT $2`
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
if err != nil {
return nil, "", fmt.Errorf("list attachments: %w", err)
}
defer rows.Close()
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
var attachment model.Attachment
err := scanAttachmentWithReference(row, &attachment)
return publicAttachment(attachment), err
}))
if err != nil {
return nil, "", fmt.Errorf("scan attachments: %w", err)
}
var next string
if len(attachments) > limit {
attachments = attachments[:limit]
next = attachments[len(attachments)-1].ObjectKey
}
return attachments, next, nil
}
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
&attachment.Referenced,
)
}
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
var referenced bool
if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil {
return false, fmt.Errorf("check attachment references: %w", err)
}
return referenced, nil
}
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
var status string
var createdAt time.Time
var readyAt *time.Time
err = tx.QueryRow(
ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID,
).Scan(&status, &createdAt, &readyAt)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
}
if status != candidate.Status {
return false, nil
}
var age time.Time
switch status {
case model.AttachmentPending:
age = createdAt
case model.AttachmentReady:
if readyAt == nil {
return false, nil
}
age = *readyAt
default:
return false, nil
}
if age.After(createdBefore) {
return false, nil
}
var referenced bool
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil {
return false, fmt.Errorf("recheck attachment references: %w", err)
}
if referenced {
return false, nil
}
if _, err := tx.Exec(
ctx,
`UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`,
candidate.ID, candidate.Status,
); err != nil {
return false, fmt.Errorf("mark attachment deleting: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return false, fmt.Errorf("commit attachment deletion claim: %w", err)
}
return true, nil
}
func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) {
const query = `
DELETE FROM attachments a
WHERE a.id = $1
AND a.status = 'deleting'
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
result, err := s.pool.Exec(ctx, query, attachmentID)
if err != nil {
return false, fmt.Errorf("delete claimed attachment: %w", err)
}
return result.RowsAffected() == 1, nil
}
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
)
}
func attachmentColumnsWithAlias(alias string) string {
return fmt.Sprintf(
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
alias,
)
}
func publicAttachment(attachment model.Attachment) model.Attachment {
attachment.Variants = []model.Variant{}
return attachment
}
+11
View File
@@ -0,0 +1,11 @@
package postgres
import "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
var (
ErrConflict = model.ErrConflict
ErrNotFound = model.ErrNotFound
ErrInvalidMembers = model.ErrInvalidMembers
ErrInvalidCursor = model.ErrInvalidCursor
ErrIdempotencyConflict = model.ErrIdempotencyConflict
)
+364
View File
@@ -0,0 +1,364 @@
package postgres
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
const expiredIdempotencyCleanupBatch = 100
func (s *Store) CreateMessage(
ctx context.Context,
chatID string,
senderID string,
text string,
attachmentIDs []string,
idempotencyKey string,
idempotencyTTL time.Duration,
) (model.Message, bool, error) {
if attachmentIDs == nil {
attachmentIDs = []string{}
}
var (
tx pgx.Tx
err error
visible bool
requestHash [sha256.Size]byte
message model.Message
replay bool
)
tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID)
if err != nil {
return model.Message{}, false, err
}
if !visible {
return model.Message{}, false, ErrNotFound
}
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
if err != nil {
return model.Message{}, false, err
}
if !validAttachments {
return model.Message{}, false, ErrNotFound
}
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
return model.Message{}, false, err
}
requestHash, err = messageRequestHash(text, attachmentIDs)
if err != nil {
return model.Message{}, false, err
}
message, replay, err = createIdempotentMessage(
ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL,
)
if err != nil {
return model.Message{}, false, err
}
if err := tx.Commit(ctx); err != nil {
return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err)
}
return message, replay, nil
}
func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) {
if attachmentIDs == nil {
attachmentIDs = []string{}
}
payload, err := json.Marshal(struct {
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}{text, attachmentIDs})
if err != nil {
return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err)
}
return sha256.Sum256(payload), nil
}
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
if len(attachmentIDs) == 0 {
return true, nil
}
const query = `
SELECT id
FROM attachments
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
FOR UPDATE`
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
if err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
}
defer rows.Close()
count := 0
for rows.Next() {
count++
}
if err := rows.Err(); err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
}
return count == len(attachmentIDs), nil
}
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM chat_members
WHERE chat_id = $1 AND user_id = $2
)`
var visible bool
if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil {
return false, fmt.Errorf("check chat visibility: %w", err)
}
return visible, nil
}
func createIdempotentMessage(
ctx context.Context,
tx pgx.Tx,
chatID string,
senderID string,
text string,
attachmentIDs []string,
idempotencyKey string,
requestHash []byte,
idempotencyTTL time.Duration,
) (model.Message, bool, error) {
for {
var (
record idempotencyRecord
found bool
err error
messageID string
reserved bool
message model.Message
)
record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey)
if err != nil {
return model.Message{}, false, err
}
if found && record.active {
if !bytes.Equal(record.requestHash, requestHash) {
return model.Message{}, false, ErrIdempotencyConflict
}
message, err = messageByID(ctx, tx, record.messageID)
return message, true, err
}
if found {
if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil {
return model.Message{}, false, err
}
}
messageID = uuid.NewString()
reserved, err = reserveIdempotencyKey(
ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL,
)
if err != nil {
return model.Message{}, false, err
}
if !reserved {
continue
}
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs)
return message, false, err
}
}
func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error {
const query = `
WITH expired AS (
SELECT ctid
FROM message_idempotency
WHERE expires_at <= statement_timestamp()
ORDER BY expires_at
LIMIT $1
FOR UPDATE SKIP LOCKED
)
DELETE FROM message_idempotency records
USING expired
WHERE records.ctid = expired.ctid`
if _, err := tx.Exec(ctx, query, limit); err != nil {
return fmt.Errorf("delete expired idempotency records: %w", err)
}
return nil
}
type idempotencyRecord struct {
requestHash []byte
messageID string
active bool
}
func lockIdempotencyRecord(
ctx context.Context,
tx pgx.Tx,
userID string,
chatID string,
key string,
) (idempotencyRecord, bool, error) {
const query = `
SELECT request_hash, message_id, expires_at > statement_timestamp()
FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3
FOR UPDATE`
var (
record idempotencyRecord
err error
)
err = tx.QueryRow(ctx, query, userID, chatID, key).Scan(
&record.requestHash,
&record.messageID,
&record.active,
)
if errors.Is(err, pgx.ErrNoRows) {
return idempotencyRecord{}, false, nil
}
if err != nil {
return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err)
}
return record, true, nil
}
func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error {
const query = `
DELETE FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3`
if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil {
return fmt.Errorf("delete expired idempotency key: %w", err)
}
return nil
}
func reserveIdempotencyKey(
ctx context.Context,
tx pgx.Tx,
userID string,
chatID string,
key string,
requestHash []byte,
messageID string,
ttl time.Duration,
) (bool, error) {
const query = `
INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at)
VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second')
ON CONFLICT (user_id, chat_id, key) DO NOTHING
RETURNING true`
var (
inserted bool
err error
)
err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("reserve idempotency key: %w", err)
}
return inserted, nil
}
func insertMessage(
ctx context.Context,
tx pgx.Tx,
messageID string,
chatID string,
senderID string,
text string,
attachmentIDs []string,
) (model.Message, error) {
const query = `
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES ($1, $2, $3, $4)
RETURNING id, chat_id, sender_id, text, created_at`
var (
message model.Message
err error
)
err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
)
if err != nil {
return model.Message{}, fmt.Errorf("insert message: %w", err)
}
if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil {
return model.Message{}, err
}
message.AttachmentIDs = make([]string, len(attachmentIDs))
copy(message.AttachmentIDs, attachmentIDs)
return message, nil
}
func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error {
const query = `
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES ($1, $2, $3, $4)`
for position, attachmentID := range attachmentIDs {
if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil {
return fmt.Errorf("insert message attachment: %w", err)
}
}
return nil
}
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
const query = `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id::text
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
)
FROM messages m
WHERE m.id = $1`
var (
message model.Message
err error
)
err = tx.QueryRow(ctx, query, messageID).Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, fmt.Errorf("idempotency result message not found")
}
if err != nil {
return model.Message{}, fmt.Errorf("get idempotency result message: %w", err)
}
return message, nil
}
+442
View File
@@ -0,0 +1,442 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type Store struct {
pool *pgxpool.Pool
}
func New(pool *pgxpool.Pool) *Store {
return &Store{pool: pool}
}
func (s *Store) Ping(ctx context.Context) error {
return s.pool.Ping(ctx)
}
func (s *Store) CreateUser(
ctx context.Context,
username string,
displayName string,
passwordHash string,
) (model.User, error) {
const query = `
INSERT INTO users (id, username, display_name, password_hash)
VALUES ($1, $2, $3, $4)
RETURNING id, username, display_name, created_at`
var user model.User
err := s.pool.QueryRow(ctx, query, uuid.NewString(), username, displayName, passwordHash).Scan(
&user.ID,
&user.Username,
&user.DisplayName,
&user.CreatedAt,
)
if isUniqueViolation(err) {
return model.User{}, ErrConflict
}
if err != nil {
return model.User{}, fmt.Errorf("create user: %w", err)
}
return user, nil
}
func (s *Store) UserByID(ctx context.Context, id string) (model.User, error) {
const query = `
SELECT id, username, display_name, created_at
FROM users
WHERE id = $1`
var user model.User
err := s.pool.QueryRow(ctx, query, id).Scan(
&user.ID,
&user.Username,
&user.DisplayName,
&user.CreatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.User{}, ErrNotFound
}
if err != nil {
return model.User{}, fmt.Errorf("get user by id: %w", err)
}
return user, nil
}
func (s *Store) UserByUsername(ctx context.Context, username string) (model.UserCredentials, error) {
const query = `
SELECT id, username, display_name, created_at, password_hash
FROM users
WHERE username = $1`
var credentials model.UserCredentials
err := s.pool.QueryRow(ctx, query, username).Scan(
&credentials.ID,
&credentials.Username,
&credentials.DisplayName,
&credentials.CreatedAt,
&credentials.PasswordHash,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.UserCredentials{}, ErrNotFound
}
if err != nil {
return model.UserCredentials{}, fmt.Errorf("get user by username: %w", err)
}
return credentials, nil
}
func (s *Store) CreateChat(
ctx context.Context,
title string,
memberIDs []string,
callerID string,
) (model.Chat, error) {
if !validMembers(memberIDs, callerID) {
return model.Chat{}, ErrInvalidMembers
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
valid, err := allUsersExist(ctx, tx, memberIDs)
if err != nil {
return model.Chat{}, err
}
if !valid {
return model.Chat{}, ErrInvalidMembers
}
chat, err := insertChat(ctx, tx, title)
if err != nil {
return model.Chat{}, err
}
if err := insertMembers(ctx, tx, chat.ID, memberIDs); err != nil {
return model.Chat{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.Chat{}, fmt.Errorf("commit create chat transaction: %w", err)
}
chat.MemberIDs = append([]string(nil), memberIDs...)
return chat, nil
}
func validMembers(memberIDs []string, callerID string) bool {
callerUUID, err := uuid.Parse(callerID)
if err != nil {
return false
}
seen := make(map[uuid.UUID]struct{}, len(memberIDs))
callerIncluded := false
for _, memberID := range memberIDs {
memberUUID, err := uuid.Parse(memberID)
if err != nil {
return false
}
if _, exists := seen[memberUUID]; exists {
return false
}
seen[memberUUID] = struct{}{}
callerIncluded = callerIncluded || memberUUID == callerUUID
}
return callerIncluded
}
func allUsersExist(ctx context.Context, tx pgx.Tx, memberIDs []string) (bool, error) {
const query = `
SELECT count(*)
FROM users
WHERE id::text = ANY($1)`
var count int
if err := tx.QueryRow(ctx, query, memberIDs).Scan(&count); err != nil {
return false, fmt.Errorf("check chat members: %w", err)
}
return count == len(memberIDs), nil
}
func insertChat(ctx context.Context, tx pgx.Tx, title string) (model.Chat, error) {
const query = `
INSERT INTO chats (id, title)
VALUES ($1, $2)
RETURNING id, title, created_at`
var chat model.Chat
if err := tx.QueryRow(ctx, query, uuid.NewString(), title).Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil {
return model.Chat{}, fmt.Errorf("insert chat: %w", err)
}
return chat, nil
}
func insertMembers(ctx context.Context, tx pgx.Tx, chatID string, memberIDs []string) error {
const query = `
INSERT INTO chat_members (chat_id, user_id, position)
VALUES ($1, $2, $3)`
batch := &pgx.Batch{}
for position, memberID := range memberIDs {
batch.Queue(query, chatID, memberID, position)
}
results := tx.SendBatch(ctx, batch)
defer func() { _ = results.Close() }()
for range memberIDs {
if _, err := results.Exec(); err != nil {
return fmt.Errorf("insert chat member: %w", err)
}
}
if err := results.Close(); err != nil {
return fmt.Errorf("close chat members batch: %w", err)
}
return nil
}
func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (model.Chat, error) {
const query = `
SELECT c.id, c.title, c.created_at,
array_agg(cm.user_id::text ORDER BY cm.position)
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE c.id = $1
AND EXISTS (
SELECT 1
FROM chat_members visible
WHERE visible.chat_id = c.id AND visible.user_id = $2
)
GROUP BY c.id, c.title, c.created_at`
var chat model.Chat
err := s.pool.QueryRow(ctx, query, chatID, userID).Scan(
&chat.ID,
&chat.Title,
&chat.CreatedAt,
&chat.MemberIDs,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Chat{}, ErrNotFound
}
if err != nil {
return model.Chat{}, fmt.Errorf("get chat for user: %w", err)
}
return chat, nil
}
func (s *Store) ListChats(
ctx context.Context,
userID string,
after *model.Position,
limit int,
) ([]model.Chat, bool, error) {
if after != nil {
valid, err := s.visibleChatPositionExists(ctx, userID, *after)
if err != nil {
return nil, false, err
}
if !valid {
return nil, false, ErrInvalidCursor
}
}
const query = `
SELECT c.id, c.title, c.created_at,
array_agg(all_members.user_id::text ORDER BY all_members.position)
FROM chats c
JOIN chat_members visible ON visible.chat_id = c.id AND visible.user_id = $1
JOIN chat_members all_members ON all_members.chat_id = c.id
WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid))
GROUP BY c.id, c.title, c.created_at
ORDER BY c.created_at, c.id
LIMIT $4`
var afterTime any
var afterID any
if after != nil {
afterTime = after.CreatedAt
afterID = after.ID
}
rows, err := s.pool.Query(ctx, query, userID, afterTime, afterID, limit+1)
if err != nil {
return nil, false, fmt.Errorf("list chats: %w", err)
}
defer rows.Close()
chats, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Chat](func(row pgx.CollectableRow) (model.Chat, error) {
var chat model.Chat
err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs)
return chat, err
}))
if err != nil {
return nil, false, fmt.Errorf("scan chats: %w", err)
}
return page(chats, limit)
}
func (s *Store) visibleChatPositionExists(
ctx context.Context,
userID string,
position model.Position,
) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE cm.user_id = $1 AND c.created_at = $2 AND c.id = $3
)`
var exists bool
if err := s.pool.QueryRow(ctx, query, userID, position.CreatedAt, position.ID).Scan(&exists); err != nil {
return false, fmt.Errorf("validate chat cursor: %w", err)
}
return exists, nil
}
func (s *Store) ListMessages(
ctx context.Context,
chatID string,
userID string,
after *model.Position,
limit int,
) ([]model.Message, bool, error) {
visible, err := s.chatVisible(ctx, chatID, userID)
if err != nil {
return nil, false, err
}
if !visible {
return nil, false, ErrNotFound
}
if after != nil {
valid, err := s.messagePositionExists(ctx, chatID, *after)
if err != nil {
return nil, false, err
}
if !valid {
return nil, false, ErrInvalidCursor
}
}
const query = `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id::text
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
)
FROM messages m
WHERE m.chat_id = $1
AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid))
ORDER BY m.created_at, m.id
LIMIT $4`
var afterTime any
var afterID any
if after != nil {
afterTime = after.CreatedAt
afterID = after.ID
}
rows, err := s.pool.Query(ctx, query, chatID, afterTime, afterID, limit+1)
if err != nil {
return nil, false, fmt.Errorf("list messages: %w", err)
}
defer rows.Close()
messages, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Message](func(row pgx.CollectableRow) (model.Message, error) {
var message model.Message
err := row.Scan(
&message.ID,
&message.ChatID,
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
)
return message, err
}))
if err != nil {
return nil, false, fmt.Errorf("scan messages: %w", err)
}
return page(messages, limit)
}
func (s *Store) chatVisible(ctx context.Context, chatID, userID string) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM chat_members
WHERE chat_id = $1 AND user_id = $2
)`
var exists bool
if err := s.pool.QueryRow(ctx, query, chatID, userID).Scan(&exists); err != nil {
return false, fmt.Errorf("check chat visibility: %w", err)
}
return exists, nil
}
func (s *Store) messagePositionExists(
ctx context.Context,
chatID string,
position model.Position,
) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM messages
WHERE chat_id = $1 AND created_at = $2 AND id = $3
)`
var exists bool
if err := s.pool.QueryRow(ctx, query, chatID, position.CreatedAt, position.ID).Scan(&exists); err != nil {
return false, fmt.Errorf("validate message cursor: %w", err)
}
return exists, nil
}
func page[T any](items []T, limit int) ([]T, bool, error) {
if limit < 1 {
return nil, false, fmt.Errorf("limit must be positive")
}
if len(items) <= limit {
return items, false, nil
}
return items[:limit], true, nil
}
func isUniqueViolation(err error) bool {
var postgresError *pgconn.PgError
return errors.As(err, &postgresError) && postgresError.Code == "23505"
}
+439
View File
@@ -0,0 +1,439 @@
package postgres_test
import (
"context"
"errors"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
)
type PostgresSuite struct {
suite.Suite
pool *pgxpool.Pool
store *postgres.Store
}
func TestPostgresSuite(t *testing.T) {
suite.Run(t, new(PostgresSuite))
}
func (s *PostgresSuite) SetupSuite() {
s.pool = openTestDatabase(s.T())
s.store = postgres.New(s.pool)
}
func (s *PostgresSuite) TearDownSuite() {
if s.pool != nil {
s.pool.Close()
}
}
func (s *PostgresSuite) TestConcurrentUsernameCreation() {
t := s.T()
store := s.store
username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16]
const workers = 8
results := make(chan error, workers)
var group sync.WaitGroup
for range workers {
group.Add(1)
go func() {
defer group.Done()
_, err := store.CreateUser(context.Background(), username, "Race user", "password-hash")
results <- err
}()
}
group.Wait()
close(results)
var created, conflicts int
for err := range results {
switch {
case err == nil:
created++
case errors.Is(err, postgres.ErrConflict):
conflicts++
default:
t.Fatalf("CreateUser() error = %v", err)
}
}
if created != 1 || conflicts != workers-1 {
t.Fatalf("created = %d, conflicts = %d", created, conflicts)
}
}
func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() {
t := s.T()
pool := s.pool
store := s.store
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
}
before := chatCount(t, pool)
_, err = store.CreateChat(context.Background(), "Invalid chat", []string{owner.ID, uuid.NewString()}, owner.ID)
if !errors.Is(err, postgres.ErrInvalidMembers) {
t.Fatalf("CreateChat() error = %v, want ErrInvalidMembers", err)
}
if after := chatCount(t, pool); after != before {
t.Fatalf("chat count changed from %d to %d", before, after)
}
}
func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
t := s.T()
pool := s.pool
store := s.store
owner, chatID := createOwnerAndChat(t, store)
key := uuid.NewString()
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour)
if err != nil || replay {
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
}
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour)
if err != nil || !replay {
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
}
s.Equal(first, second)
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
}
if count := messageCount(t, pool, chatID); count != 1 {
t.Fatalf("message count = %d, want 1", count)
}
}
func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash")
s.Require().NoError(err)
attachment := model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
}
pending, err := s.store.CreatePendingAttachment(context.Background(), attachment)
s.Require().NoError(err)
created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
s.Equal("ready", created.Status)
s.Empty(created.Variants)
s.Nil(created.ErrorCode)
visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID)
s.Require().NoError(err)
s.Equal(created, visible)
_, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID)
s.ErrorIs(err, postgres.ErrNotFound)
message, replay, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour,
)
s.Require().NoError(err)
s.False(replay)
s.Equal([]string{created.ID}, message.AttachmentIDs)
messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10)
s.Require().NoError(err)
s.Len(messages, 1)
s.Equal([]string{created.ID}, messages[0].AttachmentIDs)
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
s.Require().NoError(err)
s.True(referenced)
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(deleted)
}
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.True(claimed)
deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID)
s.Require().NoError(err)
s.True(deleted)
pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(claimed)
visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID)
s.Require().NoError(err)
s.Equal(model.AttachmentReady, visible.Status)
}
func (s *PostgresSuite) TestReconciliationDryRunPreservesDeletingAttachment() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "deleting.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("e", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.True(claimed)
reconciler, err := reconcile.New(s.store, emptyObjectStore{}, reconcile.Config{
Prefix: "attachments/", PageSize: 1000, GraceTime: time.Minute,
UnreferencedRetention: time.Hour,
})
s.Require().NoError(err)
report, err := reconciler.Run(context.Background())
s.Require().NoError(err)
s.Zero(report.DeletedUnreferencedRows)
var status string
err = s.pool.QueryRow(context.Background(), "SELECT status FROM attachments WHERE id = $1", pending.ID).Scan(&status)
s.Require().NoError(err)
s.Equal(model.AttachmentDeleting, status)
}
type emptyObjectStore struct{}
func (emptyObjectStore) List(context.Context, string, string, int) ([]model.StoredObject, string, error) {
return nil, "", nil
}
func (emptyObjectStore) Delete(context.Context, string) error {
return errors.New("unexpected object deletion")
}
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
s.Require().NoError(err)
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
_, _, err = s.store.CreateMessage(
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
)
s.ErrorIs(err, postgres.ErrNotFound)
}
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
t := s.T()
pool := s.pool
stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)}
owner, chatID := createOwnerAndChat(t, stores[0])
key := uuid.NewString()
const workers = 8
type result struct {
message model.Message
replay bool
err error
}
results := make(chan result, workers)
var group sync.WaitGroup
for worker := range workers {
group.Add(1)
go func() {
defer group.Done()
message, replay, err := stores[worker%len(stores)].CreateMessage(
context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour,
)
results <- result{message: message, replay: replay, err: err}
}()
}
group.Wait()
close(results)
var messageID string
var created int
for result := range results {
if result.err != nil {
t.Fatalf("CreateMessage() error = %v", result.err)
}
if messageID == "" {
messageID = result.message.ID
}
if result.message.ID != messageID {
t.Fatalf("message ID = %s, want %s", result.message.ID, messageID)
}
if !result.replay {
created++
}
}
if created != 1 {
t.Fatalf("new message results = %d, want 1", created)
}
if count := messageCount(t, pool, chatID); count != 1 {
t.Fatalf("message count = %d, want 1", count)
}
}
func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
t := s.T()
pool := s.pool
store := s.store
owner, chatID := createOwnerAndChat(t, store)
second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
}
chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID)
if err != nil {
t.Fatalf("CreateChat() error = %v", err)
}
key := uuid.NewString()
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil {
t.Fatalf("owner CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil {
t.Fatalf("second user CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil {
t.Fatalf("other chat CreateMessage() error = %v", err)
}
if _, err := pool.Exec(context.Background(), `
UPDATE message_idempotency
SET created_at = clock_timestamp() - INTERVAL '2 days',
expires_at = clock_timestamp() - INTERVAL '1 day'
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
t.Fatalf("expire idempotency key: %v", err)
}
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour)
if err != nil || replay || message.Text != "after expiry" {
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
}
}
func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
t := s.T()
owner, chatID := createOwnerAndChat(t, s.store)
expiredKey := uuid.NewString()
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil {
t.Fatalf("create expired message: %v", err)
}
if _, err := s.pool.Exec(context.Background(), `
UPDATE message_idempotency
SET created_at = clock_timestamp() - INTERVAL '2 days',
expires_at = clock_timestamp() - INTERVAL '1 day'
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil {
t.Fatalf("expire idempotency key: %v", err)
}
if _, _, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour,
); err != nil {
t.Fatalf("create new message: %v", err)
}
var remaining int
if err := s.pool.QueryRow(context.Background(), `
SELECT count(*)
FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil {
t.Fatalf("count expired idempotency records: %v", err)
}
if remaining != 0 {
t.Fatalf("expired idempotency records = %d, want 0", remaining)
}
}
func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) {
t.Helper()
owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash")
if err != nil {
t.Fatalf("CreateUser() error = %v", err)
}
chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID)
if err != nil {
t.Fatalf("CreateChat() error = %v", err)
}
return owner, chat.ID
}
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
t.Helper()
var count int
if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil {
t.Fatalf("count messages: %v", err)
}
return count
}
func openTestDatabase(t *testing.T) *pgxpool.Pool {
t.Helper()
url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE")
pool, err := database.Open(context.Background(), database.Config{
URL: url, MinConnections: 0, MaxConnections: 10,
MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute,
HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second,
})
if err != nil {
t.Fatalf("database.Open() error = %v", err)
}
if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil {
pool.Close()
t.Fatalf("database.Migrate() error = %v", err)
}
return pool
}
func testSecret(t *testing.T, valueName, fileName string) string {
t.Helper()
if value := strings.TrimSpace(os.Getenv(valueName)); value != "" {
return value
}
path := strings.TrimSpace(os.Getenv(fileName))
if path == "" {
t.Skipf("set %s or %s to run integration tests", valueName, fileName)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", fileName, err)
}
return strings.TrimSpace(string(data))
}
func uniqueUsername() string {
return "u_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
}
func chatCount(t *testing.T, pool *pgxpool.Pool) int {
t.Helper()
var count int
if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM chats").Scan(&count); err != nil {
t.Fatalf("count chats: %v", err)
}
return count
}
+354
View File
@@ -0,0 +1,354 @@
package reconcile
import (
"context"
"errors"
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type MetadataStore interface {
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
}
type ObjectStore interface {
List(context.Context, string, string, int) ([]model.StoredObject, string, error)
Delete(context.Context, string) error
}
type Config struct {
Prefix string
PageSize int
GraceTime time.Duration
UnreferencedRetention time.Duration
Apply bool
Now func() time.Time
}
type Report struct {
MetadataScanned int `json:"metadata_scanned"`
ObjectsScanned int `json:"objects_scanned"`
Consistent int `json:"consistent"`
FreshPendingAttachments int `json:"fresh_pending_attachments"`
OldPendingAttachments int `json:"old_pending_attachments"`
DeletingAttachments int `json:"deleting_attachments"`
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
FreshOrphanObjects int `json:"fresh_orphan_objects"`
OldOrphanObjects int `json:"old_orphan_objects"`
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
FreshMissingObjects int `json:"fresh_missing_objects"`
ReferencedMissingObjects int `json:"referenced_missing_objects"`
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
}
type Reconciler struct {
metadata MetadataStore
objects ObjectStore
config Config
}
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
if config.PageSize < 1 || config.PageSize > 1000 {
return nil, errors.New("page size must be between 1 and 1000")
}
if config.GraceTime < 0 {
return nil, errors.New("grace time must not be negative")
}
if config.UnreferencedRetention <= 0 {
return nil, errors.New("unreferenced retention must be positive")
}
if config.Now == nil {
config.Now = time.Now
}
return &Reconciler{metadata: metadata, objects: objects, config: config}, nil
}
func (r *Reconciler) Run(ctx context.Context) (Report, error) {
metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize)
if err != nil {
return Report{}, err
}
objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize)
if err != nil {
return Report{}, err
}
now := r.config.Now()
anomalyCutoff := now.Add(-r.config.GraceTime)
retentionCutoff := now.Add(-r.config.UnreferencedRetention)
var report Report
for metadata.current != nil || objects.current != nil {
switch {
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil {
return report, err
}
if err := objects.advance(ctx); err != nil {
return report, err
}
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil {
return report, err
}
if err := metadata.advance(ctx); err != nil {
return report, err
}
default:
if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil {
return report, err
}
if err := metadata.advance(ctx); err != nil {
return report, err
}
if err := objects.advance(ctx); err != nil {
return report, err
}
}
}
return report, nil
}
func (r *Reconciler) handlePair(
ctx context.Context,
metadata model.Attachment,
retentionCutoff time.Time,
anomalyCutoff time.Time,
report *Report,
) error {
report.MetadataScanned++
report.ObjectsScanned++
switch metadata.Status {
case model.AttachmentPending:
if metadata.CreatedAt.After(anomalyCutoff) {
report.FreshPendingAttachments++
report.Consistent++
return nil
}
report.OldPendingAttachments++
return r.deletePair(ctx, metadata, anomalyCutoff, report)
case model.AttachmentDeleting:
report.DeletingAttachments++
return r.deleteClaimedPair(ctx, metadata, report)
case model.AttachmentReady:
if metadata.Referenced {
report.Consistent++
return nil
}
age := metadata.CreatedAt
if metadata.ReadyAt != nil {
age = *metadata.ReadyAt
}
if age.After(retentionCutoff) {
report.FreshUnreferencedAttachments++
report.Consistent++
return nil
}
report.OldUnreferencedAttachments++
return r.deletePair(ctx, metadata, retentionCutoff, report)
default:
return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status)
}
}
func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error {
if !r.config.Apply {
return nil
}
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff)
if err != nil {
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
}
if !claimed {
return nil
}
return r.deleteClaimedPair(ctx, attachment, report)
}
func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error {
if !r.config.Apply {
return nil
}
if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err)
}
report.DeletedAttachmentObjects++
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID)
if err != nil {
return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err)
}
if deleted {
report.DeletedUnreferencedRows++
}
return nil
}
func (r *Reconciler) handleOrphanObject(
ctx context.Context,
object model.StoredObject,
cutoff time.Time,
report *Report,
) error {
report.ObjectsScanned++
if object.LastModified.After(cutoff) {
report.FreshOrphanObjects++
return nil
}
report.OldOrphanObjects++
if !r.config.Apply {
return nil
}
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
if err != nil {
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
}
if exists {
return nil
}
if err := r.objects.Delete(ctx, object.Key); err != nil {
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
}
report.DeletedOrphanObjects++
return nil
}
func (r *Reconciler) handleMissingObject(
ctx context.Context,
metadata model.Attachment,
cutoff time.Time,
report *Report,
) error {
report.MetadataScanned++
if metadata.Status == model.AttachmentDeleting {
report.DeletingAttachments++
return r.deleteClaimedMetadata(ctx, metadata, report)
}
if metadata.CreatedAt.After(cutoff) {
report.FreshMissingObjects++
return nil
}
if metadata.Referenced {
report.ReferencedMissingObjects++
return nil
}
report.OldUnreferencedMetadata++
if !r.config.Apply {
return nil
}
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff)
if err != nil {
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
}
if !claimed {
return nil
}
return r.deleteClaimedMetadata(ctx, metadata, report)
}
func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error {
if !r.config.Apply {
return nil
}
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
if err != nil {
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
}
if deleted {
report.DeletedUnreferencedRows++
}
return nil
}
type metadataIterator struct {
store MetadataStore
pageSize int
after string
page []model.Attachment
index int
current *model.Attachment
done bool
}
func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) {
iterator := &metadataIterator{store: store, pageSize: pageSize}
if err := iterator.advance(ctx); err != nil {
return nil, err
}
return iterator, nil
}
func (i *metadataIterator) advance(ctx context.Context) error {
for {
if i.index < len(i.page) {
i.current = &i.page[i.index]
i.index++
return nil
}
if i.done {
i.current = nil
return nil
}
page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize)
if err != nil {
return fmt.Errorf("list attachment metadata: %w", err)
}
i.page = page
i.index = 0
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return errors.New("metadata page is empty but has continuation")
}
}
}
type objectIterator struct {
store ObjectStore
prefix string
pageSize int
after string
page []model.StoredObject
index int
current *model.StoredObject
done bool
}
func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) {
iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize}
if err := iterator.advance(ctx); err != nil {
return nil, err
}
return iterator, nil
}
func (i *objectIterator) advance(ctx context.Context) error {
for {
if i.index < len(i.page) {
i.current = &i.page[i.index]
i.index++
return nil
}
if i.done {
i.current = nil
return nil
}
page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize)
if err != nil {
return fmt.Errorf("list objects: %w", err)
}
i.page = page
i.index = 0
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return errors.New("object page is empty but has continuation")
}
}
}
+228
View File
@@ -0,0 +1,228 @@
package reconcile
import (
"context"
"sort"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
old := now.Add(-48 * time.Hour)
fresh := now.Add(-10 * time.Minute)
metadata := &metadataStub{items: map[string]model.Attachment{
"attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false),
"attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true),
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
}}
objects := &objectStub{items: map[string]model.StoredObject{
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
"attachments/b/original": {Key: "attachments/b/original", LastModified: old},
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
}}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Equal(t, 1, report.OldPendingAttachments)
require.Equal(t, 1, report.OldUnreferencedAttachments)
require.Equal(t, 1, report.FreshUnreferencedAttachments)
require.Equal(t, 1, report.DeletedOrphanObjects)
require.Equal(t, 2, report.DeletedAttachmentObjects)
require.Equal(t, 3, report.DeletedUnreferencedRows)
require.Contains(t, metadata.items, "attachments/b/original")
require.Contains(t, metadata.items, "attachments/d/original")
require.NotContains(t, metadata.items, "attachments/a/original")
require.NotContains(t, metadata.items, "attachments/c/original")
require.NotContains(t, metadata.items, "attachments/missing/original")
require.NotContains(t, objects.items, "attachments/orphan/original")
repeated, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Zero(t, repeated.DeletedOrphanObjects)
require.Zero(t, repeated.DeletedAttachmentObjects)
require.Zero(t, repeated.DeletedUnreferencedRows)
}
func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) {
now := time.Now().UTC()
metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}}
objects := &objectStub{items: map[string]model.StoredObject{
"attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)},
}}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Zero(t, report.DeletedOrphanObjects)
require.Contains(t, objects.items, "attachments/a/original")
}
func TestReconcilerDeletingAttachmentRespectsApplyMode(t *testing.T) {
now := time.Now().UTC()
key := "attachments/a/original"
tests := []struct {
name string
apply bool
objectPresent bool
}{
{name: "dry run with object", objectPresent: true},
{name: "dry run without object"},
{name: "apply with object", apply: true, objectPresent: true},
{name: "apply without object", apply: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
metadata := &metadataStub{items: map[string]model.Attachment{
key: attachment("a", key, model.AttachmentDeleting, now.Add(-time.Hour), false),
}}
objects := &objectStub{items: make(map[string]model.StoredObject)}
if test.objectPresent {
objects.items[key] = model.StoredObject{Key: key, LastModified: now.Add(-time.Hour)}
}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
UnreferencedRetention: time.Hour, Apply: test.apply, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Equal(t, 1, report.DeletingAttachments)
var expectedDeletedRows, expectedDeletedObjects int
if test.apply {
expectedDeletedRows = 1
}
if test.apply && test.objectPresent {
expectedDeletedObjects = 1
}
require.Equal(t, expectedDeletedRows, report.DeletedUnreferencedRows)
require.Equal(t, expectedDeletedObjects, report.DeletedAttachmentObjects)
_, metadataExists := metadata.items[key]
require.Equal(t, !test.apply, metadataExists)
_, objectExists := objects.items[key]
require.Equal(t, test.objectPresent && !test.apply, objectExists)
})
}
}
func TestNewRejectsInvalidRetention(t *testing.T) {
_, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1})
require.Error(t, err)
}
func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment {
readyAt := createdAt
return model.Attachment{
ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt,
ReadyAt: &readyAt, Referenced: referenced,
}
}
type metadataStub struct {
items map[string]model.Attachment
appeared map[string]bool
}
func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) {
keys := make([]string, 0, len(s.items))
for key := range s.items {
if key > after {
keys = append(keys, key)
}
}
sort.Strings(keys)
var next string
if len(keys) > limit {
keys = keys[:limit]
next = keys[len(keys)-1]
}
items := make([]model.Attachment, 0, len(keys))
for _, key := range keys {
items = append(items, s.items[key])
}
return items, next, nil
}
func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) {
if s.appeared[key] {
return true, nil
}
_, exists := s.items[key]
return exists, nil
}
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
for key, item := range s.items {
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
continue
}
age := item.CreatedAt
if item.Status == model.AttachmentReady && item.ReadyAt != nil {
age = *item.ReadyAt
}
if age.After(cutoff) {
return false, nil
}
item.Status = model.AttachmentDeleting
s.items[key] = item
return true, nil
}
return false, nil
}
func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) {
for key, item := range s.items {
if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced {
delete(s.items, key)
return true, nil
}
}
return false, nil
}
type objectStub struct {
items map[string]model.StoredObject
}
func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
keys := make([]string, 0, len(s.items))
for key := range s.items {
if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix {
keys = append(keys, key)
}
}
sort.Strings(keys)
var next string
if len(keys) > limit {
keys = keys[:limit]
next = keys[len(keys)-1]
}
items := make([]model.StoredObject, 0, len(keys))
for _, key := range keys {
items = append(items, s.items[key])
}
return items, next, nil
}
func (s *objectStub) Delete(_ context.Context, key string) error {
delete(s.items, key)
return nil
}
+163
View File
@@ -0,0 +1,163 @@
package session
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/valkey-io/valkey-go"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
const (
tokenLength = 32
keyPrefix = "session:"
)
var (
ErrNotFound = model.ErrSessionNotFound
ErrInvalidToken = model.ErrInvalidToken
ErrInvalidRecord = errors.New("invalid session record")
ErrInvalidTTL = errors.New("invalid session TTL")
ErrTokenGeneration = errors.New("session token generation failed")
ErrStore = model.ErrUnavailable
)
type record struct {
UserID string `json:"user_id"`
ExpiresAt time.Time `json:"expires_at"`
}
type Store struct {
client valkey.Client
}
func New(rawURL string) (*Store, error) {
option, err := valkey.ParseURL(rawURL)
if err != nil {
return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err)
}
client, err := valkey.NewClient(option)
if err != nil {
return nil, fmt.Errorf("%w: create Valkey client: %w", ErrStore, err)
}
return &Store{client: client}, nil
}
func (s *Store) Create(ctx context.Context, userID string, ttl time.Duration) (string, time.Time, error) {
if strings.TrimSpace(userID) == "" {
return "", time.Time{}, ErrInvalidRecord
}
if ttl < time.Second {
return "", time.Time{}, ErrInvalidTTL
}
token, err := newToken()
if err != nil {
return "", time.Time{}, err
}
record := record{
UserID: userID,
ExpiresAt: time.Now().UTC().Add(ttl).Truncate(time.Microsecond),
}
payload, err := json.Marshal(record)
if err != nil {
return "", time.Time{}, fmt.Errorf("%w: encode session: %w", ErrStore, err)
}
command := s.client.B().Set().Key(s.key(token)).Value(valkey.BinaryString(payload)).Ex(ttl).Build()
if err := s.client.Do(ctx, command).Error(); err != nil {
return "", time.Time{}, fmt.Errorf("%w: create session: %w", ErrStore, err)
}
return token, record.ExpiresAt, nil
}
func newToken() (string, error) {
random := make([]byte, tokenLength)
if _, err := rand.Read(random); err != nil {
return "", fmt.Errorf("%w: %v", ErrTokenGeneration, err)
}
return base64.RawURLEncoding.EncodeToString(random), nil
}
func (s *Store) UserID(ctx context.Context, token string) (string, error) {
key, err := s.sessionKey(token)
if err != nil {
return "", err
}
payload, err := s.client.Do(ctx, s.client.B().Get().Key(key).Build()).AsBytes()
if valkey.IsValkeyNil(err) {
return "", ErrNotFound
}
if err != nil {
return "", fmt.Errorf("%w: get session: %w", ErrStore, err)
}
var record record
if err := json.Unmarshal(payload, &record); err != nil {
return "", fmt.Errorf("%w: decode session: %w", ErrStore, err)
}
if strings.TrimSpace(record.UserID) == "" || record.ExpiresAt.IsZero() {
return "", fmt.Errorf("%w: %w", ErrStore, ErrInvalidRecord)
}
if !record.ExpiresAt.After(time.Now()) {
return "", ErrNotFound
}
return record.UserID, nil
}
func (s *Store) Delete(ctx context.Context, token string) error {
key, err := s.sessionKey(token)
if err != nil {
return err
}
deleted, err := s.client.Do(ctx, s.client.B().Del().Key(key).Build()).AsInt64()
if err != nil {
return fmt.Errorf("%w: delete session: %w", ErrStore, err)
}
if deleted == 0 {
return ErrNotFound
}
return nil
}
func (s *Store) Ping(ctx context.Context) error {
if err := s.client.Do(ctx, s.client.B().Ping().Build()).Error(); err != nil {
return fmt.Errorf("%w: ping Valkey: %w", ErrStore, err)
}
return nil
}
func (s *Store) Close() error {
s.client.Close()
return nil
}
func (s *Store) sessionKey(token string) (string, error) {
decoded, err := base64.RawURLEncoding.Strict().DecodeString(token)
if err != nil || len(decoded) != tokenLength {
return "", ErrInvalidToken
}
return s.key(token), nil
}
func (s *Store) key(token string) string {
digest := sha256.Sum256([]byte(token))
return keyPrefix + hex.EncodeToString(digest[:])
}
@@ -0,0 +1,72 @@
package session_test
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
type SessionSuite struct {
suite.Suite
store *session.Store
}
func TestSessionSuite(t *testing.T) {
suite.Run(t, new(SessionSuite))
}
func (s *SessionSuite) SetupSuite() {
rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL"))
if rawURL == "" {
path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE"))
if path == "" {
s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests")
}
data, err := os.ReadFile(path)
s.Require().NoError(err)
rawURL = strings.TrimSpace(string(data))
}
store, err := session.New(rawURL)
s.Require().NoError(err)
s.store = store
}
func (s *SessionSuite) TearDownSuite() {
if s.store != nil {
s.Require().NoError(s.store.Close())
}
}
func (s *SessionSuite) TestSessionLifecycle() {
userID := uuid.NewString()
token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute)
s.Require().NoError(err)
s.NotEmpty(token)
s.True(expiresAt.After(time.Now()))
actualUserID, err := s.store.UserID(context.Background(), token)
s.Require().NoError(err)
s.Equal(userID, actualUserID)
s.Require().NoError(s.store.Delete(context.Background(), token))
_, err = s.store.UserID(context.Background(), token)
s.ErrorIs(err, session.ErrNotFound)
}
func (s *SessionSuite) TestSessionExpires() {
token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second)
s.Require().NoError(err)
s.Eventually(func() bool {
_, err = s.store.UserID(context.Background(), token)
return errors.Is(err, session.ErrNotFound)
}, 3*time.Second, 50*time.Millisecond)
}
+100
View File
@@ -0,0 +1,100 @@
# Адрес и внутренний порт HTTP-сервера.
APP_HOST=0.0.0.0
APP_PORT=8080
# Адреса публикации сервисов на локальной машине.
API_BIND_ADDRESS=127.0.0.1:8080
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
# Время жизни сессии, идемпотентности и используемое хранилище.
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
SESSION_STORE=valkey
# Число API-реплик за Nginx.
API_REPLICAS=2
# Учётные данные локального PostgreSQL и строка подключения API.
POSTGRES_USER=messenger
POSTGRES_DB=messenger
POSTGRES_PASSWORD=local-postgres-password
DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable
# Пароль локального Valkey и строка подключения API.
VALKEY_PASSWORD=local-valkey-password
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
# Административные данные MinIO используются только при bootstrap.
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=local-minio-admin-password
# API использует отдельного пользователя с доступом только к bucket вложений.
S3_ENDPOINT=s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY=messenger-app
S3_SECRET_KEY=local-messenger-s3-password
S3_USE_TLS=false
S3_TIMEOUT_SECONDS=5
ATTACHMENT_MAX_BYTES=10485760
ATTACHMENT_MAX_PIXELS=40000000
ATTACHMENT_BODY_MEMORY_BYTES=67108864
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
# Размер пула и обслуживание соединений PostgreSQL.
DB_MIN_CONNS=1
DB_MAX_CONNS=10
DB_MAX_CONNECTION_AGE_SECONDS=1800
DB_MAX_CONNECTION_IDLE_SECONDS=300
DB_HEALTH_CHECK_SECONDS=30
# Таймауты зависимостей, миграций и HTTP-запросов.
DEPENDENCY_TIMEOUT_SECONDS=5
MIGRATION_TIMEOUT_SECONDS=30
REQUEST_TIMEOUT_SECONDS=5
HTTP_READ_HEADER_TIMEOUT_SECONDS=5
HTTP_READ_TIMEOUT_SECONDS=10
HTTP_WRITE_TIMEOUT_SECONDS=10
HTTP_IDLE_TIMEOUT_SECONDS=60
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
# Параметры Argon2id для новых хешей паролей.
PASSWORD_MEMORY_KIB=65536
PASSWORD_ITERATIONS=3
PASSWORD_PARALLELISM=2
PASSWORD_SALT_LENGTH=16
PASSWORD_KEY_LENGTH=32
# Порт Swagger внутри конфигурации CORS API.
SWAGGER_PORT=8081
# Ограничения ресурсов контейнеров.
API_CPUS=1
API_MEMORY=512m
MIGRATE_CPUS=0.5
MIGRATE_MEMORY=128m
NGINX_CPUS=0.5
NGINX_MEMORY=128m
SWAGGER_CPUS=0.25
SWAGGER_MEMORY=128m
TEST_CPUS=2
TEST_MEMORY=512m
EXPORTER_CPUS=0.25
EXPORTER_MEMORY=128m
PROMETHEUS_CPUS=1
PROMETHEUS_MEMORY=512m
GRAFANA_CPUS=0.5
GRAFANA_MEMORY=256m
POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
VALKEY_MEMORY=256m
S3_CPUS=1
S3_MEMORY=512m
S3_BOOTSTRAP_CPUS=0.25
S3_BOOTSTRAP_MEMORY=256m
# Пароль локального администратора Grafana.
GRAFANA_ADMIN_PASSWORD=local-grafana-password
+50
View File
@@ -0,0 +1,50 @@
CREATE TABLE users (
id UUID PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_unique UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100),
CONSTRAINT users_display_name_not_blank CHECK (display_name ~ '[^[:space:]]'),
CONSTRAINT users_password_hash_length CHECK (char_length(password_hash) BETWEEN 1 AND 1024),
CONSTRAINT users_password_hash_not_blank CHECK (password_hash ~ '[^[:space:]]')
);
CREATE TABLE chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100),
CONSTRAINT chats_title_not_blank CHECK (title ~ '[^[:space:]]')
);
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id UUID NOT NULL,
user_id UUID NOT NULL,
position SMALLINT NOT NULL,
CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_chat_position_unique UNIQUE (chat_id, position),
CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99),
CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE,
CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE
);
CREATE INDEX chat_members_user_id_chat_id_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
sender_id UUID NOT NULL,
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000),
CONSTRAINT messages_text_not_blank CHECK (text ~ '[^[:space:]]'),
CONSTRAINT messages_sender_membership_fk FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE
);
CREATE INDEX messages_chat_id_created_at_id_idx ON messages (chat_id, created_at, id);
+19
View File
@@ -0,0 +1,19 @@
CREATE TABLE message_idempotency (
user_id UUID NOT NULL,
chat_id UUID NOT NULL,
key TEXT NOT NULL,
request_hash BYTEA NOT NULL,
message_id UUID NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
expires_at TIMESTAMPTZ NOT NULL,
CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key),
CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE,
CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id)
ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32),
CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at)
);
CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at);
+47
View File
@@ -0,0 +1,47 @@
ALTER TABLE messages
DROP CONSTRAINT messages_text_length,
DROP CONSTRAINT messages_text_not_blank;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000),
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
owner_id UUID NOT NULL,
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
);
CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id UUID NOT NULL,
chat_id UUID NOT NULL,
attachment_id UUID NOT NULL,
position SMALLINT NOT NULL,
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id)
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
REFERENCES attachments (id, chat_id) ON DELETE RESTRICT
);
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);
+16
View File
@@ -0,0 +1,16 @@
ALTER TABLE attachments
ADD COLUMN status TEXT NOT NULL DEFAULT 'ready',
ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp();
UPDATE attachments
SET ready_at = created_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')),
ADD CONSTRAINT attachments_ready_at CHECK (
(status = 'pending' AND ready_at IS NULL) OR
(status IN ('ready', 'deleting') AND ready_at IS NOT NULL)
);
CREATE INDEX attachments_status_created_at_object_key_idx
ON attachments (status, created_at, object_key);
@@ -0,0 +1,9 @@
ALTER TABLE attachments
DROP CONSTRAINT attachments_ready_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_ready_at CHECK (
(status = 'pending' AND ready_at IS NULL) OR
(status = 'ready' AND ready_at IS NOT NULL) OR
status = 'deleting'
);
+8
View File
@@ -0,0 +1,8 @@
package migrations
import "embed"
// FS contains database migrations applied by the application at startup.
//
//go:embed *.sql
var FS embed.FS
@@ -0,0 +1,143 @@
{
"annotations": {"list": []},
"editable": false,
"graphTooltip": 1,
"panels": [
{
"type": "timeseries", "title": "API requests per second", "id": 1,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}]
},
{
"type": "timeseries", "title": "API RPS by replica", "id": 2,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API responses by status class", "id": 3,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Healthy API replicas", "id": 4,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}]
},
{
"type": "timeseries", "title": "API latency p95 by replica and route", "id": 5,
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 16},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API in flight", "id": 6,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Go heap by replica", "id": 7,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Idempotency replays", "id": 8,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
"targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}]
},
{
"type": "timeseries", "title": "Nginx connections and requests", "id": 9,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
"targets": [
{"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"},
{"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL connections", "id": 10,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
"targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}]
},
{
"type": "timeseries", "title": "PostgreSQL transactions", "id": 11,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 40},
"targets": [
{"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"},
{"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL database size", "id": 12,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 40},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey memory", "id": 13,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 48},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey keys and commands", "id": 14,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 48},
"targets": [
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "Attachment uploads by result", "id": 15,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 56},
"targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Attachment upload latency p95", "id": 16,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 56},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}]
},
{
"type": "timeseries", "title": "Attachment upload traffic", "id": 17,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 64},
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
"targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO objects", "id": 18,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 64},
"targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO requests and errors", "id": 19,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 64},
"targets": [
{"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"},
{"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "MinIO availability", "id": 20,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 72},
"targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO storage", "id": 21,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 72},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [
{"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"},
{"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"}
]
}
],
"refresh": "5s",
"schemaVersion": 41,
"tags": ["highload", "lab4", "nginx", "failover"],
"templating": {"list": []},
"time": {"from": "now-15m", "to": "now"},
"timezone": "browser",
"title": "Messenger lab4 overview",
"uid": "messenger-overview",
"version": 3
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
folder: Highload
type: file
disableDeletion: true
editable: false
options:
path: /etc/grafana/dashboards
@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
+28
View File
@@ -0,0 +1,28 @@
global:
scrape_interval: 1s
evaluation_interval: 1s
scrape_configs:
- job_name: messenger
dns_sd_configs:
- names: ["api"]
type: A
port: 8080
refresh_interval: 2s
- job_name: nginx
static_configs:
- targets: ["nginx-exporter:9113"]
- job_name: postgres
static_configs:
- targets: ["postgres-exporter:9187"]
- job_name: valkey
static_configs:
- targets: ["valkey-exporter:9121"]
- job_name: minio
metrics_path: /minio/v2/metrics/cluster
static_configs:
- targets: ["s3:9000"]
+57
View File
@@ -0,0 +1,57 @@
user nginx;
worker_processes auto;
error_log /dev/stderr notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
resolver 127.0.0.11 valid=2s ipv6=off;
log_format structured escape=json
'{"time":"$time_iso8601","request_id":"$http_x_request_id",'
'"method":"$request_method","uri":"$request_uri","status":$status,'
'"request_time":$request_time,"upstream":"$upstream_addr",'
'"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}';
access_log /dev/stdout structured;
upstream messenger {
zone messenger 64k;
include /etc/nginx/conf.d/upstream.conf;
keepalive 32;
}
server {
listen 8080;
client_max_body_size 11m;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_request_buffering off;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 3s;
}
}
server {
listen 8081;
access_log off;
location = /stub_status {
stub_status;
}
}
}
+1
View File
@@ -0,0 +1 @@
server api:8080 resolve max_fails=1 fail_timeout=3s;
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
docker compose stop nginx api valkey postgres s3
docker compose start postgres valkey s3
docker compose run --rm s3-bootstrap
docker compose run --rm migrate
docker compose up -d --wait --wait-timeout 90
+223
View File
@@ -0,0 +1,223 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
printf 'usage: %s <image-manifest>\n' "$0" >&2
exit 2
fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"}
current_manifest=$deploy_dir/current-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4}
switched=false
nginx_created=false
upstream_changed=false
old_pool=""
mkdir -p "$deploy_dir"
: > "$candidate_file"
export NGINX_UPSTREAM_FILE=$upstream_file
load_manifest() {
set -a
# shellcheck disable=SC1090
. "$1"
set +a
}
compose() {
docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@"
}
container_name() {
name=$(docker inspect --format '{{.Name}}' "$1")
printf '%s\n' "${name#/}"
}
running_container() {
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
}
write_upstream() {
output=$1
containers=$2
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
wait_for_containers() {
containers=$1
deadline=$(($(date +%s) + 120))
while [ "$(date +%s)" -lt "$deadline" ]; do
count=0
healthy=0
for container in $containers; do
count=$((count + 1))
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
if [ "$status" = healthy ]; then
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
return 0
fi
sleep 1
done
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
return 1
}
smoke_check() {
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
}
remove_containers() {
containers=$1
for container in $containers; do
docker rm -f "$container" >/dev/null 2>&1 || true
done
}
restore_upstream() {
if [ -s "$upstream_backup" ]; then
cp "$upstream_backup" "$upstream_file"
compose exec -T nginx nginx -t
compose exec -T nginx nginx -s reload
fi
}
rollback() {
status=$?
trap - EXIT INT TERM
candidates=$(cat "$candidate_file" 2>/dev/null || true)
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
restore_upstream || true
if [ "$switched" = true ]; then
smoke_check || true
fi
elif [ "$nginx_created" = true ]; then
compose stop nginx >/dev/null 2>&1 || true
fi
remove_containers "$candidates"
exit "$status"
}
trap rollback EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
rm -f "$previous_containers"
fi
if [ -s "$active_containers" ]; then
for container in $(cat "$active_containers"); do
if running_container "$container"; then
old_pool="$old_pool $container"
fi
done
fi
if [ -z "$old_pool" ]; then
old_pool=$(compose ps -q api)
fi
if [ -s "$upstream_file" ]; then
:
elif [ -n "$old_pool" ]; then
write_upstream "$upstream_file" "$old_pool"
else
cp nginx/upstream.conf "$upstream_file"
fi
cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
nginx_container=$(compose ps -q nginx)
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
compose up -d --no-build --no-deps --force-recreate nginx
nginx_container=$(compose ps -q nginx)
fi
if [ -n "$nginx_container" ]; then
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
exit 1
fi
fi
API_IMAGE=$new_api_image
export API_IMAGE
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
if [ -z "$short_digest" ]; then
short_digest=$(date +%s)
fi
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
candidate_pool=""
index=1
while [ "$index" -le "$replicas" ]; do
name="${project}-api-${short_digest}-${generation}-${index}"
container=$(compose run -d --no-deps --name "$name" api)
candidate_pool="$candidate_pool $container"
printf '%s\n' "$container" >> "$candidate_file"
index=$((index + 1))
done
wait_for_containers "$candidate_pool"
write_upstream "$upstream_file.next" "$candidate_pool"
cp "$upstream_file.next" "$upstream_file"
upstream_changed=true
if [ -n "$(compose ps -q nginx)" ]; then
compose exec -T nginx nginx -t
switched=true
compose exec -T nginx nginx -s reload
else
nginx_created=true
compose up -d --no-build --no-deps nginx
switched=true
fi
smoke_check
application_network="${project}_application"
for container in $candidate_pool; do
docker network disconnect "$application_network" "$container"
docker network connect --alias api "$application_network" "$container"
done
smoke_check
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
printf '%s\n' "$candidate_pool" > "$active_containers"
cp "$manifest" "$current_manifest"
rm -f "$candidate_file" "$upstream_file.next"
trap - EXIT INT TERM
if [ -n "$old_pool" ]; then
for container in $old_pool; do
docker stop --time 6 "$container" >/dev/null || true
done
printf '%s\n' "$old_pool" > "$previous_containers"
fi
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for container in $(docker compose ps -q api); do
instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 6 "$container" >/dev/null
exit 0
fi
done
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
exit 1