forked from mtusi2026/mtusi2026lab4
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fa530790dc | ||
|
|
05d9bb0ff2 | ||
|
|
3815be3493 | ||
|
|
13151e66aa | ||
|
|
99cc86766f | ||
|
|
56e4a03bb6 | ||
|
|
37d93c340e | ||
|
|
3fde1c324c | ||
|
|
f8c31de7c0 | ||
|
|
cc4ac1ddd7 | ||
|
|
ad36ab8a92 | ||
|
|
110236d01f | ||
|
|
b3d5d76302 | ||
|
|
ec0e6707b8 | ||
|
|
7e60f099d3 |
@@ -0,0 +1,93 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.repository }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab4-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
git init .
|
||||
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||
git fetch --depth=1 origin ${{ github.sha }}
|
||||
git checkout --detach FETCH_HEAD
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Check contract
|
||||
run: make check
|
||||
|
||||
- name: Validate OpenAPI
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
with:
|
||||
version: v2.13.2
|
||||
args: --timeout=5m
|
||||
skip-cache: true
|
||||
|
||||
- name: Unit tests and build
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build ./cmd/...
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab4.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
set -a
|
||||
. ./.env
|
||||
set +a
|
||||
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||
make test BASE_URL=http://nginx:8080
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||
|
||||
- name: Stop local stack
|
||||
if: always()
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
@@ -0,0 +1,192 @@
|
||||
name: Deploy production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: production
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Verify selected commit
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
api_image: ${{ steps.publish.outputs.api_image }}
|
||||
nginx_image: ${{ steps.publish.outputs.nginx_image }}
|
||||
swagger_image: ${{ steps.publish.outputs.swagger_image }}
|
||||
prometheus_image: ${{ steps.publish.outputs.prometheus_image }}
|
||||
grafana_image: ${{ steps.publish.outputs.grafana_image }}
|
||||
s3_image: ${{ steps.publish.outputs.s3_image }}
|
||||
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab4-deploy-${{ github.run_id }}
|
||||
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-api:${{ github.sha }}
|
||||
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-nginx:${{ github.sha }}
|
||||
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-swagger:${{ github.sha }}
|
||||
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-prometheus:${{ github.sha }}
|
||||
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-grafana:${{ github.sha }}
|
||||
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio:${{ github.sha }}
|
||||
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio-client:${{ github.sha }}
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
git init .
|
||||
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||
git fetch --depth=1 origin ${{ github.sha }}
|
||||
git checkout --detach FETCH_HEAD
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Check contract
|
||||
run: make check
|
||||
|
||||
- name: Validate OpenAPI
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
with:
|
||||
version: v2.13.2
|
||||
args: --timeout=5m
|
||||
skip-cache: true
|
||||
|
||||
- name: Unit tests and build
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build ./cmd/...
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab4.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
set -a
|
||||
. ./.env
|
||||
set +a
|
||||
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||
make test BASE_URL=http://nginx:8080
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
|
||||
- name: Publish verified images
|
||||
id: publish
|
||||
env:
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin
|
||||
|
||||
publish() {
|
||||
output=$1
|
||||
image=$2
|
||||
push_output=$(docker push "$image")
|
||||
printf '%s\n' "$push_output"
|
||||
digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1)
|
||||
test -n "$digest"
|
||||
repository=${image%:*}
|
||||
printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
publish api_image "$API_IMAGE"
|
||||
publish nginx_image "$NGINX_IMAGE"
|
||||
publish swagger_image "$SWAGGER_IMAGE"
|
||||
publish prometheus_image "$PROMETHEUS_IMAGE"
|
||||
publish grafana_image "$GRAFANA_IMAGE"
|
||||
publish s3_image "$S3_IMAGE"
|
||||
publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE"
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||
|
||||
- name: Stop local stack
|
||||
if: always()
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
|
||||
deploy:
|
||||
name: Deploy selected commit
|
||||
needs: verify
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
steps:
|
||||
- name: Configure SSH
|
||||
run: |
|
||||
umask 077
|
||||
mkdir -p ~/.ssh
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||
|
||||
- name: Prepare immutable image manifest
|
||||
run: |
|
||||
cat > deploy-images.env <<EOF
|
||||
API_IMAGE=${{ needs.verify.outputs.api_image }}
|
||||
NGINX_IMAGE=${{ needs.verify.outputs.nginx_image }}
|
||||
SWAGGER_IMAGE=${{ needs.verify.outputs.swagger_image }}
|
||||
PROMETHEUS_IMAGE=${{ needs.verify.outputs.prometheus_image }}
|
||||
GRAFANA_IMAGE=${{ needs.verify.outputs.grafana_image }}
|
||||
S3_IMAGE=${{ needs.verify.outputs.s3_image }}
|
||||
S3_BOOTSTRAP_IMAGE=${{ needs.verify.outputs.s3_bootstrap_image }}
|
||||
EOF
|
||||
|
||||
- name: Authenticate registry on deployment host
|
||||
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
|
||||
|
||||
- name: Upload image manifest
|
||||
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab4-${{ github.run_id }}.env"
|
||||
|
||||
- name: Deploy and verify
|
||||
run: |
|
||||
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
|
||||
set -eu
|
||||
cd ~/mtusi2026lab4
|
||||
git fetch origin main
|
||||
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||
git reset --hard '$DEPLOY_SHA'
|
||||
manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env'
|
||||
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
|
||||
rm -f \"\$manifest\"
|
||||
else
|
||||
status=\$?
|
||||
rm -f \"\$manifest\"
|
||||
exit \"\$status\"
|
||||
fi
|
||||
test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA'
|
||||
docker compose ps
|
||||
"
|
||||
+7
-1
@@ -1,8 +1,12 @@
|
||||
FROM golang:1.26.5-alpine3.24 AS build
|
||||
|
||||
WORKDIR /src
|
||||
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go build -o /out/messenger ./cmd/messenger
|
||||
RUN go build -p=1 -o /out/messenger ./cmd/messenger
|
||||
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
|
||||
|
||||
FROM build AS test
|
||||
|
||||
@@ -25,11 +29,13 @@ COPY monitoring/grafana/dashboards /etc/grafana/dashboards
|
||||
FROM nginx:1.30.5-alpine AS loadbalancer
|
||||
|
||||
COPY nginx/nginx.conf /etc/nginx/nginx.conf
|
||||
COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
||||
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
||||
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
|
||||
|
||||
USER app
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,37 +1,181 @@
|
||||
# Отчёт — лабораторная 4
|
||||
# Отчёт — лабораторная работа № 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: Пластинин Денис, БВТ2303.
|
||||
- Целевая оценка: 5.
|
||||
- Выбранный трек: 2, согласованность PostgreSQL и S3.
|
||||
- Ветка: `solution/bvt2303-plastinin`.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Engine с Compose V2, Git и свободные локальные порты 8080, 8081, 9090 и 3000. Для запуска Go-проверок вне контейнера нужен Go 1.26. Образы приложения и MinIO собираются из исходников внутри Docker. Версии Go, PostgreSQL, Valkey, Nginx, MinIO и остальных компонентов закреплены в Dockerfile и `compose.yaml`.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp lab4.env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
После запуска доступны:
|
||||
|
||||
- API через Nginx: `http://localhost:8080`;
|
||||
- Swagger UI: `http://localhost:8081`;
|
||||
- Prometheus: `http://localhost:9090`;
|
||||
- Grafana: `http://localhost:3000`, пользователь `admin`, пароль из `GRAFANA_ADMIN_PASSWORD`.
|
||||
|
||||
Публичные и внутренние проверки:
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://localhost:8080
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
```
|
||||
|
||||
Сверка PostgreSQL и S3 выполняется внутри запущенной API-реплики. Без `--apply` команда ничего не удаляет:
|
||||
|
||||
```sh
|
||||
docker compose exec -T api /app/messenger reconcile-attachments --grace-period=1h
|
||||
docker compose exec -T api /app/messenger reconcile-attachments --apply --grace-period=1h
|
||||
```
|
||||
|
||||
Эксперименты запускаются на чистом стенде:
|
||||
|
||||
```sh
|
||||
docker compose exec -T api /app/lab4experiment memory
|
||||
docker compose exec -T api /app/lab4experiment consistency
|
||||
```
|
||||
|
||||
Безопасная остановка сохраняет данные в Docker volumes:
|
||||
|
||||
```sh
|
||||
docker compose down
|
||||
```
|
||||
|
||||
Полное удаление учебных данных выполняется только явно:
|
||||
|
||||
```sh
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
## Архитектура
|
||||
|
||||
```text
|
||||
клиент
|
||||
|
|
||||
v
|
||||
Nginx :8080
|
||||
|
|
||||
+---- API replica 1 ----+
|
||||
| |
|
||||
+---- API replica 2 ----+---- PostgreSQL: пользователи, чаты,
|
||||
| сообщения, metadata
|
||||
+---- Valkey: сессии
|
||||
+---- MinIO: приватные байты изображений
|
||||
|
||||
Prometheus <---- API /metrics, MinIO, Nginx, PostgreSQL, Valkey
|
||||
|
|
||||
+---- Grafana
|
||||
```
|
||||
|
||||
Из лабораторной № 3 сохранены две и более API-реплики, Nginx, общие PostgreSQL и Valkey, Bearer-сессии, keyset pagination, идемпотентность сообщений, graceful shutdown, failover, метрики и управляемое развёртывание. В лабораторной № 4 добавлены приватное объектное хранилище, модель вложения, бинарная загрузка и скачивание, привязка вложений к сообщениям и сверка двух хранилищ.
|
||||
|
||||
Код разделён по ответственности. `internal/messenger/httpapi` содержит HTTP-маршруты, middleware и преобразование ошибок в ответы. `internal/messenger/attachment` владеет сценарием загрузки и чтения вложений. `internal/postgres`, `internal/objectstore` и `internal/session` являются адаптерами PostgreSQL, S3 и Valkey. `internal/reconcile` выполняет сверку хранилищ, а `internal/bootstrap` собирает зависимости для сервера, миграций и служебной команды. HTTP-слой зависит от узких интерфейсов и общих ошибок модели, поэтому детали PostgreSQL и S3 в handlers не попадают.
|
||||
|
||||
PostgreSQL остаётся источником истины для прав доступа и метаданных. Таблица `attachments` хранит владельца, чат, отображаемое имя, MIME, размер, SHA-256 и внутренний object key. Таблица `message_attachments` задаёт упорядоченную связь многие ко многим с сообщениями. Object key генерируется сервером и не зависит от имени файла.
|
||||
|
||||
MinIO хранит только байты оригиналов. Bucket закрыт для анонимного доступа. Bootstrap повторяемо создаёт bucket, policy и отдельного пользователя приложения. API получает только действия list/get/put/delete в одном bucket. Root credentials используются bootstrap-контейнером и не передаются API. Web-консоль MinIO отключена и её порт не опубликован.
|
||||
|
||||
## Загрузка и чтение вложения
|
||||
|
||||
Клиент передаёт сырые байты PNG или JPEG. Сервер ограничивает HTTP body до 10 MiB до полного чтения, проверяет заявленный и фактический MIME, декодирует конфигурацию изображения, ограничивает число пикселей 40 миллионами и проверяет полную декодируемость. Filename содержит от 1 до 255 Unicode-символов и не может содержать разделители пути, NUL, управляющие символы или состоять только из пробелов.
|
||||
|
||||
При успешной загрузке сервер генерирует UUID для вложения и отдельный случайный object key, считает SHA-256 и создаёт в PostgreSQL строку со статусом `pending`. Затем он записывает объект в MinIO и переводит строку в `ready`. Только после этого возвращается `201`. При отказе S3 строка `pending` удаляется. Если результат перевода в `ready` неоднозначен, сервер повторно читает строку по attachment ID и object key: уже сохранённый `ready` считается успехом, а подтверждённый `pending` компенсируется удалением объекта и metadata.
|
||||
|
||||
Метаданные и содержимое читаются только после проверки Bearer-сессии и членства пользователя в чате. Посторонний пользователь и неизвестный идентификатор дают одинаковый `404`. API отдаёт байты самостоятельно без redirect и возвращает сохранённый `Content-Type` и точный `Content-Length`.
|
||||
|
||||
Сообщение принимает до десяти уникальных `attachment_ids`. В одной транзакции PostgreSQL проверяются членство отправителя и принадлежность каждого вложения нужному чату, создаются сообщение, связи и запись идемпотентности. Отсутствующее поле `attachment_ids` и пустой массив дают одинаковый семантический hash. Пустой текст допустим только при наличии вложения.
|
||||
|
||||
## Согласованность PostgreSQL и S3
|
||||
|
||||
Операции PostgreSQL и S3 не входят в общую распределённую транзакцию. Возможны два вида расхождений:
|
||||
|
||||
1. object существует в S3, но metadata отсутствует;
|
||||
2. metadata существует, но object отсутствует.
|
||||
|
||||
Команда `reconcile-attachments` проходит оба набора в порядке object key постранично, поэтому объём памяти ограничен размером двух страниц. По умолчанию включён dry-run. Grace period защищает свежие объекты и строки, которые могут относиться к ещё выполняющейся операции.
|
||||
|
||||
В режиме `--apply` старый orphan в S3 удаляется после повторной проверки object key в PostgreSQL. Старые `pending` и готовые, но неиспользуемые вложения сначала атомарно переводятся в `deleting` под блокировкой строки. Под той же блокировкой повторно проверяются ожидаемый статус, возраст и отсутствие ссылки из `message_attachments`. После удаления объекта строка удаляется только в статусе `deleting`. Если upload успел изменить `pending` на `ready`, устаревший кандидат не захватывается. Используемая строка с отсутствующим объектом сохраняется и попадает в отчёт для ручного восстановления. Повторный запуск продолжает незавершённый `deleting` и не повторяет уже выполненные удаления.
|
||||
|
||||
## Ограничение памяти
|
||||
|
||||
До чтения body weighted semaphore резервирует полный лимит одной загрузки — 10 MiB. Поэтому отсутствующий или заниженный `Content-Length` не позволяет одновременно накопить неограниченное число буферов. После чтения дополнительно резервируется восемь байт на пиксель перед полной декодировкой. Бюджеты одной реплики разделены: 64 MiB для исходных body и 384 MiB для декодирования; лимит контейнера равен 512 MiB. Проверки размеров выполняются до умножения, чтобы исключить integer overflow. Если одного из бюджетов недостаточно, API отвечает `429` с `Retry-After: 1`.
|
||||
|
||||
Go-инструмент `cmd/lab4experiment` создаёт случайное JPEG-изображение и выполняет четыре волны по 12 одновременных загрузок. Передача каждой копии ограничена по скорости, чтобы нагрузка длилась дольше интервала scrape. Наблюдаемый пик heap и resident memory двух API-процессов читается из Prometheus с интервалом одна секунда. Это измерение локального стенда, а не универсальная оценка производительности S3.
|
||||
|
||||
В эксперименте успешно завершились все 48 загрузок файла размером 3 759 914 байт. Четыре волны заняли 12,974 секунды. После трёх секунд стабилизации суммарный heap двух реплик вырос с 153 985 824 до 202 505 208 байт, то есть на 48 519 384 байта. Суммарный RSS вырос с 335 298 560 до 345 976 832 байт, то есть на 10 678 272 байта. Наблюдаемый пик остался ниже общего контейнерного лимита двух реплик 1 GiB.
|
||||
|
||||
## Наблюдаемость
|
||||
|
||||
API экспортирует общие HTTP-метрики и отдельные показатели загрузок:
|
||||
|
||||
- `messenger_attachment_uploads_total{result}`;
|
||||
- `messenger_attachment_upload_duration_seconds`;
|
||||
- `messenger_attachment_upload_bytes_total`.
|
||||
|
||||
Prometheus обнаруживает все API-реплики через Docker DNS и отдельно собирает MinIO, Nginx, PostgreSQL и Valkey. Grafana показывает частоту и p95 загрузок, объём принятых байтов, число и ёмкость объектов MinIO, ошибки S3, доступность компонентов, RPS, классы HTTP-ответов и память Go.
|
||||
|
||||
`/health/live` проверяет только процесс. `/health/ready` требует доступных PostgreSQL, Valkey и приватного bucket S3. Недоступность Grafana или Prometheus не останавливает API.
|
||||
|
||||
## CI/CD и blue/green deploy
|
||||
|
||||
CI проверяет contract, OpenAPI, форматирование, lint, unit и integration tests, persistence и отказ одной API-реплики. После успешных проверок образы публикуются в Gitea Container Registry. В manifest передаются ссылки вида `repository@sha256:digest`, поэтому сервер запускает те же байты, которые прошли проверки.
|
||||
|
||||
Production workflow запускается вручную и использует защищённое environment `production`. Перед переключением применяются только обратно совместимые миграции. Скрипт поднимает новый пул API под отдельными DNS-именами и ждёт readiness каждой реплики. Nginx сначала проверяет новую конфигурацию через `nginx -t`, затем атомарно выполняет reload upstream. Старый пул остаётся запущенным до успешного smoke-check нового и только после этого останавливается. При ошибке до завершения переключения восстанавливается предыдущий upstream, а candidate-контейнеры удаляются.
|
||||
|
||||
Сам балансировщик относится к стабильному control plane. Если его verified digest изменился, обычный API deploy завершается до переключения и требует отдельного обслуживания Nginx. Это не подменяет новый образ старым и не создаёт скрытое окно недоступности внутри blue/green обновления API.
|
||||
|
||||
Локальная проверка механизма подняла две отдельные candidate-реплики, переключила upstream и выполнила 20 запросов `/health/live`: все ответы имели статус `200`, а `X-Instance-Id` чередовался только между двумя контейнерами нового пула.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда или сценарий | Результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт и OpenAPI | `make check`, `make validate` | offline check PASS; validate входит в CI | `evidence/lab4-verification.txt` |
|
||||
| Unit и integration | `go test ./...`, race/tests profile в CI | unit PASS | `evidence/lab4-verification.txt` |
|
||||
| Upload/download и ACL | `make test` | PASS | `evidence/lab4-verification.txt` |
|
||||
| Межрепличный failover и restart | public resilience hooks | проверяются CI точного commit | `.gitea/workflows/ci.yml` |
|
||||
| Приватный bucket и повторный bootstrap | anonymous HTTP check, два запуска bootstrap | PASS | `evidence/lab4-verification.txt` |
|
||||
| 12 конкурентных загрузок и память | `lab4experiment memory` | 48/48 PASS, observed RSS peak 345 976 832 B | `evidence/lab4-memory.json` |
|
||||
| Трек 2: dry-run/apply/repeat | `lab4experiment consistency` | PASS, повторный apply: 0 удалений | `evidence/lab4-consistency.json` |
|
||||
| Blue/green API switch | два candidate-контейнера, `nginx -t`, reload, 20 health-запросов | 20/20 HTTP 200, только новый пул | `evidence/lab4-verification.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Эксперимент согласованности
|
||||
|
||||
## Эксперименты и отказы
|
||||
Гипотеза: постраничная сверка с grace period может автоматически удалить старые безопасные расхождения, не удаляя свежие данные и metadata используемого вложения; второй apply будет идемпотентным.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Методика на чистом стенде:
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
1. создать согласованное вложение;
|
||||
2. удалить S3 object у непривязанного вложения;
|
||||
3. привязать другое вложение к сообщению и удалить его S3 object;
|
||||
4. создать object без metadata;
|
||||
5. создать незавершённое вложение `pending`;
|
||||
6. выполнить apply с grace period один час и проверить сохранность свежих аномалий;
|
||||
7. состарить `pending`, выполнить dry-run с нулевым grace period;
|
||||
8. выполнить apply и проверить удаление orphan, непривязанной строки и старого `pending`;
|
||||
9. повторить apply и проверить отсутствие новых действий.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Фактическая стоимость обхода записывается как число просмотренных объектов и строк, размер страницы и wall time. Алгоритм выполняет линейный merge двух отсортированных потоков; дополнительные запросы проверки ссылок нужны только для metadata с отсутствующим объектом.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
При размере страницы 10 фактический запуск пересёк границу страницы. Apply с часовым grace period просмотрел 14 строк metadata и 13 objects за 4 миллисекунды, сохранил свежие orphan, missing metadata и `pending`. После искусственного старения dry-run за 5 миллисекунд обнаружил один старый orphan, одну непривязанную строку без объекта, одну используемую строку без объекта и один старый `pending`. Apply за 13 миллисекунд удалил object и metadata старого `pending`, orphan object и непривязанную missing metadata. Повторный apply занял 1 миллисекунду и выполнил ноль удалений; используемая строка осталась в PostgreSQL.
|
||||
|
||||
## Ограничения
|
||||
|
||||
- Между PostgreSQL и S3 нет общей транзакции; lifecycle `pending/ready/deleting`, компенсация и reconciler обеспечивают eventual consistency.
|
||||
- Удалённый object используемого вложения автоматически не восстанавливается: строка сохраняется и требует восстановления байтов из резервной копии либо решения оператора.
|
||||
- Grace period основан на времени PostgreSQL и S3; для удалённого стенда требуется синхронизация часов.
|
||||
- Admission control действует отдельно в каждой API-реплике; при изменении контейнерного лимита нужно совместно настраивать `ATTACHMENT_BODY_MEMORY_BYTES` и `ATTACHMENT_DECODE_MEMORY_BYTES`.
|
||||
- Автоматический rollback переключает API обратно на предыдущий пул, но не откатывает уже применённые миграции и stateful-хранилища. Поэтому миграции должны оставаться обратно совместимыми как минимум с предыдущей версией приложения.
|
||||
- MinIO собирается из официального закреплённого source tag, поскольку готовые community images больше не публикуются. Это увеличивает время первого CI build.
|
||||
- Локальный HTTP допустим только для учебного стенда с синтетическими данными; удалённое развёртывание требует TLS.
|
||||
|
||||
@@ -0,0 +1,676 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||
)
|
||||
|
||||
const experimentPassword = "Lab4-experiment-password"
|
||||
|
||||
type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
}
|
||||
|
||||
type attachment struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type setup struct {
|
||||
client *apiClient
|
||||
chatID string
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(arguments []string, output io.Writer) error {
|
||||
if len(arguments) == 0 {
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
switch arguments[0] {
|
||||
case "memory":
|
||||
return runMemory(arguments[1:], output)
|
||||
case "consistency":
|
||||
return runConsistency(arguments[1:], output)
|
||||
default:
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
}
|
||||
|
||||
type memoryResult struct {
|
||||
Replicas int `json:"replicas"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Rounds int `json:"rounds"`
|
||||
SettleTimeMillis int64 `json:"settle_time_ms"`
|
||||
ImageBytes int `json:"image_bytes"`
|
||||
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
|
||||
SuccessfulUploads int64 `json:"successful_uploads"`
|
||||
FailedUploads int64 `json:"failed_uploads"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
|
||||
HeapPeakBytes int64 `json:"heap_peak_bytes"`
|
||||
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
|
||||
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
|
||||
RSSPeakBytes int64 `json:"rss_peak_bytes"`
|
||||
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
|
||||
}
|
||||
|
||||
func runMemory(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
|
||||
replicas := flags.Int("replicas", 2, "expected API replicas")
|
||||
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
|
||||
rounds := flags.Int("rounds", 4, "number of upload waves")
|
||||
width := flags.Int("width", 2048, "image width")
|
||||
height := flags.Int("height", 2048, "image height")
|
||||
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
|
||||
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid memory experiment flags")
|
||||
}
|
||||
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
|
||||
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
fixture, err := jpegFixture(*width, *height)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
|
||||
return err
|
||||
}
|
||||
settle := time.NewTimer(*settleTime)
|
||||
defer settle.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-settle.C:
|
||||
}
|
||||
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var successful atomic.Int64
|
||||
var failed atomic.Int64
|
||||
var firstError struct {
|
||||
sync.Mutex
|
||||
err error
|
||||
}
|
||||
heapPeak := heapBaseline
|
||||
rssPeak := rssBaseline
|
||||
loadDone := make(chan struct{})
|
||||
sampleDone := make(chan error, 1)
|
||||
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
|
||||
started := time.Now()
|
||||
for round := range *rounds {
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(*concurrency)
|
||||
for index := range *concurrency {
|
||||
go func(round, index int) {
|
||||
defer workers.Done()
|
||||
reader := newRateReader(fixture, *uploadRate)
|
||||
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
|
||||
failed.Add(1)
|
||||
firstError.Lock()
|
||||
if firstError.err == nil {
|
||||
firstError.err = err
|
||||
}
|
||||
firstError.Unlock()
|
||||
return
|
||||
}
|
||||
successful.Add(1)
|
||||
}(round, index)
|
||||
}
|
||||
workers.Wait()
|
||||
}
|
||||
close(loadDone)
|
||||
if err := <-sampleDone; err != nil {
|
||||
return err
|
||||
}
|
||||
result := memoryResult{
|
||||
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
|
||||
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
|
||||
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
|
||||
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
|
||||
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
|
||||
}
|
||||
if result.FailedUploads > 0 {
|
||||
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func sampleMemory(
|
||||
ctx context.Context,
|
||||
prometheusURL string,
|
||||
expectedReplicas int,
|
||||
done <-chan struct{},
|
||||
heapPeak *int64,
|
||||
rssPeak *int64,
|
||||
result chan<- error,
|
||||
) {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
result <- ctx.Err()
|
||||
return
|
||||
case <-done:
|
||||
result <- nil
|
||||
return
|
||||
case <-ticker.C:
|
||||
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
if replicas != int64(expectedReplicas) {
|
||||
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
|
||||
return
|
||||
}
|
||||
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
*heapPeak = max(*heapPeak, heap)
|
||||
*rssPeak = max(*rssPeak, rss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type consistencyResult struct {
|
||||
PageSize int `json:"page_size"`
|
||||
SeededConsistent int `json:"seeded_consistent"`
|
||||
FreshApply reconciliationRun `json:"fresh_apply"`
|
||||
DryRun reconciliationRun `json:"dry_run"`
|
||||
Apply reconciliationRun `json:"apply"`
|
||||
RepeatedApply reconciliationRun `json:"repeated_apply"`
|
||||
FreshPreserved bool `json:"fresh_preserved"`
|
||||
OrphanDeleted bool `json:"orphan_deleted"`
|
||||
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
||||
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
||||
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
||||
TotalDurationMillis int64 `json:"total_duration_ms"`
|
||||
}
|
||||
|
||||
type reconciliationRun struct {
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func runConsistency(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
pageSize := flags.Int("page-size", 10, "reconciliation page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid consistency experiment flags")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
settings, err := config.LoadReconciliation()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := postgres.New(pool)
|
||||
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
initial, err := clean.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hasAnomalies(initial) {
|
||||
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := jpegFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seededConsistent := *pageSize + 1
|
||||
for index := range seededConsistent {
|
||||
filename := fmt.Sprintf("consistent-%03d.jpg", index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
|
||||
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
||||
UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshRun, err := measuredReconciliation(ctx, fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
||||
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
||||
attachmentExists(ctx, pool, abandoned.ID)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||
}
|
||||
|
||||
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dryRunResult, err := measuredReconciliation(ctx, dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeatedResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := consistencyResult{
|
||||
PageSize: *pageSize, SeededConsistent: seededConsistent,
|
||||
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||
}
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
||||
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
|
||||
}
|
||||
|
||||
func hasAnomalies(report reconcile.Report) bool {
|
||||
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
||||
}
|
||||
|
||||
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query attachment keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
keys := make(map[string]string, len(ids))
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var key string
|
||||
if err := rows.Scan(&id, &key); err != nil {
|
||||
return nil, fmt.Errorf("scan attachment key: %w", err)
|
||||
}
|
||||
keys[id] = key
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate attachment keys: %w", err)
|
||||
}
|
||||
if len(keys) != len(ids) {
|
||||
return nil, errors.New("not all attachment keys were found")
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
|
||||
var exists bool
|
||||
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
|
||||
}
|
||||
|
||||
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
|
||||
_, err := objects.Stat(ctx, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func prepare(ctx context.Context, baseURL string) (setup, error) {
|
||||
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
|
||||
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
|
||||
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
|
||||
}, &user, ""); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
|
||||
var session struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
client.token = session.Token
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
|
||||
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
|
||||
}, &chat, "Bearer "+client.token); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
return setup{client: client, chatID: chat.ID}, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) upload(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
filename string,
|
||||
body io.Reader,
|
||||
contentLength int64,
|
||||
) (attachment, error) {
|
||||
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
request.ContentLength = contentLength
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "image/jpeg")
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
|
||||
}
|
||||
var result attachment
|
||||
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
|
||||
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
|
||||
"text": "", "attachment_ids": []string{attachmentID},
|
||||
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
|
||||
}
|
||||
|
||||
func (c *apiClient) jsonRequest(
|
||||
ctx context.Context,
|
||||
method string,
|
||||
path string,
|
||||
payload any,
|
||||
result any,
|
||||
authorization string,
|
||||
extraHeader ...string,
|
||||
) error {
|
||||
var body io.Reader
|
||||
if payload != nil {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(encoded)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if payload != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if authorization != "" {
|
||||
request.Header.Set("Authorization", authorization)
|
||||
}
|
||||
for index := 0; index+1 < len(extraHeader); index += 2 {
|
||||
request.Header.Set(extraHeader[index], extraHeader[index+1])
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
|
||||
}
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return json.NewDecoder(response.Body).Decode(result)
|
||||
}
|
||||
|
||||
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err == nil && count == int64(expected) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func activeReplicasQuery() string {
|
||||
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
||||
}
|
||||
|
||||
func freshMetricQuery(metric string) string {
|
||||
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
||||
}
|
||||
|
||||
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
values := request.URL.Query()
|
||||
values.Set("query", query)
|
||||
request.URL.RawQuery = values.Encode()
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
|
||||
return 0, errors.New("prometheus query returned no scalar value")
|
||||
}
|
||||
var raw string
|
||||
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
return int64(value), err
|
||||
}
|
||||
|
||||
func jpegFixture(width, height int) ([]byte, error) {
|
||||
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
state := uint64(0x9e3779b97f4a7c15)
|
||||
for index := range pixels.Pix {
|
||||
state ^= state << 13
|
||||
state ^= state >> 7
|
||||
state ^= state << 17
|
||||
pixels.Pix[index] = byte(state >> 24)
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encoded.Bytes(), nil
|
||||
}
|
||||
|
||||
type rateReader struct {
|
||||
reader *bytes.Reader
|
||||
rate int64
|
||||
started time.Time
|
||||
read int64
|
||||
}
|
||||
|
||||
func newRateReader(data []byte, rate int64) *rateReader {
|
||||
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
|
||||
}
|
||||
|
||||
func (r *rateReader) Read(buffer []byte) (int, error) {
|
||||
for {
|
||||
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
|
||||
if allowed > 0 {
|
||||
if int64(len(buffer)) > allowed {
|
||||
buffer = buffer[:allowed]
|
||||
}
|
||||
count, err := r.reader.Read(buffer)
|
||||
r.read += int64(count)
|
||||
return count, err
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(output io.Writer, value any) error {
|
||||
encoder := json.NewEncoder(output)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(value)
|
||||
}
|
||||
+8
-109
@@ -1,132 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||
)
|
||||
|
||||
func main() {
|
||||
appLogger := logger.New()
|
||||
if err := run(appLogger, os.Args[1:]); err != nil {
|
||||
appLogger.Error("server stopped", "error", err)
|
||||
appLogger.Error("application stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 0 {
|
||||
return runServer(logger)
|
||||
return bootstrap.RunServer(logger)
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return runMigrations()
|
||||
return bootstrap.RunMigrations()
|
||||
}
|
||||
return fmt.Errorf("usage: messenger [migrate]")
|
||||
}
|
||||
|
||||
func runServer(logger *slog.Logger) error {
|
||||
settings, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
if arguments[0] == "reconcile-attachments" {
|
||||
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
sessions, err := session.New(settings.CacheURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sessions.Close() }()
|
||||
passwords, err := password.New(password.Config{
|
||||
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
|
||||
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
|
||||
KeyLength: settings.PasswordKeyLength,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app := messenger.NewServer(
|
||||
logger,
|
||||
postgres.New(pool),
|
||||
sessions,
|
||||
passwords,
|
||||
settings.SessionTTL,
|
||||
settings.IdempotencyTTL,
|
||||
settings.RequestTimeout,
|
||||
instanceID,
|
||||
settings.SwaggerPort,
|
||||
)
|
||||
server := &http.Server{
|
||||
Addr: settings.Address,
|
||||
Handler: app.Handler(),
|
||||
ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
|
||||
ReadTimeout: settings.HTTPReadTimeout,
|
||||
WriteTimeout: settings.HTTPWriteTimeout,
|
||||
IdleTimeout: settings.HTTPIdleTimeout,
|
||||
}
|
||||
listener, err := net.Listen("tcp", server.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serveResult := make(chan error, 1)
|
||||
go func() {
|
||||
serveResult <- server.Serve(listener)
|
||||
}()
|
||||
logger.Info("server started", "address", server.Addr)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
select {
|
||||
case err := <-serveResult:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
logger.Info("shutdown requested")
|
||||
app.SetReady(false)
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
_ = server.Close()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runMigrations() error {
|
||||
settings, err := config.LoadMigration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||
return errors.New("usage: messenger [migrate|reconcile-attachments]")
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
cpus: ${POSTGRES_CPUS}
|
||||
mem_limit: ${POSTGRES_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.4-alpine
|
||||
@@ -31,6 +32,58 @@ services:
|
||||
cpus: ${VALKEY_CPUS}
|
||||
mem_limit: ${VALKEY_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
|
||||
s3:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/server.Dockerfile
|
||||
args:
|
||||
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
|
||||
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
|
||||
command: ["server", "/data", "--address", ":9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_PROMETHEUS_AUTH_TYPE: public
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
expose:
|
||||
- "9000"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${S3_CPUS}
|
||||
mem_limit: ${S3_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/client.Dockerfile
|
||||
args:
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z
|
||||
entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: ${S3_BOOTSTRAP_CPUS}
|
||||
mem_limit: ${S3_BOOTSTRAP_MEMORY}
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
@@ -52,6 +105,7 @@ services:
|
||||
cpus: ${MIGRATE_CPUS}
|
||||
mem_limit: ${MIGRATE_MEMORY}
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
|
||||
api:
|
||||
build: .
|
||||
@@ -82,6 +136,17 @@ services:
|
||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
@@ -89,6 +154,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
|
||||
interval: 2s
|
||||
@@ -101,6 +168,7 @@ services:
|
||||
mem_limit: ${API_MEMORY}
|
||||
stop_grace_period: 8s
|
||||
restart: unless-stopped
|
||||
networks: [application, data]
|
||||
|
||||
nginx:
|
||||
build:
|
||||
@@ -120,6 +188,7 @@ services:
|
||||
cpus: ${NGINX_CPUS}
|
||||
mem_limit: ${NGINX_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, application, monitoring]
|
||||
|
||||
swagger:
|
||||
build:
|
||||
@@ -134,6 +203,7 @@ services:
|
||||
cpus: ${SWAGGER_CPUS}
|
||||
mem_limit: ${SWAGGER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge]
|
||||
|
||||
tests:
|
||||
profiles: ["tests"]
|
||||
@@ -150,6 +220,7 @@ services:
|
||||
condition: service_healthy
|
||||
cpus: ${TEST_CPUS}
|
||||
mem_limit: ${TEST_MEMORY}
|
||||
networks: [data]
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
|
||||
@@ -164,6 +235,7 @@ services:
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
valkey-exporter:
|
||||
image: oliver006/redis_exporter:v1.89.0-alpine
|
||||
@@ -177,6 +249,7 @@ services:
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
nginx-exporter:
|
||||
image: nginx/nginx-prometheus-exporter:1.5.1
|
||||
@@ -187,6 +260,7 @@ services:
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [monitoring]
|
||||
|
||||
prometheus:
|
||||
build:
|
||||
@@ -211,9 +285,11 @@ services:
|
||||
- nginx-exporter
|
||||
- postgres-exporter
|
||||
- valkey-exporter
|
||||
- s3
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, application, data, monitoring]
|
||||
|
||||
grafana:
|
||||
build:
|
||||
@@ -233,9 +309,20 @@ services:
|
||||
cpus: ${GRAFANA_CPUS}
|
||||
mem_limit: ${GRAFANA_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
edge:
|
||||
application:
|
||||
internal: true
|
||||
data:
|
||||
internal: true
|
||||
monitoring:
|
||||
internal: true
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
services:
|
||||
api:
|
||||
image: ${API_IMAGE:?API_IMAGE is required}
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:?API_IMAGE is required}
|
||||
|
||||
nginx:
|
||||
image: ${NGINX_IMAGE:?NGINX_IMAGE is required}
|
||||
|
||||
swagger:
|
||||
image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required}
|
||||
|
||||
prometheus:
|
||||
image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required}
|
||||
|
||||
grafana:
|
||||
image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required}
|
||||
|
||||
s3:
|
||||
image: ${S3_IMAGE:?S3_IMAGE is required}
|
||||
|
||||
s3-bootstrap:
|
||||
image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required}
|
||||
@@ -0,0 +1,4 @@
|
||||
services:
|
||||
nginx:
|
||||
volumes:
|
||||
- ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"page_size": 10,
|
||||
"seeded_consistent": 11,
|
||||
"fresh_apply": {
|
||||
"duration_ms": 4,
|
||||
"report": {
|
||||
"metadata_scanned": 14,
|
||||
"objects_scanned": 13,
|
||||
"consistent": 12,
|
||||
"fresh_pending_attachments": 1,
|
||||
"old_pending_attachments": 0,
|
||||
"deleting_attachments": 0,
|
||||
"fresh_unreferenced_attachments": 11,
|
||||
"old_unreferenced_attachments": 0,
|
||||
"deleted_attachment_objects": 0,
|
||||
"fresh_orphan_objects": 1,
|
||||
"old_orphan_objects": 0,
|
||||
"deleted_orphan_objects": 0,
|
||||
"fresh_missing_objects": 2,
|
||||
"referenced_missing_objects": 0,
|
||||
"old_unreferenced_metadata": 0,
|
||||
"deleted_unreferenced_rows": 0
|
||||
}
|
||||
},
|
||||
"dry_run": {
|
||||
"duration_ms": 5,
|
||||
"report": {
|
||||
"metadata_scanned": 14,
|
||||
"objects_scanned": 13,
|
||||
"consistent": 11,
|
||||
"fresh_pending_attachments": 0,
|
||||
"old_pending_attachments": 1,
|
||||
"deleting_attachments": 0,
|
||||
"fresh_unreferenced_attachments": 11,
|
||||
"old_unreferenced_attachments": 0,
|
||||
"deleted_attachment_objects": 0,
|
||||
"fresh_orphan_objects": 0,
|
||||
"old_orphan_objects": 1,
|
||||
"deleted_orphan_objects": 0,
|
||||
"fresh_missing_objects": 0,
|
||||
"referenced_missing_objects": 1,
|
||||
"old_unreferenced_metadata": 1,
|
||||
"deleted_unreferenced_rows": 0
|
||||
}
|
||||
},
|
||||
"apply": {
|
||||
"duration_ms": 13,
|
||||
"report": {
|
||||
"metadata_scanned": 14,
|
||||
"objects_scanned": 13,
|
||||
"consistent": 11,
|
||||
"fresh_pending_attachments": 0,
|
||||
"old_pending_attachments": 1,
|
||||
"deleting_attachments": 0,
|
||||
"fresh_unreferenced_attachments": 11,
|
||||
"old_unreferenced_attachments": 0,
|
||||
"deleted_attachment_objects": 1,
|
||||
"fresh_orphan_objects": 0,
|
||||
"old_orphan_objects": 1,
|
||||
"deleted_orphan_objects": 1,
|
||||
"fresh_missing_objects": 0,
|
||||
"referenced_missing_objects": 1,
|
||||
"old_unreferenced_metadata": 1,
|
||||
"deleted_unreferenced_rows": 2
|
||||
}
|
||||
},
|
||||
"repeated_apply": {
|
||||
"duration_ms": 1,
|
||||
"report": {
|
||||
"metadata_scanned": 12,
|
||||
"objects_scanned": 11,
|
||||
"consistent": 11,
|
||||
"fresh_pending_attachments": 0,
|
||||
"old_pending_attachments": 0,
|
||||
"deleting_attachments": 0,
|
||||
"fresh_unreferenced_attachments": 11,
|
||||
"old_unreferenced_attachments": 0,
|
||||
"deleted_attachment_objects": 0,
|
||||
"fresh_orphan_objects": 0,
|
||||
"old_orphan_objects": 0,
|
||||
"deleted_orphan_objects": 0,
|
||||
"fresh_missing_objects": 0,
|
||||
"referenced_missing_objects": 1,
|
||||
"old_unreferenced_metadata": 0,
|
||||
"deleted_unreferenced_rows": 0
|
||||
}
|
||||
},
|
||||
"fresh_preserved": true,
|
||||
"orphan_deleted": true,
|
||||
"unused_metadata_deleted": true,
|
||||
"in_use_metadata_preserved": true,
|
||||
"abandoned_attachment_deleted": true,
|
||||
"total_duration_ms": 328
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"replicas": 2,
|
||||
"concurrency": 12,
|
||||
"rounds": 4,
|
||||
"settle_time_ms": 3000,
|
||||
"image_bytes": 3759914,
|
||||
"upload_rate_bytes_per_second": 1572864,
|
||||
"successful_uploads": 48,
|
||||
"failed_uploads": 0,
|
||||
"duration_ms": 12974,
|
||||
"heap_baseline_bytes": 153985824,
|
||||
"heap_peak_bytes": 202505208,
|
||||
"heap_delta_bytes": 48519384,
|
||||
"rss_baseline_bytes": 335298560,
|
||||
"rss_peak_bytes": 345976832,
|
||||
"rss_delta_bytes": 10678272
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
Date: 2026-10-09
|
||||
Source commit: 37d93c340e394ed4a923972accc6c872abb96076
|
||||
Result collection state: application, migrations, Compose and Nginx configuration matched the source commit; REPORT.md and evidence/ were refreshed afterward
|
||||
Environment: Docker 29.2.1, linux/arm64, 2 CPUs, 2054299648 bytes RAM
|
||||
Application topology: 2 API replicas, 512 MiB limit per replica
|
||||
|
||||
PASS gofmt for changed Go packages
|
||||
PASS go test ./internal/...
|
||||
PASS go build ./cmd/messenger and ./cmd/lab4experiment in Docker build
|
||||
PASS make check
|
||||
PASS docker compose config --quiet
|
||||
PASS sh -n scripts/rolling-update.sh
|
||||
PASS make test BASE_URL=http://127.0.0.1:28080
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
PASS PostgreSQL and Valkey integration suites
|
||||
PASS MinIO bootstrap completed twice without an error
|
||||
PASS anonymous bucket request returned HTTP 403
|
||||
PASS real pending -> deleting -> delete on PostgreSQL
|
||||
PASS stale pending candidate cannot claim an attachment already changed to ready
|
||||
PASS 48 of 48 uploads succeeded in 4 waves with concurrency 12
|
||||
PASS reconciler preserved fresh and referenced anomalies
|
||||
PASS reconciler deleted an old orphan, old unreferenced missing metadata, and old pending data
|
||||
PASS repeated reconciliation apply made no further changes
|
||||
PASS Nginx accepted the candidate upstream with nginx -t and atomic reload
|
||||
PASS 20 of 20 health requests returned HTTP 200 from only the two candidate API instances
|
||||
|
||||
CI additionally runs full OpenAPI validation, race tests, golangci-lint,
|
||||
persistence and API-replica failover on the exact pushed commit. Production
|
||||
deployment publishes verified images and passes immutable repository@sha256
|
||||
references to the manually approved environment job.
|
||||
@@ -5,37 +5,49 @@ go 1.26.0
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.0.99
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sync v0.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
|
||||
@@ -5,10 +5,14 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -23,20 +27,33 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
|
||||
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
||||
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
@@ -49,6 +66,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
@@ -68,6 +87,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
||||
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
alias_name=bootstrap
|
||||
policy_name=messenger-attachments
|
||||
policy_file=/tmp/messenger-attachments-policy.json
|
||||
|
||||
mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing "$alias_name/$S3_BUCKET"
|
||||
mc anonymous set none "$alias_name/$S3_BUCKET"
|
||||
|
||||
cat >"$policy_file" <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
mc admin policy create "$alias_name" "$policy_name" "$policy_file"
|
||||
mc admin user add "$alias_name" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||
mc admin policy attach "$alias_name" "$policy_name" --user "$S3_ACCESS_KEY"
|
||||
|
||||
mc alias set application http://s3:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||
mc ls "application/$S3_BUCKET" >/dev/null
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM golang:1.24.8-alpine3.22 AS build
|
||||
|
||||
ARG MC_VERSION
|
||||
RUN apk add --no-cache git
|
||||
RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc
|
||||
WORKDIR /src/mc
|
||||
RUN go mod download
|
||||
RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc .
|
||||
|
||||
FROM alpine:3.22.2
|
||||
|
||||
COPY --from=build /out/mc /usr/local/bin/mc
|
||||
COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh
|
||||
|
||||
ENTRYPOINT ["mc"]
|
||||
@@ -0,0 +1,18 @@
|
||||
FROM golang:1.24.8-alpine3.22 AS build
|
||||
|
||||
ARG MINIO_VERSION
|
||||
RUN apk add --no-cache git
|
||||
RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio
|
||||
WORKDIR /src/minio
|
||||
RUN go mod download
|
||||
RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio .
|
||||
|
||||
FROM alpine:3.22.2
|
||||
|
||||
RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data
|
||||
COPY --from=build /out/minio /usr/local/bin/minio
|
||||
|
||||
USER minio
|
||||
VOLUME ["/data"]
|
||||
ENTRYPOINT ["minio"]
|
||||
@@ -0,0 +1,25 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
)
|
||||
|
||||
func RunMigrations() error {
|
||||
settings, err := config.LoadMigration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"io"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||
)
|
||||
|
||||
type reconciliationOutput struct {
|
||||
Mode string `json:"mode"`
|
||||
GracePeriod string `json:"grace_period"`
|
||||
UnreferencedRetention string `json:"unreferenced_retention"`
|
||||
PageSize int `json:"page_size"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func RunReconciliation(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
apply := flags.Bool("apply", false, "apply safe repairs")
|
||||
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
|
||||
retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments")
|
||||
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]")
|
||||
}
|
||||
if *apply && *gracePeriod <= 0 {
|
||||
return errors.New("grace-period must be positive in apply mode")
|
||||
}
|
||||
settings, err := config.LoadReconciliation()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
|
||||
UnreferencedRetention: *retention, Apply: *apply,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mode := "dry-run"
|
||||
if *apply {
|
||||
mode = "apply"
|
||||
}
|
||||
return json.NewEncoder(output).Encode(reconciliationOutput{
|
||||
Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(),
|
||||
PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) {
|
||||
err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard)
|
||||
require.EqualError(t, err, "grace-period must be positive in apply mode")
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/httpapi"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||
)
|
||||
|
||||
func RunServer(logger *slog.Logger) error {
|
||||
settings, err := config.LoadServer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), databaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
sessions, err := session.New(settings.CacheURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sessions.Close() }()
|
||||
passwords, err := password.New(password.Config{
|
||||
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
|
||||
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
|
||||
KeyLength: settings.PasswordKeyLength,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
objects, err := objectstore.New(objectStoreConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repository := postgres.New(pool)
|
||||
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
|
||||
CleanupTimeout: settings.RequestTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app := httpapi.NewServer(logger, httpapi.Dependencies{
|
||||
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
|
||||
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
|
||||
}, httpapi.Config{
|
||||
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
|
||||
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
|
||||
})
|
||||
server := &http.Server{
|
||||
Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
|
||||
ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout,
|
||||
IdleTimeout: settings.HTTPIdleTimeout,
|
||||
}
|
||||
listener, err := net.Listen("tcp", server.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serveResult := make(chan error, 1)
|
||||
go func() { serveResult <- server.Serve(listener) }()
|
||||
logger.Info("server started", "address", server.Addr)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
select {
|
||||
case err := <-serveResult:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
logger.Info("shutdown requested")
|
||||
app.SetReady(false)
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
_ = server.Close()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func databaseConfig(settings config.ServerConfig) database.Config {
|
||||
return database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func objectStoreConfig(settings config.ServerConfig) objectstore.Config {
|
||||
return objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
}
|
||||
}
|
||||
+185
-47
@@ -11,30 +11,41 @@ import (
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Address string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HTTPReadHeaderTimeout time.Duration
|
||||
HTTPReadTimeout time.Duration
|
||||
HTTPWriteTimeout time.Duration
|
||||
HTTPIdleTimeout time.Duration
|
||||
HTTPShutdownTimeout time.Duration
|
||||
PasswordMemoryKiB uint32
|
||||
PasswordIterations uint32
|
||||
PasswordParallelism uint8
|
||||
PasswordSaltLength uint32
|
||||
PasswordKeyLength uint32
|
||||
SwaggerPort string
|
||||
type ServerConfig struct {
|
||||
Address string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HTTPReadHeaderTimeout time.Duration
|
||||
HTTPReadTimeout time.Duration
|
||||
HTTPWriteTimeout time.Duration
|
||||
HTTPIdleTimeout time.Duration
|
||||
HTTPShutdownTimeout time.Duration
|
||||
PasswordMemoryKiB uint32
|
||||
PasswordIterations uint32
|
||||
PasswordParallelism uint8
|
||||
PasswordSaltLength uint32
|
||||
PasswordKeyLength uint32
|
||||
SwaggerPort string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3UseTLS bool
|
||||
S3Timeout time.Duration
|
||||
AttachmentMaxBytes int64
|
||||
AttachmentMaxPixels int64
|
||||
AttachmentBodyMemory int64
|
||||
AttachmentDecodeMemory int64
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
@@ -48,6 +59,23 @@ type MigrationConfig struct {
|
||||
MigrationTimeout time.Duration
|
||||
}
|
||||
|
||||
type ReconciliationConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3UseTLS bool
|
||||
S3Timeout time.Duration
|
||||
}
|
||||
|
||||
type databaseConfig struct {
|
||||
url string
|
||||
maxConns int
|
||||
@@ -58,92 +86,129 @@ type databaseConfig struct {
|
||||
dependencyTimeout time.Duration
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
type objectStorageConfig struct {
|
||||
endpoint string
|
||||
region string
|
||||
bucket string
|
||||
accessKey string
|
||||
secretKey string
|
||||
useTLS bool
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func LoadServer() (ServerConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
|
||||
host, err := requiredString(values, "APP_HOST")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
port, err := requiredPort(values, "APP_PORT")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
cacheURL, err := requiredString(values, "CACHE_URL")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
store, err := requiredString(values, "SESSION_STORE")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if store != "valkey" {
|
||||
return Config{}, errors.New("SESSION_STORE must be valkey")
|
||||
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
|
||||
}
|
||||
|
||||
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
storage, err := loadObjectStorageConfig(values)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
|
||||
}
|
||||
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
|
||||
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||
}
|
||||
|
||||
return Config{
|
||||
return ServerConfig{
|
||||
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
||||
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
|
||||
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
@@ -155,7 +220,12 @@ func Load() (Config, error) {
|
||||
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
|
||||
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
|
||||
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
|
||||
SwaggerPort: swaggerPort,
|
||||
SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
|
||||
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
AttachmentBodyMemory: int64(attachmentBodyMemory),
|
||||
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -178,6 +248,62 @@ func LoadMigration() (MigrationConfig, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadReconciliation() (ReconciliationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
}
|
||||
storage, err := loadObjectStorageConfig(values)
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
}
|
||||
return ReconciliationConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
|
||||
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||
endpoint, err := requiredString(values, "S3_ENDPOINT")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
region, err := requiredString(values, "S3_REGION")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
bucket, err := requiredString(values, "S3_BUCKET")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
secretKey, err := requiredString(values, "S3_SECRET_KEY")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
useTLS, err := boolean(values, "S3_USE_TLS")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
return objectStorageConfig{
|
||||
endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey,
|
||||
secretKey: secretKey, useTLS: useTLS, timeout: timeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||
if err != nil {
|
||||
@@ -260,3 +386,15 @@ func integerValue(value string, minimum, maximum int) (int, error) {
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func boolean(values *viper.Viper, name string) (bool, error) {
|
||||
value, err := requiredString(values, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: must be true or false", name)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
|
||||
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
|
||||
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
|
||||
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
|
||||
}
|
||||
for name, value := range values {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
t.Setenv("APP_HOST", "")
|
||||
t.Setenv("CACHE_URL", "")
|
||||
t.Setenv("SWAGGER_PORT", "")
|
||||
|
||||
settings, err := LoadReconciliation()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "attachments", settings.S3Bucket)
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"mime"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
|
||||
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
|
||||
DeletePendingAttachment(context.Context, string, string) error
|
||||
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Put(context.Context, string, string, []byte) error
|
||||
Get(context.Context, string) (io.ReadCloser, error)
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
BodyMemoryBytes int64
|
||||
DecodeMemoryBytes int64
|
||||
CleanupTimeout time.Duration
|
||||
}
|
||||
|
||||
type Upload struct {
|
||||
ChatID string
|
||||
OwnerID string
|
||||
Filename string
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
Body io.Reader
|
||||
}
|
||||
|
||||
type Content struct {
|
||||
Attachment model.Attachment
|
||||
Body io.ReadCloser
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
repository Repository
|
||||
objects ObjectStore
|
||||
config Config
|
||||
bodyMemory *semaphore.Weighted
|
||||
decodeMemory *semaphore.Weighted
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
|
||||
return nil, errors.New("invalid attachment service configuration")
|
||||
}
|
||||
if config.BodyMemoryBytes < config.MaxBytes {
|
||||
return nil, errors.New("attachment body memory must fit one maximum upload")
|
||||
}
|
||||
if config.DecodeMemoryBytes < config.MaxPixels*8 {
|
||||
return nil, errors.New("attachment decode memory must fit one maximum image")
|
||||
}
|
||||
return &Service{
|
||||
repository: repository, objects: objects, config: config,
|
||||
bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes),
|
||||
decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) {
|
||||
if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil {
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
data, contentType, release, err := s.readImage(input)
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
attachmentID := uuid.NewString()
|
||||
objectKey := "attachments/" + uuid.NewString() + "/original"
|
||||
digest := sha256.Sum256(data)
|
||||
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||
ObjectKey: objectKey, Status: model.AttachmentPending,
|
||||
})
|
||||
if err != nil {
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
|
||||
s.deletePending(pending.ID, objectKey)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
|
||||
if err == nil {
|
||||
return attachment, nil
|
||||
}
|
||||
return s.resolveReadyOutcome(pending.ID, objectKey)
|
||||
}
|
||||
|
||||
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
|
||||
if err == nil && stored.Status == model.AttachmentReady {
|
||||
return stored, nil
|
||||
}
|
||||
if err == nil && stored.Status == model.AttachmentPending {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
|
||||
func (s *Service) deletePending(attachmentID, objectKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
|
||||
attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID)
|
||||
if err != nil {
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
|
||||
attachment, err := s.Metadata(ctx, attachmentID, userID)
|
||||
if err != nil {
|
||||
return Content{}, err
|
||||
}
|
||||
body, err := s.objects.Get(ctx, attachment.ObjectKey)
|
||||
if err != nil {
|
||||
return Content{}, model.ErrUnavailable
|
||||
}
|
||||
return Content{Attachment: attachment, Body: body}, nil
|
||||
}
|
||||
|
||||
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
|
||||
mediaType, parameters, err := mime.ParseMediaType(input.ContentType)
|
||||
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
if input.ContentLength > s.config.MaxBytes {
|
||||
return nil, "", nil, model.ErrPayloadTooLarge
|
||||
}
|
||||
reservation := s.config.MaxBytes
|
||||
if !s.bodyMemory.TryAcquire(reservation) {
|
||||
return nil, "", nil, model.ErrCapacity
|
||||
}
|
||||
releaseBody := func() { s.bodyMemory.Release(reservation) }
|
||||
data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1))
|
||||
if err != nil {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrInvalidInput
|
||||
}
|
||||
if int64(len(data)) > s.config.MaxBytes {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrPayloadTooLarge
|
||||
}
|
||||
if len(data) == 0 {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
detected := http.DetectContentType(data)
|
||||
if detected != mediaType {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || imageContentType(format) != mediaType {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrPayloadTooLarge
|
||||
}
|
||||
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
|
||||
decodeBytes := pixels * 8
|
||||
if !s.decodeMemory.TryAcquire(decodeBytes) {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrCapacity
|
||||
}
|
||||
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
|
||||
s.decodeMemory.Release(decodeBytes)
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
release := func() {
|
||||
s.decodeMemory.Release(decodeBytes)
|
||||
releaseBody()
|
||||
}
|
||||
return data, mediaType, release, nil
|
||||
}
|
||||
|
||||
func imageContentType(format string) string {
|
||||
switch format {
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapDependencyError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, model.ErrNotFound):
|
||||
return model.ErrNotFound
|
||||
case errors.Is(err, model.ErrConflict):
|
||||
return model.ErrConflict
|
||||
default:
|
||||
return model.ErrUnavailable
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
func TestUploadLifecycle(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
events := make([]string, 0, 3)
|
||||
repository := &repositoryStub{events: &events}
|
||||
objects := &objectStub{events: &events}
|
||||
service := newTestService(t, repository, objects, 1024, 4)
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
require.Equal(t, []string{"pending", "put", "ready"}, events)
|
||||
}
|
||||
|
||||
func TestUploadValidation(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
tests := []struct {
|
||||
name string
|
||||
contentType string
|
||||
data []byte
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
want error
|
||||
}{
|
||||
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
|
||||
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge},
|
||||
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge},
|
||||
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels)
|
||||
input := upload(test.data, test.contentType)
|
||||
_, err := service.Upload(context.Background(), input)
|
||||
require.ErrorIs(t, err, test.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
|
||||
require.True(t, service.bodyMemory.TryAcquire(1024))
|
||||
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrCapacity)
|
||||
service.bodyMemory.Release(1024)
|
||||
|
||||
require.True(t, service.decodeMemory.TryAcquire(32))
|
||||
_, err = service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrCapacity)
|
||||
service.decodeMemory.Release(32)
|
||||
}
|
||||
|
||||
func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) {
|
||||
oversized := pngFixture(t, 3, 2)
|
||||
valid := pngFixture(t, 2, 2)
|
||||
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
|
||||
|
||||
_, err := service.Upload(context.Background(), upload(oversized, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrPayloadTooLarge)
|
||||
_, err = service.Upload(context.Background(), upload(valid, "image/png"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
|
||||
_, err := NewService(
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
|
||||
Config{
|
||||
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
|
||||
},
|
||||
)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
repository := &repositoryStub{}
|
||||
objects := &objectStub{putErr: errors.New("s3 unavailable")}
|
||||
service := newTestService(t, repository, objects, 1024, 4)
|
||||
|
||||
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrUnavailable)
|
||||
require.True(t, repository.pendingDeleted)
|
||||
}
|
||||
|
||||
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
|
||||
objects := &objectStub{}
|
||||
service := newTestService(t, repository, objects, 1024, 4)
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
require.False(t, objects.deleted)
|
||||
}
|
||||
|
||||
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
|
||||
t.Helper()
|
||||
service, err := NewService(
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
|
||||
Config{
|
||||
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
|
||||
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return service
|
||||
}
|
||||
|
||||
func upload(data []byte, contentType string) Upload {
|
||||
return Upload{
|
||||
ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType,
|
||||
ContentLength: int64(len(data)), Body: bytes.NewReader(data),
|
||||
}
|
||||
}
|
||||
|
||||
type repositoryStub struct {
|
||||
events *[]string
|
||||
markErr error
|
||||
lookupReady bool
|
||||
pendingDeleted bool
|
||||
pending model.Attachment
|
||||
}
|
||||
|
||||
func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) {
|
||||
return model.Chat{}, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) {
|
||||
attachment.Status = model.AttachmentPending
|
||||
s.pending = attachment
|
||||
s.record("pending")
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.markErr != nil {
|
||||
return model.Attachment{}, s.markErr
|
||||
}
|
||||
s.pending.Status = model.AttachmentReady
|
||||
s.record("ready")
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error {
|
||||
s.pendingDeleted = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.lookupReady {
|
||||
s.pending.Status = model.AttachmentReady
|
||||
return s.pending, nil
|
||||
}
|
||||
return model.Attachment{}, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) record(event string) {
|
||||
if s.events != nil {
|
||||
*s.events = append(*s.events, event)
|
||||
}
|
||||
}
|
||||
|
||||
type objectStub struct {
|
||||
events *[]string
|
||||
putErr error
|
||||
deleted bool
|
||||
}
|
||||
|
||||
func (s *objectStub) Put(context.Context, string, string, []byte) error {
|
||||
if s.events != nil {
|
||||
*s.events = append(*s.events, "put")
|
||||
}
|
||||
return s.putErr
|
||||
}
|
||||
|
||||
func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(nil)), nil
|
||||
}
|
||||
|
||||
func (s *objectStub) Delete(context.Context, string) error {
|
||||
s.deleted = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func pngFixture(t *testing.T, width, height int) []byte {
|
||||
t.Helper()
|
||||
var data bytes.Buffer
|
||||
value := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
value.Set(0, 0, color.White)
|
||||
require.NoError(t, png.Encode(&data, value))
|
||||
return data.Bytes()
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
started := time.Now()
|
||||
result := "client_error"
|
||||
defer func() {
|
||||
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
|
||||
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
|
||||
}()
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
return methodNotAllowed(w, http.MethodPost)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil {
|
||||
return invalidRequest()
|
||||
}
|
||||
filename, valid := attachmentFilename(query)
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, model.ErrCapacity) {
|
||||
result = "capacity_rejected"
|
||||
w.Header().Set("Retry-After", "1")
|
||||
} else if errors.Is(err, model.ErrUnavailable) {
|
||||
result = "dependency_error"
|
||||
}
|
||||
return attachmentError(err)
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||
writeJSON(w, http.StatusCreated, uploaded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func attachmentFilename(query url.Values) (string, bool) {
|
||||
if len(query) != 1 || len(query["filename"]) != 1 {
|
||||
return "", false
|
||||
}
|
||||
filename := query["filename"][0]
|
||||
length := utf8.RuneCountInString(filename)
|
||||
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
|
||||
return "", false
|
||||
}
|
||||
for _, char := range filename {
|
||||
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return filename, true
|
||||
}
|
||||
|
||||
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if r.Method != http.MethodGet {
|
||||
return methodNotAllowed(w, http.MethodGet)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID)
|
||||
if err != nil {
|
||||
return attachmentError(err)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if r.Method != http.MethodGet {
|
||||
return methodNotAllowed(w, http.MethodGet)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || !originalVariant(query) {
|
||||
return invalidRequest()
|
||||
}
|
||||
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
|
||||
if err != nil {
|
||||
return attachmentError(err)
|
||||
}
|
||||
defer func() { _ = content.Body.Close() }()
|
||||
w.Header().Set("Content-Type", content.Attachment.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content.Body); err != nil {
|
||||
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func originalVariant(query url.Values) bool {
|
||||
if len(query) == 0 {
|
||||
return true
|
||||
}
|
||||
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
|
||||
}
|
||||
|
||||
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
|
||||
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||
attachmentID, valid := canonicalID(parts[3])
|
||||
if !valid {
|
||||
return "", invalidRequest()
|
||||
}
|
||||
return attachmentID, nil
|
||||
}
|
||||
|
||||
func attachmentError(err error) *apiError {
|
||||
switch {
|
||||
case errors.Is(err, model.ErrNotFound):
|
||||
return notFound()
|
||||
case errors.Is(err, model.ErrInvalidInput):
|
||||
return invalidRequest()
|
||||
case errors.Is(err, model.ErrUnsupportedMedia):
|
||||
return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
|
||||
case errors.Is(err, model.ErrPayloadTooLarge):
|
||||
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||
case errors.Is(err, model.ErrCapacity):
|
||||
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
|
||||
default:
|
||||
return unavailable()
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||
)
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
@@ -23,8 +21,8 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError
|
||||
if !ok {
|
||||
return unauthorized()
|
||||
}
|
||||
credentials, err := s.messengerStore.UserByUsername(r.Context(), username)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
credentials, err := s.userRepository.UserByUsername(r.Context(), username)
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil {
|
||||
return internalError()
|
||||
}
|
||||
@@ -69,7 +67,7 @@ func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, session.ErrNotFound) {
|
||||
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) {
|
||||
return unauthorized()
|
||||
} else if err != nil {
|
||||
return unavailable()
|
||||
@@ -88,14 +86,14 @@ func (s *Server) authenticate(r *http.Request) (User, string, *apiError) {
|
||||
return User{}, "", unauthorized()
|
||||
}
|
||||
userID, err := s.sessions.UserID(r.Context(), token)
|
||||
if errors.Is(err, session.ErrNotFound) || errors.Is(err, session.ErrInvalidToken) {
|
||||
if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) {
|
||||
return User{}, "", unauthorized()
|
||||
}
|
||||
if err != nil {
|
||||
return User{}, "", unavailable()
|
||||
}
|
||||
user, err := s.messengerStore.UserByID(r.Context(), userID)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
user, err := s.userRepository.UserByID(r.Context(), userID)
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
_ = s.sessions.Delete(r.Context(), token)
|
||||
return User{}, "", unauthorized()
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -14,6 +14,7 @@ const (
|
||||
errorCodePayloadTooLarge = "payload_too_large"
|
||||
errorCodeUnavailable = "unavailable"
|
||||
errorCodeConflict = "conflict"
|
||||
errorCodeRateLimited = "rate_limited"
|
||||
)
|
||||
|
||||
type apiError struct {
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
)
|
||||
|
||||
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
@@ -32,12 +31,15 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := s.messengerStore.Ping(ctx); err != nil {
|
||||
if err := s.database.Ping(ctx); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
if err := s.sessions.Ping(ctx); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
if err := s.objects.Ping(ctx); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
return nil
|
||||
}
|
||||
@@ -61,8 +63,8 @@ func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if err != nil {
|
||||
return internalError()
|
||||
}
|
||||
user, err := s.messengerStore.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
|
||||
if errors.Is(err, postgres.ErrConflict) {
|
||||
user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
|
||||
if errors.Is(err, model.ErrConflict) {
|
||||
return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -125,8 +127,8 @@ func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if _, included := members[user.ID]; !included {
|
||||
return invalidRequest()
|
||||
}
|
||||
chat, err := s.messengerStore.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
|
||||
if errors.Is(err, postgres.ErrInvalidMembers) {
|
||||
chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
|
||||
if errors.Is(err, model.ErrInvalidMembers) {
|
||||
return invalidRequest()
|
||||
}
|
||||
if err != nil {
|
||||
@@ -141,8 +143,8 @@ func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *a
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items, hasNext, listErr := s.messengerStore.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
|
||||
if errors.Is(listErr, postgres.ErrInvalidCursor) {
|
||||
items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
|
||||
if errors.Is(listErr, model.ErrInvalidCursor) {
|
||||
return invalidRequest()
|
||||
}
|
||||
if listErr != nil {
|
||||
@@ -176,8 +178,8 @@ func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
chat, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID)
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
return notFound()
|
||||
}
|
||||
if err != nil {
|
||||
@@ -206,22 +208,21 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
var input struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
var input createMessageRequest
|
||||
if err := readJSON(w, r, &input); err != nil {
|
||||
return err
|
||||
}
|
||||
if !validText(input.Text, 4000) {
|
||||
text, attachmentIDs, valid := input.values()
|
||||
if !valid || !validMessage(text, attachmentIDs) || !canonicalAttachmentIDs(attachmentIDs) {
|
||||
return invalidRequest()
|
||||
}
|
||||
message, replay, err := s.messengerStore.CreateMessage(
|
||||
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
|
||||
message, replay, err := s.messageRepository.CreateMessage(
|
||||
r.Context(), chatID, user.ID, text, attachmentIDs, key, s.idempotencyTTL,
|
||||
)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
return notFound()
|
||||
}
|
||||
if errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
if errors.Is(err, model.ErrIdempotencyConflict) {
|
||||
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -234,16 +235,42 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validMessage(text string, attachmentIDs []string) bool {
|
||||
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 {
|
||||
return false
|
||||
}
|
||||
return len(attachmentIDs) > 0 || strings.TrimSpace(text) != ""
|
||||
}
|
||||
|
||||
func canonicalAttachmentIDs(attachmentIDs []string) bool {
|
||||
if len(attachmentIDs) > 10 {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(attachmentIDs))
|
||||
for index, attachmentID := range attachmentIDs {
|
||||
canonical, valid := canonicalID(attachmentID)
|
||||
if !valid {
|
||||
return false
|
||||
}
|
||||
if _, duplicate := seen[canonical]; duplicate {
|
||||
return false
|
||||
}
|
||||
seen[canonical] = struct{}{}
|
||||
attachmentIDs[index] = canonical
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
|
||||
limit, cursor, err := pageParams(r, "message", chatID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items, hasNext, listErr := s.messengerStore.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
|
||||
if errors.Is(listErr, postgres.ErrNotFound) {
|
||||
items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
|
||||
if errors.Is(listErr, model.ErrNotFound) {
|
||||
return notFound()
|
||||
}
|
||||
if errors.Is(listErr, postgres.ErrInvalidCursor) {
|
||||
if errors.Is(listErr, model.ErrInvalidCursor) {
|
||||
return invalidRequest()
|
||||
}
|
||||
if listErr != nil {
|
||||
+21
-5
@@ -1,26 +1,42 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
type MessengerStore interface {
|
||||
Ping(context.Context) error
|
||||
type UserRepository interface {
|
||||
CreateUser(context.Context, string, string, string) (model.User, error)
|
||||
UserByID(context.Context, string) (model.User, error)
|
||||
UserByUsername(context.Context, string) (model.UserCredentials, error)
|
||||
}
|
||||
|
||||
type ChatRepository interface {
|
||||
CreateChat(context.Context, string, []string, string) (model.Chat, error)
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
|
||||
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
|
||||
}
|
||||
|
||||
type MessageRepository interface {
|
||||
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
|
||||
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
|
||||
}
|
||||
|
||||
type SessionStore interface {
|
||||
type AttachmentService interface {
|
||||
Upload(context.Context, attachment.Upload) (model.Attachment, error)
|
||||
Metadata(context.Context, string, string) (model.Attachment, error)
|
||||
Content(context.Context, string, string) (attachment.Content, error)
|
||||
}
|
||||
|
||||
type HealthChecker interface {
|
||||
Ping(context.Context) error
|
||||
}
|
||||
|
||||
type SessionStore interface {
|
||||
HealthChecker
|
||||
Create(context.Context, string, time.Duration) (string, time.Time, error)
|
||||
UserID(context.Context, string) (string, error)
|
||||
Delete(context.Context, string) error
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -12,11 +12,14 @@ import (
|
||||
)
|
||||
|
||||
type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
handler http.Handler
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
attachmentUploads *prometheus.CounterVec
|
||||
attachmentDuration prometheus.Histogram
|
||||
attachmentBytes prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
@@ -47,12 +50,31 @@ func newMetrics() *metrics {
|
||||
Name: "idempotency_replays_total",
|
||||
Help: "Message requests served from an idempotency record.",
|
||||
}),
|
||||
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_uploads_total",
|
||||
Help: "Completed attachment uploads by result.",
|
||||
}, []string{"result"}),
|
||||
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_duration_seconds",
|
||||
Help: "Attachment upload duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}),
|
||||
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_bytes_total",
|
||||
Help: "Bytes accepted by successful attachment uploads.",
|
||||
}),
|
||||
}
|
||||
registry.MustRegister(
|
||||
result.requests,
|
||||
result.duration,
|
||||
result.inFlight,
|
||||
result.replays,
|
||||
result.attachmentUploads,
|
||||
result.attachmentDuration,
|
||||
result.attachmentBytes,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
@@ -79,5 +101,14 @@ func routeTemplate(path string) string {
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||
return "/api/v1/chats/{chat_id}/messages"
|
||||
}
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||
return "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||
return "/api/v1/attachments/{attachment_id}"
|
||||
}
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||
return "/api/v1/attachments/{attachment_id}/content"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
type User = model.User
|
||||
type Chat = model.Chat
|
||||
type Message = model.Message
|
||||
type Attachment = model.Attachment
|
||||
|
||||
func newID() (string, error) {
|
||||
id, err := uuid.NewRandom()
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -20,6 +20,26 @@ const maxRequestBytes = 1 << 20
|
||||
|
||||
var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text json.RawMessage `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (r createMessageRequest) values() (string, []string, bool) {
|
||||
var text *string
|
||||
if len(r.Text) == 0 || json.Unmarshal(r.Text, &text) != nil || text == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
if len(r.AttachmentIDs) == 0 {
|
||||
return *text, nil, true
|
||||
}
|
||||
var attachmentIDs *[]string
|
||||
if json.Unmarshal(r.AttachmentIDs, &attachmentIDs) != nil || attachmentIDs == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return *text, *attachmentIDs, true
|
||||
}
|
||||
|
||||
func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError {
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -33,6 +33,12 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
handler = s.chat
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||
handler = s.messages
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||
handler = s.uploadAttachment
|
||||
} else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||
handler = s.attachmentMetadata
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||
handler = s.attachmentContent
|
||||
} else {
|
||||
return notFound()
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,18 +8,42 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type Dependencies struct {
|
||||
Users UserRepository
|
||||
Chats ChatRepository
|
||||
Messages MessageRepository
|
||||
Attachments AttachmentService
|
||||
Sessions SessionStore
|
||||
Passwords PasswordService
|
||||
Database HealthChecker
|
||||
Objects HealthChecker
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
RequestTTL time.Duration
|
||||
InstanceID string
|
||||
SwaggerPort string
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
messengerStore MessengerStore
|
||||
sessions SessionStore
|
||||
passwords PasswordService
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
requestTTL time.Duration
|
||||
instanceID string
|
||||
logger *slog.Logger
|
||||
swaggerPort string
|
||||
metrics *metrics
|
||||
ready atomic.Bool
|
||||
userRepository UserRepository
|
||||
chatRepository ChatRepository
|
||||
messageRepository MessageRepository
|
||||
attachments AttachmentService
|
||||
sessions SessionStore
|
||||
passwords PasswordService
|
||||
database HealthChecker
|
||||
objects HealthChecker
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
requestTTL time.Duration
|
||||
instanceID string
|
||||
logger *slog.Logger
|
||||
swaggerPort string
|
||||
metrics *metrics
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
type statusWriter struct {
|
||||
@@ -34,19 +58,16 @@ func (w *statusWriter) WriteHeader(status int) {
|
||||
|
||||
func NewServer(
|
||||
logger *slog.Logger,
|
||||
messages MessengerStore,
|
||||
sessions SessionStore,
|
||||
passwords PasswordService,
|
||||
sessionTTL time.Duration,
|
||||
idempotencyTTL time.Duration,
|
||||
requestTTL time.Duration,
|
||||
instanceID string,
|
||||
swaggerPort string,
|
||||
dependencies Dependencies,
|
||||
config Config,
|
||||
) *Server {
|
||||
s := &Server{
|
||||
messengerStore: messages, sessions: sessions, passwords: passwords,
|
||||
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
|
||||
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
|
||||
userRepository: dependencies.Users, chatRepository: dependencies.Chats,
|
||||
messageRepository: dependencies.Messages,
|
||||
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
|
||||
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
|
||||
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
|
||||
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
|
||||
}
|
||||
s.ready.Store(true)
|
||||
return s
|
||||
@@ -1,4 +1,4 @@
|
||||
package messenger
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -14,8 +14,8 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
)
|
||||
|
||||
var errDependency = errors.New("dependency unavailable")
|
||||
@@ -63,13 +63,34 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
|
||||
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
|
||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) {
|
||||
return s.message, s.messageReplay, s.messageErr
|
||||
}
|
||||
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
type objectStoreStub struct {
|
||||
pingErr error
|
||||
}
|
||||
|
||||
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
|
||||
|
||||
type attachmentServiceStub struct {
|
||||
uploaded model.Attachment
|
||||
err error
|
||||
}
|
||||
|
||||
func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) {
|
||||
return s.uploaded, s.err
|
||||
}
|
||||
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.uploaded, s.err
|
||||
}
|
||||
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
|
||||
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
|
||||
}
|
||||
|
||||
type sessionStub struct {
|
||||
pingErr error
|
||||
userID string
|
||||
@@ -100,14 +121,16 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
|
||||
name string
|
||||
store messengerStoreStub
|
||||
sessions sessionStub
|
||||
objects objectStoreStub
|
||||
}{
|
||||
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
|
||||
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
|
||||
{name: "s3", objects: objectStoreStub{pingErr: errDependency}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{})
|
||||
response := serveRequest(server, http.MethodGet, "/health/ready", "")
|
||||
if response.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
|
||||
@@ -160,6 +183,18 @@ func (s *MessengerSuite) TestAuthenticationChallenges() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() {
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
response := serveRequest(
|
||||
server,
|
||||
http.MethodGet,
|
||||
"/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid",
|
||||
"",
|
||||
)
|
||||
s.Equal(http.StatusUnauthorized, response.Code)
|
||||
s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate"))
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
@@ -224,11 +259,52 @@ func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestMessageCreationValidatesFieldPresenceAndTypes() {
|
||||
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
|
||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||
attachmentID := "00000000-0000-0000-0000-000000000002"
|
||||
invalidBodies := []string{
|
||||
`{"attachment_ids":["` + attachmentID + `"]}`,
|
||||
`{"text":null,"attachment_ids":["` + attachmentID + `"]}`,
|
||||
`{"text":"x","attachment_ids":null}`,
|
||||
`{"text":1}`,
|
||||
`{"text":"x","attachment_ids":{}}`,
|
||||
}
|
||||
for _, body := range invalidBodies {
|
||||
response := createMessageRequestResponse(server, body)
|
||||
s.Equal(http.StatusBadRequest, response.Code, body)
|
||||
}
|
||||
|
||||
validBodies := []string{
|
||||
`{"text":"message"}`,
|
||||
`{"text":"message","attachment_ids":[]}`,
|
||||
`{"text":"","attachment_ids":["` + attachmentID + `"]}`,
|
||||
}
|
||||
for _, body := range validBodies {
|
||||
response := createMessageRequestResponse(server, body)
|
||||
s.Equal(http.StatusCreated, response.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func createMessageRequestResponse(server *Server, body string) *httptest.ResponseRecorder {
|
||||
request := httptest.NewRequest(
|
||||
http.MethodPost,
|
||||
"/api/v1/chats/00000000-0000-0000-0000-000000000001/messages",
|
||||
strings.NewReader(body),
|
||||
)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", "message-key")
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
return response
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() {
|
||||
t := s.T()
|
||||
store := messengerStoreStub{
|
||||
userByID: model.User{ID: "user-id"},
|
||||
messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict),
|
||||
messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict),
|
||||
}
|
||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
|
||||
@@ -242,8 +318,49 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
|
||||
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
|
||||
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
|
||||
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
|
||||
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
|
||||
s.False(valid, filename)
|
||||
}
|
||||
filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}})
|
||||
s.True(valid)
|
||||
s.Equal("фото.png", filename)
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestUploadRejectsMalformedQuery() {
|
||||
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
|
||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||
request := httptest.NewRequest(
|
||||
http.MethodPost,
|
||||
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG",
|
||||
strings.NewReader("image"),
|
||||
)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
s.Equal(http.StatusBadRequest, response.Code)
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server {
|
||||
return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{})
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) server(
|
||||
store messengerStoreStub,
|
||||
sessions SessionStore,
|
||||
passwords PasswordService,
|
||||
objects objectStoreStub,
|
||||
attachments AttachmentService,
|
||||
) *Server {
|
||||
return NewServer(s.logger, Dependencies{
|
||||
Users: store, Chats: store, Messages: store, Attachments: attachments,
|
||||
Sessions: sessions, Passwords: passwords, Database: store, Objects: objects,
|
||||
}, Config{
|
||||
SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second,
|
||||
InstanceID: "test-instance", SwaggerPort: "8081",
|
||||
})
|
||||
}
|
||||
|
||||
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
|
||||
@@ -0,0 +1,18 @@
|
||||
package model
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrInvalidMembers = errors.New("invalid members")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrIdempotencyConflict = errors.New("idempotency conflict")
|
||||
ErrInvalidToken = errors.New("invalid token")
|
||||
ErrSessionNotFound = errors.New("session not found")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrUnsupportedMedia = errors.New("unsupported media")
|
||||
ErrPayloadTooLarge = errors.New("payload too large")
|
||||
ErrCapacity = errors.New("capacity exhausted")
|
||||
ErrUnavailable = errors.New("dependency unavailable")
|
||||
)
|
||||
+82
-11
@@ -5,6 +5,12 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
AttachmentPending = "pending"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentDeleting = "deleting"
|
||||
)
|
||||
|
||||
const timestampLayout = "2006-01-02T15:04:05.000000Z"
|
||||
|
||||
type User struct {
|
||||
@@ -27,11 +33,45 @@ type Chat struct {
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
ReadyAt *time.Time `json:"-"`
|
||||
Referenced bool `json:"-"`
|
||||
}
|
||||
|
||||
type StoredObject struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type Position struct {
|
||||
@@ -54,11 +94,27 @@ type chatJSON struct {
|
||||
}
|
||||
|
||||
type messageJSON struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type attachmentJSON struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (u User) MarshalJSON() ([]byte, error) {
|
||||
@@ -70,7 +126,22 @@ func (c Chat) MarshalJSON() ([]byte, error) {
|
||||
}
|
||||
|
||||
func (m Message) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt)})
|
||||
attachments := m.AttachmentIDs
|
||||
if attachments == nil {
|
||||
attachments = []string{}
|
||||
}
|
||||
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments})
|
||||
}
|
||||
|
||||
func (a Attachment) MarshalJSON() ([]byte, error) {
|
||||
variants := a.Variants
|
||||
if variants == nil {
|
||||
variants = []Variant{}
|
||||
}
|
||||
return json.Marshal(attachmentJSON{
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
|
||||
a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt),
|
||||
})
|
||||
}
|
||||
|
||||
func FormatTime(value time.Time) string {
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseTLS bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func New(config Config) (*Store, error) {
|
||||
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
|
||||
return nil, errors.New("invalid object store configuration")
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
|
||||
Secure: config.UseTLS,
|
||||
Region: config.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create S3 client: %w", err)
|
||||
}
|
||||
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
|
||||
}
|
||||
|
||||
func endpointHost(value string, useTLS bool) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if !strings.Contains(value, "://") {
|
||||
if value == "" {
|
||||
return "", errors.New("S3 endpoint is required")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
|
||||
return "", errors.New("invalid S3 endpoint")
|
||||
}
|
||||
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
|
||||
}
|
||||
return parsed.Host, nil
|
||||
}
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check S3 bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("S3 bucket does not exist")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
_, err := s.client.PutObject(
|
||||
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, objectError("get S3 object", err)
|
||||
}
|
||||
if _, err := object.Stat(); err != nil {
|
||||
cancel()
|
||||
_ = object.Close()
|
||||
return nil, objectError("stat S3 object", err)
|
||||
}
|
||||
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return model.StoredObject{}, objectError("stat S3 object", err)
|
||||
}
|
||||
return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
return objectError("delete S3 object", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
|
||||
if limit < 1 || limit > 1000 {
|
||||
return nil, "", errors.New("object list limit must be between 1 and 1000")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
objects := make([]model.StoredObject, 0, limit+1)
|
||||
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
|
||||
for info := range s.client.ListObjects(ctx, s.bucket, options) {
|
||||
if info.Err != nil {
|
||||
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
|
||||
}
|
||||
objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
|
||||
if len(objects) == limit+1 {
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
}
|
||||
var next string
|
||||
if len(objects) > limit {
|
||||
objects = objects[:limit]
|
||||
next = objects[len(objects)-1].Key
|
||||
}
|
||||
return objects, next, nil
|
||||
}
|
||||
|
||||
type timeoutReader struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r *timeoutReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
r.cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
func objectError(action string, err error) error {
|
||||
response := minio.ToErrorResponse(err)
|
||||
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
|
||||
return ErrNotFound
|
||||
}
|
||||
return fmt.Errorf("%s: %w", action, err)
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
const attachmentColumns = `
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, created_at, updated_at, ready_at`
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(...any) error
|
||||
}
|
||||
|
||||
func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
|
||||
query := `
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at
|
||||
)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
|
||||
)
|
||||
RETURNING ` + attachmentColumns
|
||||
|
||||
err := scanAttachment(s.pool.QueryRow(
|
||||
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
|
||||
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
|
||||
), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||
query := `
|
||||
UPDATE attachments
|
||||
SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND object_key = $2 AND status = 'pending'
|
||||
RETURNING ` + attachmentColumns
|
||||
|
||||
var attachment model.Attachment
|
||||
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error {
|
||||
const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'`
|
||||
if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil {
|
||||
return fmt.Errorf("delete pending attachment: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
|
||||
query := `
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachments a
|
||||
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
|
||||
WHERE a.id = $1 AND a.status = 'ready'`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
|
||||
var exists bool
|
||||
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check attachment by object key: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
|
||||
if limit < 1 || limit > 1000 {
|
||||
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
|
||||
}
|
||||
query := `
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `,
|
||||
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
|
||||
FROM attachments a
|
||||
WHERE a.object_key > $1
|
||||
ORDER BY a.object_key
|
||||
LIMIT $2`
|
||||
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("list attachments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
|
||||
var attachment model.Attachment
|
||||
err := scanAttachmentWithReference(row, &attachment)
|
||||
return publicAttachment(attachment), err
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("scan attachments: %w", err)
|
||||
}
|
||||
var next string
|
||||
if len(attachments) > limit {
|
||||
attachments = attachments[:limit]
|
||||
next = attachments[len(attachments)-1].ObjectKey
|
||||
}
|
||||
return attachments, next, nil
|
||||
}
|
||||
|
||||
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.Referenced,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
|
||||
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
|
||||
var referenced bool
|
||||
if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil {
|
||||
return false, fmt.Errorf("check attachment references: %w", err)
|
||||
}
|
||||
return referenced, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
var status string
|
||||
var createdAt time.Time
|
||||
var readyAt *time.Time
|
||||
err = tx.QueryRow(
|
||||
ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID,
|
||||
).Scan(&status, &createdAt, &readyAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
|
||||
}
|
||||
if status != candidate.Status {
|
||||
return false, nil
|
||||
}
|
||||
var age time.Time
|
||||
switch status {
|
||||
case model.AttachmentPending:
|
||||
age = createdAt
|
||||
case model.AttachmentReady:
|
||||
if readyAt == nil {
|
||||
return false, nil
|
||||
}
|
||||
age = *readyAt
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
if age.After(createdBefore) {
|
||||
return false, nil
|
||||
}
|
||||
var referenced bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil {
|
||||
return false, fmt.Errorf("recheck attachment references: %w", err)
|
||||
}
|
||||
if referenced {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
ctx,
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`,
|
||||
candidate.ID, candidate.Status,
|
||||
); err != nil {
|
||||
return false, fmt.Errorf("mark attachment deleting: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit attachment deletion claim: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) {
|
||||
const query = `
|
||||
DELETE FROM attachments a
|
||||
WHERE a.id = $1
|
||||
AND a.status = 'deleting'
|
||||
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
|
||||
result, err := s.pool.Exec(ctx, query, attachmentID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("delete claimed attachment: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
)
|
||||
}
|
||||
|
||||
func attachmentColumnsWithAlias(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
|
||||
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
|
||||
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
|
||||
alias,
|
||||
)
|
||||
}
|
||||
|
||||
func publicAttachment(attachment model.Attachment) model.Attachment {
|
||||
attachment.Variants = []model.Variant{}
|
||||
return attachment
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
package postgres
|
||||
|
||||
import "errors"
|
||||
import "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
|
||||
var (
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrInvalidMembers = errors.New("invalid members")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrIdempotencyConflict = errors.New("idempotency conflict")
|
||||
ErrConflict = model.ErrConflict
|
||||
ErrNotFound = model.ErrNotFound
|
||||
ErrInvalidMembers = model.ErrInvalidMembers
|
||||
ErrInvalidCursor = model.ErrInvalidCursor
|
||||
ErrIdempotencyConflict = model.ErrIdempotencyConflict
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
@@ -21,9 +22,13 @@ func (s *Store) CreateMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
idempotencyKey string,
|
||||
idempotencyTTL time.Duration,
|
||||
) (model.Message, bool, error) {
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = []string{}
|
||||
}
|
||||
var (
|
||||
tx pgx.Tx
|
||||
err error
|
||||
@@ -46,13 +51,23 @@ func (s *Store) CreateMessage(
|
||||
if !visible {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !validAttachments {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
|
||||
requestHash = sha256.Sum256([]byte(text))
|
||||
requestHash, err = messageRequestHash(text, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
message, replay, err = createIdempotentMessage(
|
||||
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||
ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||
)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
@@ -63,6 +78,44 @@ func (s *Store) CreateMessage(
|
||||
return message, replay, nil
|
||||
}
|
||||
|
||||
func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) {
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = []string{}
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}{text, attachmentIDs})
|
||||
if err != nil {
|
||||
return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err)
|
||||
}
|
||||
return sha256.Sum256(payload), nil
|
||||
}
|
||||
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
|
||||
if len(attachmentIDs) == 0 {
|
||||
return true, nil
|
||||
}
|
||||
const query = `
|
||||
SELECT id
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
|
||||
FOR UPDATE`
|
||||
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
count++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
return count == len(attachmentIDs), nil
|
||||
}
|
||||
|
||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||
const query = `
|
||||
SELECT EXISTS (
|
||||
@@ -84,6 +137,7 @@ func createIdempotentMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
idempotencyKey string,
|
||||
requestHash []byte,
|
||||
idempotencyTTL time.Duration,
|
||||
@@ -126,7 +180,7 @@ func createIdempotentMessage(
|
||||
continue
|
||||
}
|
||||
|
||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
|
||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs)
|
||||
return message, false, err
|
||||
}
|
||||
}
|
||||
@@ -235,6 +289,7 @@ func insertMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
) (model.Message, error) {
|
||||
const query = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
@@ -255,14 +310,37 @@ func insertMessage(
|
||||
if err != nil {
|
||||
return model.Message{}, fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil {
|
||||
return model.Message{}, err
|
||||
}
|
||||
message.AttachmentIDs = make([]string, len(attachmentIDs))
|
||||
copy(message.AttachmentIDs, attachmentIDs)
|
||||
return message, nil
|
||||
}
|
||||
|
||||
func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error {
|
||||
const query = `
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES ($1, $2, $3, $4)`
|
||||
for position, attachmentID := range attachmentIDs {
|
||||
if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil {
|
||||
return fmt.Errorf("insert message attachment: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
|
||||
const query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE id = $1`
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(
|
||||
SELECT ma.attachment_id::text
|
||||
FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id
|
||||
ORDER BY ma.position
|
||||
)
|
||||
FROM messages m
|
||||
WHERE m.id = $1`
|
||||
|
||||
var (
|
||||
message model.Message
|
||||
@@ -274,6 +352,7 @@ func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Messag
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Message{}, fmt.Errorf("idempotency result message not found")
|
||||
|
||||
@@ -344,11 +344,17 @@ func (s *Store) ListMessages(
|
||||
}
|
||||
|
||||
const query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid))
|
||||
ORDER BY created_at, id
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(
|
||||
SELECT ma.attachment_id::text
|
||||
FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id
|
||||
ORDER BY ma.position
|
||||
)
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid))
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $4`
|
||||
|
||||
var afterTime any
|
||||
@@ -372,6 +378,7 @@ func (s *Store) ListMessages(
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
)
|
||||
return message, err
|
||||
}))
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||
)
|
||||
|
||||
type PostgresSuite struct {
|
||||
@@ -100,18 +101,16 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
||||
owner, chatID := createOwnerAndChat(t, store)
|
||||
key := uuid.NewString()
|
||||
|
||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour)
|
||||
if err != nil || replay {
|
||||
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
|
||||
}
|
||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour)
|
||||
if err != nil || !replay {
|
||||
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("replay message = %v, want %v", second, first)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
s.Equal(first, second)
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
|
||||
}
|
||||
if count := messageCount(t, pool, chatID); count != 1 {
|
||||
@@ -119,6 +118,136 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash")
|
||||
s.Require().NoError(err)
|
||||
attachment := model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
}
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), attachment)
|
||||
s.Require().NoError(err)
|
||||
created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||
s.Require().NoError(err)
|
||||
s.Equal("ready", created.Status)
|
||||
s.Empty(created.Variants)
|
||||
s.Nil(created.ErrorCode)
|
||||
|
||||
visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(created, visible)
|
||||
_, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID)
|
||||
s.ErrorIs(err, postgres.ErrNotFound)
|
||||
|
||||
message, replay, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.False(replay)
|
||||
s.Equal([]string{created.ID}, message.AttachmentIDs)
|
||||
|
||||
messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10)
|
||||
s.Require().NoError(err)
|
||||
s.Len(messages, 1)
|
||||
s.Equal([]string{created.ID}, messages[0].AttachmentIDs)
|
||||
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(referenced)
|
||||
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.False(deleted)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
|
||||
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.True(claimed)
|
||||
deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(deleted)
|
||||
|
||||
pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||
s.Require().NoError(err)
|
||||
|
||||
claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.False(claimed)
|
||||
visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentReady, visible.Status)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestReconciliationDryRunPreservesDeletingAttachment() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "deleting.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("e", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.True(claimed)
|
||||
|
||||
reconciler, err := reconcile.New(s.store, emptyObjectStore{}, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: 1000, GraceTime: time.Minute,
|
||||
UnreferencedRetention: time.Hour,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
report, err := reconciler.Run(context.Background())
|
||||
s.Require().NoError(err)
|
||||
s.Zero(report.DeletedUnreferencedRows)
|
||||
|
||||
var status string
|
||||
err = s.pool.QueryRow(context.Background(), "SELECT status FROM attachments WHERE id = $1", pending.ID).Scan(&status)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentDeleting, status)
|
||||
}
|
||||
|
||||
type emptyObjectStore struct{}
|
||||
|
||||
func (emptyObjectStore) List(context.Context, string, string, int) ([]model.StoredObject, string, error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func (emptyObjectStore) Delete(context.Context, string) error {
|
||||
return errors.New("unexpected object deletion")
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
|
||||
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
|
||||
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||
s.Require().NoError(err)
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.ErrorIs(err, postgres.ErrNotFound)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
@@ -139,7 +268,7 @@ func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||
go func() {
|
||||
defer group.Done()
|
||||
message, replay, err := stores[worker%len(stores)].CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
|
||||
context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour,
|
||||
)
|
||||
results <- result{message: message, replay: replay, err: err}
|
||||
}()
|
||||
@@ -186,13 +315,13 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
||||
}
|
||||
key := uuid.NewString()
|
||||
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("owner CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("second user CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("other chat CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(context.Background(), `
|
||||
@@ -202,7 +331,7 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
|
||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour)
|
||||
if err != nil || replay || message.Text != "after expiry" {
|
||||
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
|
||||
}
|
||||
@@ -212,7 +341,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
||||
t := s.T()
|
||||
owner, chatID := createOwnerAndChat(t, s.store)
|
||||
expiredKey := uuid.NewString()
|
||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
|
||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil {
|
||||
t.Fatalf("create expired message: %v", err)
|
||||
}
|
||||
if _, err := s.pool.Exec(context.Background(), `
|
||||
@@ -223,7 +352,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
if _, _, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
|
||||
context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour,
|
||||
); err != nil {
|
||||
t.Fatalf("create new message: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
type MetadataStore interface {
|
||||
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
|
||||
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
|
||||
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
|
||||
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
List(context.Context, string, string, int) ([]model.StoredObject, string, error)
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Prefix string
|
||||
PageSize int
|
||||
GraceTime time.Duration
|
||||
UnreferencedRetention time.Duration
|
||||
Apply bool
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
MetadataScanned int `json:"metadata_scanned"`
|
||||
ObjectsScanned int `json:"objects_scanned"`
|
||||
Consistent int `json:"consistent"`
|
||||
FreshPendingAttachments int `json:"fresh_pending_attachments"`
|
||||
OldPendingAttachments int `json:"old_pending_attachments"`
|
||||
DeletingAttachments int `json:"deleting_attachments"`
|
||||
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
|
||||
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
|
||||
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
|
||||
FreshOrphanObjects int `json:"fresh_orphan_objects"`
|
||||
OldOrphanObjects int `json:"old_orphan_objects"`
|
||||
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
|
||||
FreshMissingObjects int `json:"fresh_missing_objects"`
|
||||
ReferencedMissingObjects int `json:"referenced_missing_objects"`
|
||||
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
|
||||
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
metadata MetadataStore
|
||||
objects ObjectStore
|
||||
config Config
|
||||
}
|
||||
|
||||
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
|
||||
if config.PageSize < 1 || config.PageSize > 1000 {
|
||||
return nil, errors.New("page size must be between 1 and 1000")
|
||||
}
|
||||
if config.GraceTime < 0 {
|
||||
return nil, errors.New("grace time must not be negative")
|
||||
}
|
||||
if config.UnreferencedRetention <= 0 {
|
||||
return nil, errors.New("unreferenced retention must be positive")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &Reconciler{metadata: metadata, objects: objects, config: config}, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) Run(ctx context.Context) (Report, error) {
|
||||
metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
now := r.config.Now()
|
||||
anomalyCutoff := now.Add(-r.config.GraceTime)
|
||||
retentionCutoff := now.Add(-r.config.UnreferencedRetention)
|
||||
var report Report
|
||||
for metadata.current != nil || objects.current != nil {
|
||||
switch {
|
||||
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
|
||||
if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
|
||||
if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := metadata.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
default:
|
||||
if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := metadata.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handlePair(
|
||||
ctx context.Context,
|
||||
metadata model.Attachment,
|
||||
retentionCutoff time.Time,
|
||||
anomalyCutoff time.Time,
|
||||
report *Report,
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
report.ObjectsScanned++
|
||||
switch metadata.Status {
|
||||
case model.AttachmentPending:
|
||||
if metadata.CreatedAt.After(anomalyCutoff) {
|
||||
report.FreshPendingAttachments++
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
report.OldPendingAttachments++
|
||||
return r.deletePair(ctx, metadata, anomalyCutoff, report)
|
||||
case model.AttachmentDeleting:
|
||||
report.DeletingAttachments++
|
||||
return r.deleteClaimedPair(ctx, metadata, report)
|
||||
case model.AttachmentReady:
|
||||
if metadata.Referenced {
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
age := metadata.CreatedAt
|
||||
if metadata.ReadyAt != nil {
|
||||
age = *metadata.ReadyAt
|
||||
}
|
||||
if age.After(retentionCutoff) {
|
||||
report.FreshUnreferencedAttachments++
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
report.OldUnreferencedAttachments++
|
||||
return r.deletePair(ctx, metadata, retentionCutoff, report)
|
||||
default:
|
||||
return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error {
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
|
||||
}
|
||||
if !claimed {
|
||||
return nil
|
||||
}
|
||||
return r.deleteClaimedPair(ctx, attachment, report)
|
||||
}
|
||||
|
||||
func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error {
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
|
||||
return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err)
|
||||
}
|
||||
report.DeletedAttachmentObjects++
|
||||
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err)
|
||||
}
|
||||
if deleted {
|
||||
report.DeletedUnreferencedRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handleOrphanObject(
|
||||
ctx context.Context,
|
||||
object model.StoredObject,
|
||||
cutoff time.Time,
|
||||
report *Report,
|
||||
) error {
|
||||
report.ObjectsScanned++
|
||||
if object.LastModified.After(cutoff) {
|
||||
report.FreshOrphanObjects++
|
||||
return nil
|
||||
}
|
||||
report.OldOrphanObjects++
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
if err := r.objects.Delete(ctx, object.Key); err != nil {
|
||||
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
report.DeletedOrphanObjects++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handleMissingObject(
|
||||
ctx context.Context,
|
||||
metadata model.Attachment,
|
||||
cutoff time.Time,
|
||||
report *Report,
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
if metadata.Status == model.AttachmentDeleting {
|
||||
report.DeletingAttachments++
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
if metadata.CreatedAt.After(cutoff) {
|
||||
report.FreshMissingObjects++
|
||||
return nil
|
||||
}
|
||||
if metadata.Referenced {
|
||||
report.ReferencedMissingObjects++
|
||||
return nil
|
||||
}
|
||||
report.OldUnreferencedMetadata++
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
|
||||
}
|
||||
if !claimed {
|
||||
return nil
|
||||
}
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
|
||||
func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error {
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
|
||||
}
|
||||
if deleted {
|
||||
report.DeletedUnreferencedRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type metadataIterator struct {
|
||||
store MetadataStore
|
||||
pageSize int
|
||||
after string
|
||||
page []model.Attachment
|
||||
index int
|
||||
current *model.Attachment
|
||||
done bool
|
||||
}
|
||||
|
||||
func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) {
|
||||
iterator := &metadataIterator{store: store, pageSize: pageSize}
|
||||
if err := iterator.advance(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return iterator, nil
|
||||
}
|
||||
|
||||
func (i *metadataIterator) advance(ctx context.Context) error {
|
||||
for {
|
||||
if i.index < len(i.page) {
|
||||
i.current = &i.page[i.index]
|
||||
i.index++
|
||||
return nil
|
||||
}
|
||||
if i.done {
|
||||
i.current = nil
|
||||
return nil
|
||||
}
|
||||
page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list attachment metadata: %w", err)
|
||||
}
|
||||
i.page = page
|
||||
i.index = 0
|
||||
i.after = next
|
||||
i.done = next == ""
|
||||
if len(page) == 0 && !i.done {
|
||||
return errors.New("metadata page is empty but has continuation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type objectIterator struct {
|
||||
store ObjectStore
|
||||
prefix string
|
||||
pageSize int
|
||||
after string
|
||||
page []model.StoredObject
|
||||
index int
|
||||
current *model.StoredObject
|
||||
done bool
|
||||
}
|
||||
|
||||
func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) {
|
||||
iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize}
|
||||
if err := iterator.advance(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return iterator, nil
|
||||
}
|
||||
|
||||
func (i *objectIterator) advance(ctx context.Context) error {
|
||||
for {
|
||||
if i.index < len(i.page) {
|
||||
i.current = &i.page[i.index]
|
||||
i.index++
|
||||
return nil
|
||||
}
|
||||
if i.done {
|
||||
i.current = nil
|
||||
return nil
|
||||
}
|
||||
page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
i.page = page
|
||||
i.index = 0
|
||||
i.after = next
|
||||
i.done = next == ""
|
||||
if len(page) == 0 && !i.done {
|
||||
return errors.New("object page is empty but has continuation")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
fresh := now.Add(-10 * time.Minute)
|
||||
metadata := &metadataStub{items: map[string]model.Attachment{
|
||||
"attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false),
|
||||
"attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true),
|
||||
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
|
||||
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
|
||||
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
|
||||
}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
|
||||
"attachments/b/original": {Key: "attachments/b/original", LastModified: old},
|
||||
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
|
||||
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
|
||||
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
|
||||
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, report.OldPendingAttachments)
|
||||
require.Equal(t, 1, report.OldUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.FreshUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.DeletedOrphanObjects)
|
||||
require.Equal(t, 2, report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 3, report.DeletedUnreferencedRows)
|
||||
require.Contains(t, metadata.items, "attachments/b/original")
|
||||
require.Contains(t, metadata.items, "attachments/d/original")
|
||||
require.NotContains(t, metadata.items, "attachments/a/original")
|
||||
require.NotContains(t, metadata.items, "attachments/c/original")
|
||||
require.NotContains(t, metadata.items, "attachments/missing/original")
|
||||
require.NotContains(t, objects.items, "attachments/orphan/original")
|
||||
|
||||
repeated, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, repeated.DeletedOrphanObjects)
|
||||
require.Zero(t, repeated.DeletedAttachmentObjects)
|
||||
require.Zero(t, repeated.DeletedUnreferencedRows)
|
||||
}
|
||||
|
||||
func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
"attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
|
||||
UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, report.DeletedOrphanObjects)
|
||||
require.Contains(t, objects.items, "attachments/a/original")
|
||||
}
|
||||
|
||||
func TestReconcilerDeletingAttachmentRespectsApplyMode(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
key := "attachments/a/original"
|
||||
tests := []struct {
|
||||
name string
|
||||
apply bool
|
||||
objectPresent bool
|
||||
}{
|
||||
{name: "dry run with object", objectPresent: true},
|
||||
{name: "dry run without object"},
|
||||
{name: "apply with object", apply: true, objectPresent: true},
|
||||
{name: "apply without object", apply: true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
metadata := &metadataStub{items: map[string]model.Attachment{
|
||||
key: attachment("a", key, model.AttachmentDeleting, now.Add(-time.Hour), false),
|
||||
}}
|
||||
objects := &objectStub{items: make(map[string]model.StoredObject)}
|
||||
if test.objectPresent {
|
||||
objects.items[key] = model.StoredObject{Key: key, LastModified: now.Add(-time.Hour)}
|
||||
}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
|
||||
UnreferencedRetention: time.Hour, Apply: test.apply, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, report.DeletingAttachments)
|
||||
var expectedDeletedRows, expectedDeletedObjects int
|
||||
if test.apply {
|
||||
expectedDeletedRows = 1
|
||||
}
|
||||
if test.apply && test.objectPresent {
|
||||
expectedDeletedObjects = 1
|
||||
}
|
||||
require.Equal(t, expectedDeletedRows, report.DeletedUnreferencedRows)
|
||||
require.Equal(t, expectedDeletedObjects, report.DeletedAttachmentObjects)
|
||||
_, metadataExists := metadata.items[key]
|
||||
require.Equal(t, !test.apply, metadataExists)
|
||||
_, objectExists := objects.items[key]
|
||||
require.Equal(t, test.objectPresent && !test.apply, objectExists)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRejectsInvalidRetention(t *testing.T) {
|
||||
_, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment {
|
||||
readyAt := createdAt
|
||||
return model.Attachment{
|
||||
ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt,
|
||||
ReadyAt: &readyAt, Referenced: referenced,
|
||||
}
|
||||
}
|
||||
|
||||
type metadataStub struct {
|
||||
items map[string]model.Attachment
|
||||
appeared map[string]bool
|
||||
}
|
||||
|
||||
func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) {
|
||||
keys := make([]string, 0, len(s.items))
|
||||
for key := range s.items {
|
||||
if key > after {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var next string
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
next = keys[len(keys)-1]
|
||||
}
|
||||
items := make([]model.Attachment, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
items = append(items, s.items[key])
|
||||
}
|
||||
return items, next, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) {
|
||||
if s.appeared[key] {
|
||||
return true, nil
|
||||
}
|
||||
_, exists := s.items[key]
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
|
||||
for key, item := range s.items {
|
||||
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
|
||||
continue
|
||||
}
|
||||
age := item.CreatedAt
|
||||
if item.Status == model.AttachmentReady && item.ReadyAt != nil {
|
||||
age = *item.ReadyAt
|
||||
}
|
||||
if age.After(cutoff) {
|
||||
return false, nil
|
||||
}
|
||||
item.Status = model.AttachmentDeleting
|
||||
s.items[key] = item
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) {
|
||||
for key, item := range s.items {
|
||||
if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced {
|
||||
delete(s.items, key)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type objectStub struct {
|
||||
items map[string]model.StoredObject
|
||||
}
|
||||
|
||||
func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
|
||||
keys := make([]string, 0, len(s.items))
|
||||
for key := range s.items {
|
||||
if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var next string
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
next = keys[len(keys)-1]
|
||||
}
|
||||
items := make([]model.StoredObject, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
items = append(items, s.items[key])
|
||||
}
|
||||
return items, next, nil
|
||||
}
|
||||
|
||||
func (s *objectStub) Delete(_ context.Context, key string) error {
|
||||
delete(s.items, key)
|
||||
return nil
|
||||
}
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/valkey-io/valkey-go"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,12 +23,12 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("session not found")
|
||||
ErrInvalidToken = errors.New("invalid session token")
|
||||
ErrNotFound = model.ErrSessionNotFound
|
||||
ErrInvalidToken = model.ErrInvalidToken
|
||||
ErrInvalidRecord = errors.New("invalid session record")
|
||||
ErrInvalidTTL = errors.New("invalid session TTL")
|
||||
ErrTokenGeneration = errors.New("session token generation failed")
|
||||
ErrStore = errors.New("session store failure")
|
||||
ErrStore = model.ErrUnavailable
|
||||
)
|
||||
|
||||
type record struct {
|
||||
|
||||
+22
-1
@@ -26,6 +26,23 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge
|
||||
VALKEY_PASSWORD=local-valkey-password
|
||||
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
||||
|
||||
# Административные данные MinIO используются только при bootstrap.
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=local-minio-admin-password
|
||||
|
||||
# API использует отдельного пользователя с доступом только к bucket вложений.
|
||||
S3_ENDPOINT=s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY=messenger-app
|
||||
S3_SECRET_KEY=local-messenger-s3-password
|
||||
S3_USE_TLS=false
|
||||
S3_TIMEOUT_SECONDS=5
|
||||
ATTACHMENT_MAX_BYTES=10485760
|
||||
ATTACHMENT_MAX_PIXELS=40000000
|
||||
ATTACHMENT_BODY_MEMORY_BYTES=67108864
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
|
||||
|
||||
# Размер пула и обслуживание соединений PostgreSQL.
|
||||
DB_MIN_CONNS=1
|
||||
DB_MAX_CONNS=10
|
||||
@@ -55,7 +72,7 @@ SWAGGER_PORT=8081
|
||||
|
||||
# Ограничения ресурсов контейнеров.
|
||||
API_CPUS=1
|
||||
API_MEMORY=256m
|
||||
API_MEMORY=512m
|
||||
MIGRATE_CPUS=0.5
|
||||
MIGRATE_MEMORY=128m
|
||||
NGINX_CPUS=0.5
|
||||
@@ -74,6 +91,10 @@ POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
VALKEY_MEMORY=256m
|
||||
S3_CPUS=1
|
||||
S3_MEMORY=512m
|
||||
S3_BOOTSTRAP_CPUS=0.25
|
||||
S3_BOOTSTRAP_MEMORY=256m
|
||||
|
||||
# Пароль локального администратора Grafana.
|
||||
GRAFANA_ADMIN_PASSWORD=local-grafana-password
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_length,
|
||||
DROP CONSTRAINT messages_text_not_blank;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000),
|
||||
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL,
|
||||
owner_id UUID NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
|
||||
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
|
||||
CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
|
||||
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL,
|
||||
position SMALLINT NOT NULL,
|
||||
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments (id, chat_id) ON DELETE RESTRICT
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,16 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN status TEXT NOT NULL DEFAULT 'ready',
|
||||
ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp();
|
||||
|
||||
UPDATE attachments
|
||||
SET ready_at = created_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')),
|
||||
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||
(status = 'pending' AND ready_at IS NULL) OR
|
||||
(status IN ('ready', 'deleting') AND ready_at IS NOT NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_status_created_at_object_key_idx
|
||||
ON attachments (status, created_at, object_key);
|
||||
@@ -0,0 +1,9 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||
(status = 'pending' AND ready_at IS NULL) OR
|
||||
(status = 'ready' AND ready_at IS NOT NULL) OR
|
||||
status = 'deleting'
|
||||
);
|
||||
@@ -85,6 +85,50 @@
|
||||
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
|
||||
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment uploads by result", "id": 15,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 56},
|
||||
"targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment upload latency p95", "id": 16,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 56},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment upload traffic", "id": 17,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 64},
|
||||
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
|
||||
"targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO objects", "id": 18,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 64},
|
||||
"targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO requests and errors", "id": 19,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 64},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"},
|
||||
{"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO availability", "id": 20,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 72},
|
||||
"targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO storage", "id": 21,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 72},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"targets": [
|
||||
{"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"},
|
||||
{"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
@@ -93,7 +137,7 @@
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-15m", "to": "now"},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger lab3 overview",
|
||||
"title": "Messenger lab4 overview",
|
||||
"uid": "messenger-overview",
|
||||
"version": 2
|
||||
"version": 3
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
@@ -21,3 +21,8 @@ scrape_configs:
|
||||
- job_name: valkey
|
||||
static_configs:
|
||||
- targets: ["valkey-exporter:9121"]
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
- targets: ["s3:9000"]
|
||||
|
||||
+3
-1
@@ -20,12 +20,13 @@ http {
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||
include /etc/nginx/conf.d/upstream.conf;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 11m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
@@ -34,6 +35,7 @@ http {
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_request_buffering off;
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 10s;
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||
+3
-2
@@ -2,7 +2,8 @@
|
||||
|
||||
set -eu
|
||||
|
||||
docker compose stop nginx api valkey postgres
|
||||
docker compose start postgres valkey
|
||||
docker compose stop nginx api valkey postgres s3
|
||||
docker compose start postgres valkey s3
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose run --rm migrate
|
||||
docker compose up -d --wait --wait-timeout 90
|
||||
|
||||
+194
-28
@@ -2,56 +2,222 @@
|
||||
|
||||
set -eu
|
||||
|
||||
wait_for_replicas() {
|
||||
expected=$1
|
||||
deadline=$(($(date +%s) + 90))
|
||||
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
||||
printf 'usage: %s <image-manifest>\n' "$0" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
manifest=$1
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"}
|
||||
current_manifest=$deploy_dir/current-images.env
|
||||
active_containers=$deploy_dir/active-api-containers
|
||||
previous_containers=$deploy_dir/previous-api-containers
|
||||
upstream_file=$deploy_dir/upstream.conf
|
||||
upstream_backup=$deploy_dir/upstream.previous.conf
|
||||
candidate_file=$deploy_dir/candidate-api-containers
|
||||
replicas=${API_REPLICAS:-2}
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4}
|
||||
switched=false
|
||||
nginx_created=false
|
||||
upstream_changed=false
|
||||
old_pool=""
|
||||
|
||||
mkdir -p "$deploy_dir"
|
||||
: > "$candidate_file"
|
||||
export NGINX_UPSTREAM_FILE=$upstream_file
|
||||
|
||||
load_manifest() {
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$1"
|
||||
set +a
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@"
|
||||
}
|
||||
|
||||
container_name() {
|
||||
name=$(docker inspect --format '{{.Name}}' "$1")
|
||||
printf '%s\n' "${name#/}"
|
||||
}
|
||||
|
||||
running_container() {
|
||||
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
|
||||
}
|
||||
|
||||
write_upstream() {
|
||||
output=$1
|
||||
containers=$2
|
||||
: > "$output"
|
||||
for container in $containers; do
|
||||
name=$(container_name "$container")
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_containers() {
|
||||
containers=$1
|
||||
deadline=$(($(date +%s) + 120))
|
||||
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
containers=$(docker compose ps -q api)
|
||||
count=0
|
||||
healthy=0
|
||||
for container in $containers; do
|
||||
count=$((count + 1))
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
if [ "$status" = healthy ]; then
|
||||
healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
|
||||
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
|
||||
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
old_containers=$(docker compose ps -q api)
|
||||
old_count=0
|
||||
for container in $old_containers; do
|
||||
old_count=$((old_count + 1))
|
||||
done
|
||||
smoke_check() {
|
||||
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||
}
|
||||
|
||||
docker compose build
|
||||
docker compose up -d postgres valkey
|
||||
docker compose run --rm migrate
|
||||
remove_containers() {
|
||||
containers=$1
|
||||
for container in $containers; do
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$old_count" -lt 2 ]; then
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 120
|
||||
exit 0
|
||||
restore_upstream() {
|
||||
if [ -s "$upstream_backup" ]; then
|
||||
cp "$upstream_backup" "$upstream_file"
|
||||
compose exec -T nginx nginx -t
|
||||
compose exec -T nginx nginx -s reload
|
||||
fi
|
||||
}
|
||||
|
||||
rollback() {
|
||||
status=$?
|
||||
trap - EXIT INT TERM
|
||||
candidates=$(cat "$candidate_file" 2>/dev/null || true)
|
||||
|
||||
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
|
||||
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
|
||||
restore_upstream || true
|
||||
if [ "$switched" = true ]; then
|
||||
smoke_check || true
|
||||
fi
|
||||
elif [ "$nginx_created" = true ]; then
|
||||
compose stop nginx >/dev/null 2>&1 || true
|
||||
fi
|
||||
remove_containers "$candidates"
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap rollback EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
load_manifest "$manifest"
|
||||
new_api_image=$API_IMAGE
|
||||
|
||||
if [ -f "$previous_containers" ]; then
|
||||
remove_containers "$(cat "$previous_containers")"
|
||||
rm -f "$previous_containers"
|
||||
fi
|
||||
|
||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
||||
wait_for_replicas 3
|
||||
if [ -s "$active_containers" ]; then
|
||||
for container in $(cat "$active_containers"); do
|
||||
if running_container "$container"; then
|
||||
old_pool="$old_pool $container"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$old_pool" ]; then
|
||||
old_pool=$(compose ps -q api)
|
||||
fi
|
||||
|
||||
for container in $old_containers; do
|
||||
docker stop --time 6 "$container" >/dev/null
|
||||
docker rm "$container" >/dev/null
|
||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
||||
wait_for_replicas 3
|
||||
if [ -s "$upstream_file" ]; then
|
||||
:
|
||||
elif [ -n "$old_pool" ]; then
|
||||
write_upstream "$upstream_file" "$old_pool"
|
||||
else
|
||||
cp nginx/upstream.conf "$upstream_file"
|
||||
fi
|
||||
cp "$upstream_file" "$upstream_backup"
|
||||
|
||||
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
|
||||
compose run --rm --no-deps s3-bootstrap
|
||||
compose run --rm --no-deps migrate
|
||||
|
||||
nginx_container=$(compose ps -q nginx)
|
||||
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
|
||||
compose up -d --no-build --no-deps --force-recreate nginx
|
||||
nginx_container=$(compose ps -q nginx)
|
||||
fi
|
||||
if [ -n "$nginx_container" ]; then
|
||||
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
|
||||
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
|
||||
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
API_IMAGE=$new_api_image
|
||||
export API_IMAGE
|
||||
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
|
||||
if [ -z "$short_digest" ]; then
|
||||
short_digest=$(date +%s)
|
||||
fi
|
||||
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
|
||||
|
||||
candidate_pool=""
|
||||
index=1
|
||||
while [ "$index" -le "$replicas" ]; do
|
||||
name="${project}-api-${short_digest}-${generation}-${index}"
|
||||
container=$(compose run -d --no-deps --name "$name" api)
|
||||
candidate_pool="$candidate_pool $container"
|
||||
printf '%s\n' "$container" >> "$candidate_file"
|
||||
index=$((index + 1))
|
||||
done
|
||||
wait_for_containers "$candidate_pool"
|
||||
|
||||
docker compose up -d --no-deps --no-recreate --scale api=2 api
|
||||
wait_for_replicas 2
|
||||
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
|
||||
write_upstream "$upstream_file.next" "$candidate_pool"
|
||||
cp "$upstream_file.next" "$upstream_file"
|
||||
upstream_changed=true
|
||||
|
||||
if [ -n "$(compose ps -q nginx)" ]; then
|
||||
compose exec -T nginx nginx -t
|
||||
switched=true
|
||||
compose exec -T nginx nginx -s reload
|
||||
else
|
||||
nginx_created=true
|
||||
compose up -d --no-build --no-deps nginx
|
||||
switched=true
|
||||
fi
|
||||
smoke_check
|
||||
|
||||
application_network="${project}_application"
|
||||
for container in $candidate_pool; do
|
||||
docker network disconnect "$application_network" "$container"
|
||||
docker network connect --alias api "$application_network" "$container"
|
||||
done
|
||||
smoke_check
|
||||
|
||||
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
|
||||
|
||||
printf '%s\n' "$candidate_pool" > "$active_containers"
|
||||
cp "$manifest" "$current_manifest"
|
||||
rm -f "$candidate_file" "$upstream_file.next"
|
||||
trap - EXIT INT TERM
|
||||
|
||||
if [ -n "$old_pool" ]; then
|
||||
for container in $old_pool; do
|
||||
docker stop --time 6 "$container" >/dev/null || true
|
||||
done
|
||||
printf '%s\n' "$old_pool" > "$previous_containers"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user