forked from mtusi2026/mtusi2026lab4
Compare commits
13
Commits
baseline/lab3
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e1117eb60 | ||
|
|
13151e66aa | ||
|
|
99cc86766f | ||
|
|
56e4a03bb6 | ||
|
|
37d93c340e | ||
|
|
3fde1c324c | ||
|
|
f8c31de7c0 | ||
|
|
cc4ac1ddd7 | ||
|
|
ad36ab8a92 | ||
|
|
110236d01f | ||
|
|
b3d5d76302 | ||
|
|
ec0e6707b8 | ||
|
|
7e60f099d3 |
@@ -0,0 +1,93 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.repository }}-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
verify:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 45
|
||||||
|
env:
|
||||||
|
COMPOSE_PROJECT_NAME: mtusi2026lab4-${{ github.run_id }}
|
||||||
|
API_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
steps:
|
||||||
|
- name: Checkout exact commit
|
||||||
|
run: |
|
||||||
|
git init .
|
||||||
|
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||||
|
git fetch --depth=1 origin ${{ github.sha }}
|
||||||
|
git checkout --detach FETCH_HEAD
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
cache: false
|
||||||
|
|
||||||
|
- name: Check contract
|
||||||
|
run: make check
|
||||||
|
|
||||||
|
- name: Validate OpenAPI
|
||||||
|
run: |
|
||||||
|
python3 -m venv .venv
|
||||||
|
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
|
make validate PYTHON=.venv/bin/python
|
||||||
|
|
||||||
|
- name: Check formatting and shell scripts
|
||||||
|
run: |
|
||||||
|
test -z "$(gofmt -l cmd internal migrations)"
|
||||||
|
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||||
|
|
||||||
|
- name: Lint
|
||||||
|
uses: golangci/golangci-lint-action@v7
|
||||||
|
with:
|
||||||
|
version: v2.13.2
|
||||||
|
args: --timeout=5m
|
||||||
|
skip-cache: true
|
||||||
|
|
||||||
|
- name: Unit tests and build
|
||||||
|
run: |
|
||||||
|
go test -race ./...
|
||||||
|
go build ./cmd/...
|
||||||
|
|
||||||
|
- name: Prepare local stack
|
||||||
|
run: |
|
||||||
|
cp lab4.env.example .env
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||||
|
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||||
|
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||||
|
|
||||||
|
- name: Integration checks
|
||||||
|
run: |
|
||||||
|
set -a
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||||
|
make test BASE_URL=http://nginx:8080
|
||||||
|
docker compose --profile tests run --rm --build tests
|
||||||
|
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||||
|
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||||
|
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||||
|
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||||
|
|
||||||
|
- name: Container logs
|
||||||
|
if: failure()
|
||||||
|
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||||
|
|
||||||
|
- name: Stop local stack
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||||
|
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||||
|
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
name: Deploy production
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: production
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
verify:
|
||||||
|
name: Verify selected commit
|
||||||
|
if: github.ref == 'refs/heads/main'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 45
|
||||||
|
outputs:
|
||||||
|
api_image: ${{ steps.publish.outputs.api_image }}
|
||||||
|
nginx_image: ${{ steps.publish.outputs.nginx_image }}
|
||||||
|
swagger_image: ${{ steps.publish.outputs.swagger_image }}
|
||||||
|
prometheus_image: ${{ steps.publish.outputs.prometheus_image }}
|
||||||
|
grafana_image: ${{ steps.publish.outputs.grafana_image }}
|
||||||
|
s3_image: ${{ steps.publish.outputs.s3_image }}
|
||||||
|
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
|
||||||
|
env:
|
||||||
|
COMPOSE_PROJECT_NAME: mtusi2026lab4-deploy-${{ github.run_id }}
|
||||||
|
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
|
||||||
|
API_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||||
|
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-api:${{ github.sha }}
|
||||||
|
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-nginx:${{ github.sha }}
|
||||||
|
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-swagger:${{ github.sha }}
|
||||||
|
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-prometheus:${{ github.sha }}
|
||||||
|
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-grafana:${{ github.sha }}
|
||||||
|
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio:${{ github.sha }}
|
||||||
|
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab4-minio-client:${{ github.sha }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout exact commit
|
||||||
|
run: |
|
||||||
|
git init .
|
||||||
|
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||||
|
git fetch --depth=1 origin ${{ github.sha }}
|
||||||
|
git checkout --detach FETCH_HEAD
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
cache: false
|
||||||
|
|
||||||
|
- name: Check contract
|
||||||
|
run: make check
|
||||||
|
|
||||||
|
- name: Validate OpenAPI
|
||||||
|
run: |
|
||||||
|
python3 -m venv .venv
|
||||||
|
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
|
make validate PYTHON=.venv/bin/python
|
||||||
|
|
||||||
|
- name: Check formatting and shell scripts
|
||||||
|
run: |
|
||||||
|
test -z "$(gofmt -l cmd internal migrations)"
|
||||||
|
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||||
|
|
||||||
|
- name: Lint
|
||||||
|
uses: golangci/golangci-lint-action@v7
|
||||||
|
with:
|
||||||
|
version: v2.13.2
|
||||||
|
args: --timeout=5m
|
||||||
|
skip-cache: true
|
||||||
|
|
||||||
|
- name: Unit tests and build
|
||||||
|
run: |
|
||||||
|
go test -race ./...
|
||||||
|
go build ./cmd/...
|
||||||
|
|
||||||
|
- name: Prepare local stack
|
||||||
|
run: |
|
||||||
|
cp lab4.env.example .env
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||||
|
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||||
|
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||||
|
|
||||||
|
- name: Integration checks
|
||||||
|
run: |
|
||||||
|
set -a
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||||
|
make test BASE_URL=http://nginx:8080
|
||||||
|
docker compose --profile tests run --rm --build tests
|
||||||
|
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||||
|
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||||
|
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||||
|
|
||||||
|
- name: Publish verified images
|
||||||
|
id: publish
|
||||||
|
env:
|
||||||
|
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||||
|
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
|
run: |
|
||||||
|
printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin
|
||||||
|
|
||||||
|
publish() {
|
||||||
|
output=$1
|
||||||
|
image=$2
|
||||||
|
push_output=$(docker push "$image")
|
||||||
|
printf '%s\n' "$push_output"
|
||||||
|
digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1)
|
||||||
|
test -n "$digest"
|
||||||
|
repository=${image%:*}
|
||||||
|
printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT"
|
||||||
|
}
|
||||||
|
|
||||||
|
publish api_image "$API_IMAGE"
|
||||||
|
publish nginx_image "$NGINX_IMAGE"
|
||||||
|
publish swagger_image "$SWAGGER_IMAGE"
|
||||||
|
publish prometheus_image "$PROMETHEUS_IMAGE"
|
||||||
|
publish grafana_image "$GRAFANA_IMAGE"
|
||||||
|
publish s3_image "$S3_IMAGE"
|
||||||
|
publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE"
|
||||||
|
|
||||||
|
- name: Container logs
|
||||||
|
if: failure()
|
||||||
|
run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi
|
||||||
|
|
||||||
|
- name: Stop local stack
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||||
|
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||||
|
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
name: Deploy selected commit
|
||||||
|
needs: verify
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: production
|
||||||
|
env:
|
||||||
|
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||||
|
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||||||
|
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||||
|
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||||
|
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||||
|
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
|
DEPLOY_SHA: ${{ github.sha }}
|
||||||
|
steps:
|
||||||
|
- name: Configure SSH
|
||||||
|
run: |
|
||||||
|
umask 077
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
|
||||||
|
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
|
||||||
|
- name: Prepare immutable image manifest
|
||||||
|
run: |
|
||||||
|
cat > deploy-images.env <<EOF
|
||||||
|
API_IMAGE=${{ needs.verify.outputs.api_image }}
|
||||||
|
NGINX_IMAGE=${{ needs.verify.outputs.nginx_image }}
|
||||||
|
SWAGGER_IMAGE=${{ needs.verify.outputs.swagger_image }}
|
||||||
|
PROMETHEUS_IMAGE=${{ needs.verify.outputs.prometheus_image }}
|
||||||
|
GRAFANA_IMAGE=${{ needs.verify.outputs.grafana_image }}
|
||||||
|
S3_IMAGE=${{ needs.verify.outputs.s3_image }}
|
||||||
|
S3_BOOTSTRAP_IMAGE=${{ needs.verify.outputs.s3_bootstrap_image }}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Authenticate registry on deployment host
|
||||||
|
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
|
||||||
|
|
||||||
|
- name: Upload image manifest
|
||||||
|
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab4-${{ github.run_id }}.env"
|
||||||
|
|
||||||
|
- name: Deploy and verify
|
||||||
|
run: |
|
||||||
|
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
|
||||||
|
set -eu
|
||||||
|
cd ~/mtusi2026lab4
|
||||||
|
git fetch origin main
|
||||||
|
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||||
|
git reset --hard '$DEPLOY_SHA'
|
||||||
|
manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env'
|
||||||
|
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
|
||||||
|
rm -f \"\$manifest\"
|
||||||
|
else
|
||||||
|
status=\$?
|
||||||
|
rm -f \"\$manifest\"
|
||||||
|
exit \"\$status\"
|
||||||
|
fi
|
||||||
|
test \"\$(git rev-parse HEAD)\" = '$DEPLOY_SHA'
|
||||||
|
docker compose ps
|
||||||
|
"
|
||||||
+7
-1
@@ -1,8 +1,12 @@
|
|||||||
FROM golang:1.26.5-alpine3.24 AS build
|
FROM golang:1.26.5-alpine3.24 AS build
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN go build -o /out/messenger ./cmd/messenger
|
RUN go build -p=1 -o /out/messenger ./cmd/messenger
|
||||||
|
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
|
||||||
|
|
||||||
FROM build AS test
|
FROM build AS test
|
||||||
|
|
||||||
@@ -25,11 +29,13 @@ COPY monitoring/grafana/dashboards /etc/grafana/dashboards
|
|||||||
FROM nginx:1.30.5-alpine AS loadbalancer
|
FROM nginx:1.30.5-alpine AS loadbalancer
|
||||||
|
|
||||||
COPY nginx/nginx.conf /etc/nginx/nginx.conf
|
COPY nginx/nginx.conf /etc/nginx/nginx.conf
|
||||||
|
COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf
|
||||||
|
|
||||||
FROM alpine:3.24.2
|
FROM alpine:3.24.2
|
||||||
|
|
||||||
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
||||||
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
||||||
|
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
|
||||||
|
|
||||||
USER app
|
USER app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|||||||
@@ -1,37 +1,181 @@
|
|||||||
# Отчёт — лабораторная 4
|
# Отчёт — лабораторная работа № 4
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор: Пластинин Денис, БВТ2303.
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: 5.
|
||||||
- Commit/PR: TODO после публикации
|
- Выбранный трек: 2, согласованность PostgreSQL и S3.
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Ветка: `solution/bvt2303-plastinin`.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Требуются Docker Engine с Compose V2, Git и свободные локальные порты 8080, 8081, 9090 и 3000. Для запуска Go-проверок вне контейнера нужен Go 1.26. Образы приложения и MinIO собираются из исходников внутри Docker. Версии Go, PostgreSQL, Valkey, Nginx, MinIO и остальных компонентов закреплены в Dockerfile и `compose.yaml`.
|
||||||
|
|
||||||
## Архитектура и решения
|
```sh
|
||||||
|
cp lab4.env.example .env
|
||||||
|
docker compose --env-file .env config --quiet
|
||||||
|
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
|
||||||
|
```
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
После запуска доступны:
|
||||||
|
|
||||||
|
- API через Nginx: `http://localhost:8080`;
|
||||||
|
- Swagger UI: `http://localhost:8081`;
|
||||||
|
- Prometheus: `http://localhost:9090`;
|
||||||
|
- Grafana: `http://localhost:3000`, пользователь `admin`, пароль из `GRAFANA_ADMIN_PASSWORD`.
|
||||||
|
|
||||||
|
Публичные и внутренние проверки:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make check
|
||||||
|
make test BASE_URL=http://localhost:8080
|
||||||
|
docker compose --profile tests run --rm --build tests
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Сверка PostgreSQL и S3 выполняется внутри запущенной API-реплики. Без `--apply` команда ничего не удаляет:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose exec -T api /app/messenger reconcile-attachments --grace-period=1h
|
||||||
|
docker compose exec -T api /app/messenger reconcile-attachments --apply --grace-period=1h
|
||||||
|
```
|
||||||
|
|
||||||
|
Эксперименты запускаются на чистом стенде:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose exec -T api /app/lab4experiment memory
|
||||||
|
docker compose exec -T api /app/lab4experiment consistency
|
||||||
|
```
|
||||||
|
|
||||||
|
Безопасная остановка сохраняет данные в Docker volumes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
Полное удаление учебных данных выполняется только явно:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down -v
|
||||||
|
```
|
||||||
|
|
||||||
|
## Архитектура
|
||||||
|
|
||||||
|
```text
|
||||||
|
клиент
|
||||||
|
|
|
||||||
|
v
|
||||||
|
Nginx :8080
|
||||||
|
|
|
||||||
|
+---- API replica 1 ----+
|
||||||
|
| |
|
||||||
|
+---- API replica 2 ----+---- PostgreSQL: пользователи, чаты,
|
||||||
|
| сообщения, metadata
|
||||||
|
+---- Valkey: сессии
|
||||||
|
+---- MinIO: приватные байты изображений
|
||||||
|
|
||||||
|
Prometheus <---- API /metrics, MinIO, Nginx, PostgreSQL, Valkey
|
||||||
|
|
|
||||||
|
+---- Grafana
|
||||||
|
```
|
||||||
|
|
||||||
|
Из лабораторной № 3 сохранены две и более API-реплики, Nginx, общие PostgreSQL и Valkey, Bearer-сессии, keyset pagination, идемпотентность сообщений, graceful shutdown, failover, метрики и управляемое развёртывание. В лабораторной № 4 добавлены приватное объектное хранилище, модель вложения, бинарная загрузка и скачивание, привязка вложений к сообщениям и сверка двух хранилищ.
|
||||||
|
|
||||||
|
Код разделён по ответственности. `internal/messenger/httpapi` содержит HTTP-маршруты, middleware и преобразование ошибок в ответы. `internal/messenger/attachment` владеет сценарием загрузки и чтения вложений. `internal/postgres`, `internal/objectstore` и `internal/session` являются адаптерами PostgreSQL, S3 и Valkey. `internal/reconcile` выполняет сверку хранилищ, а `internal/bootstrap` собирает зависимости для сервера, миграций и служебной команды. HTTP-слой зависит от узких интерфейсов и общих ошибок модели, поэтому детали PostgreSQL и S3 в handlers не попадают.
|
||||||
|
|
||||||
|
PostgreSQL остаётся источником истины для прав доступа и метаданных. Таблица `attachments` хранит владельца, чат, отображаемое имя, MIME, размер, SHA-256 и внутренний object key. Таблица `message_attachments` задаёт упорядоченную связь многие ко многим с сообщениями. Object key генерируется сервером и не зависит от имени файла.
|
||||||
|
|
||||||
|
MinIO хранит только байты оригиналов. Bucket закрыт для анонимного доступа. Bootstrap повторяемо создаёт bucket, policy и отдельного пользователя приложения. API получает только действия list/get/put/delete в одном bucket. Root credentials используются bootstrap-контейнером и не передаются API. Web-консоль MinIO отключена и её порт не опубликован.
|
||||||
|
|
||||||
|
## Загрузка и чтение вложения
|
||||||
|
|
||||||
|
Клиент передаёт сырые байты PNG или JPEG. Сервер ограничивает HTTP body до 10 MiB до полного чтения, проверяет заявленный и фактический MIME, декодирует конфигурацию изображения, ограничивает число пикселей 40 миллионами и проверяет полную декодируемость. Filename содержит от 1 до 255 Unicode-символов и не может содержать разделители пути, NUL, управляющие символы или состоять только из пробелов.
|
||||||
|
|
||||||
|
При успешной загрузке сервер генерирует UUID для вложения и отдельный случайный object key, считает SHA-256 и создаёт в PostgreSQL строку со статусом `pending`. Затем он записывает объект в MinIO и переводит строку в `ready`. Только после этого возвращается `201`. При отказе S3 строка `pending` удаляется. Если результат перевода в `ready` неоднозначен, сервер повторно читает строку по attachment ID и object key: уже сохранённый `ready` считается успехом, а подтверждённый `pending` компенсируется удалением объекта и metadata.
|
||||||
|
|
||||||
|
Метаданные и содержимое читаются только после проверки Bearer-сессии и членства пользователя в чате. Посторонний пользователь и неизвестный идентификатор дают одинаковый `404`. API отдаёт байты самостоятельно без redirect и возвращает сохранённый `Content-Type` и точный `Content-Length`.
|
||||||
|
|
||||||
|
Сообщение принимает до десяти уникальных `attachment_ids`. В одной транзакции PostgreSQL проверяются членство отправителя и принадлежность каждого вложения нужному чату, создаются сообщение, связи и запись идемпотентности. Отсутствующее поле `attachment_ids` и пустой массив дают одинаковый семантический hash. Пустой текст допустим только при наличии вложения.
|
||||||
|
|
||||||
|
## Согласованность PostgreSQL и S3
|
||||||
|
|
||||||
|
Операции PostgreSQL и S3 не входят в общую распределённую транзакцию. Возможны два вида расхождений:
|
||||||
|
|
||||||
|
1. object существует в S3, но metadata отсутствует;
|
||||||
|
2. metadata существует, но object отсутствует.
|
||||||
|
|
||||||
|
Команда `reconcile-attachments` проходит оба набора в порядке object key постранично, поэтому объём памяти ограничен размером двух страниц. По умолчанию включён dry-run. Grace period защищает свежие объекты и строки, которые могут относиться к ещё выполняющейся операции.
|
||||||
|
|
||||||
|
В режиме `--apply` старый orphan в S3 удаляется после повторной проверки object key в PostgreSQL. Старые `pending` и готовые, но неиспользуемые вложения сначала атомарно переводятся в `deleting` под блокировкой строки. Под той же блокировкой повторно проверяются ожидаемый статус, возраст и отсутствие ссылки из `message_attachments`. После удаления объекта строка удаляется только в статусе `deleting`. Если upload успел изменить `pending` на `ready`, устаревший кандидат не захватывается. Используемая строка с отсутствующим объектом сохраняется и попадает в отчёт для ручного восстановления. Повторный запуск продолжает незавершённый `deleting` и не повторяет уже выполненные удаления.
|
||||||
|
|
||||||
|
## Ограничение памяти
|
||||||
|
|
||||||
|
До чтения body weighted semaphore резервирует полный лимит одной загрузки — 10 MiB. Поэтому отсутствующий или заниженный `Content-Length` не позволяет одновременно накопить неограниченное число буферов. После чтения дополнительно резервируется восемь байт на пиксель перед полной декодировкой. Бюджеты одной реплики разделены: 64 MiB для исходных body и 384 MiB для декодирования; лимит контейнера равен 512 MiB. Проверки размеров выполняются до умножения, чтобы исключить integer overflow. Если одного из бюджетов недостаточно, API отвечает `429` с `Retry-After: 1`.
|
||||||
|
|
||||||
|
Go-инструмент `cmd/lab4experiment` создаёт случайное JPEG-изображение и выполняет четыре волны по 12 одновременных загрузок. Передача каждой копии ограничена по скорости, чтобы нагрузка длилась дольше интервала scrape. Наблюдаемый пик heap и resident memory двух API-процессов читается из Prometheus с интервалом одна секунда. Это измерение локального стенда, а не универсальная оценка производительности S3.
|
||||||
|
|
||||||
|
В эксперименте успешно завершились все 48 загрузок файла размером 3 759 914 байт. Четыре волны заняли 12,974 секунды. После трёх секунд стабилизации суммарный heap двух реплик вырос с 153 985 824 до 202 505 208 байт, то есть на 48 519 384 байта. Суммарный RSS вырос с 335 298 560 до 345 976 832 байт, то есть на 10 678 272 байта. Наблюдаемый пик остался ниже общего контейнерного лимита двух реплик 1 GiB.
|
||||||
|
|
||||||
|
## Наблюдаемость
|
||||||
|
|
||||||
|
API экспортирует общие HTTP-метрики и отдельные показатели загрузок:
|
||||||
|
|
||||||
|
- `messenger_attachment_uploads_total{result}`;
|
||||||
|
- `messenger_attachment_upload_duration_seconds`;
|
||||||
|
- `messenger_attachment_upload_bytes_total`.
|
||||||
|
|
||||||
|
Prometheus обнаруживает все API-реплики через Docker DNS и отдельно собирает MinIO, Nginx, PostgreSQL и Valkey. Grafana показывает частоту и p95 загрузок, объём принятых байтов, число и ёмкость объектов MinIO, ошибки S3, доступность компонентов, RPS, классы HTTP-ответов и память Go.
|
||||||
|
|
||||||
|
`/health/live` проверяет только процесс. `/health/ready` требует доступных PostgreSQL, Valkey и приватного bucket S3. Недоступность Grafana или Prometheus не останавливает API.
|
||||||
|
|
||||||
|
## CI/CD и blue/green deploy
|
||||||
|
|
||||||
|
CI проверяет contract, OpenAPI, форматирование, lint, unit и integration tests, persistence и отказ одной API-реплики. После успешных проверок образы публикуются в Gitea Container Registry. В manifest передаются ссылки вида `repository@sha256:digest`, поэтому сервер запускает те же байты, которые прошли проверки.
|
||||||
|
|
||||||
|
Production workflow запускается вручную и использует защищённое environment `production`. Перед переключением применяются только обратно совместимые миграции. Скрипт поднимает новый пул API под отдельными DNS-именами и ждёт readiness каждой реплики. Nginx сначала проверяет новую конфигурацию через `nginx -t`, затем атомарно выполняет reload upstream. Старый пул остаётся запущенным до успешного smoke-check нового и только после этого останавливается. При ошибке до завершения переключения восстанавливается предыдущий upstream, а candidate-контейнеры удаляются.
|
||||||
|
|
||||||
|
Сам балансировщик относится к стабильному control plane. Если его verified digest изменился, обычный API deploy завершается до переключения и требует отдельного обслуживания Nginx. Это не подменяет новый образ старым и не создаёт скрытое окно недоступности внутри blue/green обновления API.
|
||||||
|
|
||||||
|
Локальная проверка механизма подняла две отдельные candidate-реплики, переключила upstream и выполнила 20 запросов `/health/live`: все ответы имели статус `200`, а `X-Instance-Id` чередовался только между двумя контейнерами нового пула.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий | Команда или сценарий | Результат | Доказательство |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Контракт и OpenAPI | `make check`, `make validate` | offline check PASS; validate входит в CI | `evidence/lab4-verification.txt` |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Unit и integration | `go test ./...`, race/tests profile в CI | unit PASS | `evidence/lab4-verification.txt` |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| Upload/download и ACL | `make test` | PASS | `evidence/lab4-verification.txt` |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| Межрепличный failover и restart | public resilience hooks | проверяются CI точного commit | `.gitea/workflows/ci.yml` |
|
||||||
|
| Приватный bucket и повторный bootstrap | anonymous HTTP check, два запуска bootstrap | PASS | `evidence/lab4-verification.txt` |
|
||||||
|
| 12 конкурентных загрузок и память | `lab4experiment memory` | 48/48 PASS, observed RSS peak 345 976 832 B | `evidence/lab4-memory.json` |
|
||||||
|
| Трек 2: dry-run/apply/repeat | `lab4experiment consistency` | PASS, повторный apply: 0 удалений | `evidence/lab4-consistency.json` |
|
||||||
|
| Blue/green API switch | два candidate-контейнера, `nginx -t`, reload, 20 health-запросов | 20/20 HTTP 200, только новый пул | `evidence/lab4-verification.txt` |
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
## Эксперимент согласованности
|
||||||
|
|
||||||
## Эксперименты и отказы
|
Гипотеза: постраничная сверка с grace period может автоматически удалить старые безопасные расхождения, не удаляя свежие данные и metadata используемого вложения; второй apply будет идемпотентным.
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Методика на чистом стенде:
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
1. создать согласованное вложение;
|
||||||
| --- | --- | --- | --- |
|
2. удалить S3 object у непривязанного вложения;
|
||||||
| TODO | TODO | TODO | TODO |
|
3. привязать другое вложение к сообщению и удалить его S3 object;
|
||||||
|
4. создать object без metadata;
|
||||||
|
5. создать незавершённое вложение `pending`;
|
||||||
|
6. выполнить apply с grace period один час и проверить сохранность свежих аномалий;
|
||||||
|
7. состарить `pending`, выполнить dry-run с нулевым grace period;
|
||||||
|
8. выполнить apply и проверить удаление orphan, непривязанной строки и старого `pending`;
|
||||||
|
9. повторить apply и проверить отсутствие новых действий.
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
Фактическая стоимость обхода записывается как число просмотренных объектов и строк, размер страницы и wall time. Алгоритм выполняет линейный merge двух отсортированных потоков; дополнительные запросы проверки ссылок нужны только для metadata с отсутствующим объектом.
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
При размере страницы 10 фактический запуск пересёк границу страницы. Apply с часовым grace period просмотрел 14 строк metadata и 13 objects за 4 миллисекунды, сохранил свежие orphan, missing metadata и `pending`. После искусственного старения dry-run за 5 миллисекунд обнаружил один старый orphan, одну непривязанную строку без объекта, одну используемую строку без объекта и один старый `pending`. Apply за 13 миллисекунд удалил object и metadata старого `pending`, orphan object и непривязанную missing metadata. Повторный apply занял 1 миллисекунду и выполнил ноль удалений; используемая строка осталась в PostgreSQL.
|
||||||
|
|
||||||
|
## Ограничения
|
||||||
|
|
||||||
|
- Между PostgreSQL и S3 нет общей транзакции; lifecycle `pending/ready/deleting`, компенсация и reconciler обеспечивают eventual consistency.
|
||||||
|
- Удалённый object используемого вложения автоматически не восстанавливается: строка сохраняется и требует восстановления байтов из резервной копии либо решения оператора.
|
||||||
|
- Grace period основан на времени PostgreSQL и S3; для удалённого стенда требуется синхронизация часов.
|
||||||
|
- Admission control действует отдельно в каждой API-реплике; при изменении контейнерного лимита нужно совместно настраивать `ATTACHMENT_BODY_MEMORY_BYTES` и `ATTACHMENT_DECODE_MEMORY_BYTES`.
|
||||||
|
- Автоматический rollback переключает API обратно на предыдущий пул, но не откатывает уже применённые миграции и stateful-хранилища. Поэтому миграции должны оставаться обратно совместимыми как минимум с предыдущей версией приложения.
|
||||||
|
- MinIO собирается из официального закреплённого source tag, поскольку готовые community images больше не публикуются. Это увеличивает время первого CI build.
|
||||||
|
- Локальный HTTP допустим только для учебного стенда с синтетическими данными; удалённое развёртывание требует TLS.
|
||||||
|
|||||||
@@ -0,0 +1,676 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"image"
|
||||||
|
"image/jpeg"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||||
|
)
|
||||||
|
|
||||||
|
const experimentPassword = "Lab4-experiment-password"
|
||||||
|
|
||||||
|
type apiClient struct {
|
||||||
|
baseURL string
|
||||||
|
client *http.Client
|
||||||
|
token string
|
||||||
|
}
|
||||||
|
|
||||||
|
type attachment struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type setup struct {
|
||||||
|
client *apiClient
|
||||||
|
chatID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(arguments []string, output io.Writer) error {
|
||||||
|
if len(arguments) == 0 {
|
||||||
|
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||||
|
}
|
||||||
|
switch arguments[0] {
|
||||||
|
case "memory":
|
||||||
|
return runMemory(arguments[1:], output)
|
||||||
|
case "consistency":
|
||||||
|
return runConsistency(arguments[1:], output)
|
||||||
|
default:
|
||||||
|
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type memoryResult struct {
|
||||||
|
Replicas int `json:"replicas"`
|
||||||
|
Concurrency int `json:"concurrency"`
|
||||||
|
Rounds int `json:"rounds"`
|
||||||
|
SettleTimeMillis int64 `json:"settle_time_ms"`
|
||||||
|
ImageBytes int `json:"image_bytes"`
|
||||||
|
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
|
||||||
|
SuccessfulUploads int64 `json:"successful_uploads"`
|
||||||
|
FailedUploads int64 `json:"failed_uploads"`
|
||||||
|
DurationMillis int64 `json:"duration_ms"`
|
||||||
|
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
|
||||||
|
HeapPeakBytes int64 `json:"heap_peak_bytes"`
|
||||||
|
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
|
||||||
|
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
|
||||||
|
RSSPeakBytes int64 `json:"rss_peak_bytes"`
|
||||||
|
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func runMemory(arguments []string, output io.Writer) error {
|
||||||
|
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(io.Discard)
|
||||||
|
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||||
|
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
|
||||||
|
replicas := flags.Int("replicas", 2, "expected API replicas")
|
||||||
|
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
|
||||||
|
rounds := flags.Int("rounds", 4, "number of upload waves")
|
||||||
|
width := flags.Int("width", 2048, "image width")
|
||||||
|
height := flags.Int("height", 2048, "image height")
|
||||||
|
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
|
||||||
|
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
|
||||||
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||||
|
return errors.New("invalid memory experiment flags")
|
||||||
|
}
|
||||||
|
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
|
||||||
|
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
fixture, err := jpegFixture(*width, *height)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
settle := time.NewTimer(*settleTime)
|
||||||
|
defer settle.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-settle.C:
|
||||||
|
}
|
||||||
|
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
environment, err := prepare(ctx, *baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var successful atomic.Int64
|
||||||
|
var failed atomic.Int64
|
||||||
|
var firstError struct {
|
||||||
|
sync.Mutex
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
heapPeak := heapBaseline
|
||||||
|
rssPeak := rssBaseline
|
||||||
|
loadDone := make(chan struct{})
|
||||||
|
sampleDone := make(chan error, 1)
|
||||||
|
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
|
||||||
|
started := time.Now()
|
||||||
|
for round := range *rounds {
|
||||||
|
var workers sync.WaitGroup
|
||||||
|
workers.Add(*concurrency)
|
||||||
|
for index := range *concurrency {
|
||||||
|
go func(round, index int) {
|
||||||
|
defer workers.Done()
|
||||||
|
reader := newRateReader(fixture, *uploadRate)
|
||||||
|
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
|
||||||
|
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
|
||||||
|
failed.Add(1)
|
||||||
|
firstError.Lock()
|
||||||
|
if firstError.err == nil {
|
||||||
|
firstError.err = err
|
||||||
|
}
|
||||||
|
firstError.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
successful.Add(1)
|
||||||
|
}(round, index)
|
||||||
|
}
|
||||||
|
workers.Wait()
|
||||||
|
}
|
||||||
|
close(loadDone)
|
||||||
|
if err := <-sampleDone; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result := memoryResult{
|
||||||
|
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
|
||||||
|
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
|
||||||
|
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
|
||||||
|
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
|
||||||
|
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
|
||||||
|
}
|
||||||
|
if result.FailedUploads > 0 {
|
||||||
|
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
|
||||||
|
}
|
||||||
|
return writeJSON(output, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sampleMemory(
|
||||||
|
ctx context.Context,
|
||||||
|
prometheusURL string,
|
||||||
|
expectedReplicas int,
|
||||||
|
done <-chan struct{},
|
||||||
|
heapPeak *int64,
|
||||||
|
rssPeak *int64,
|
||||||
|
result chan<- error,
|
||||||
|
) {
|
||||||
|
ticker := time.NewTicker(500 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
result <- ctx.Err()
|
||||||
|
return
|
||||||
|
case <-done:
|
||||||
|
result <- nil
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||||
|
if err != nil {
|
||||||
|
result <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if replicas != int64(expectedReplicas) {
|
||||||
|
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||||
|
if err != nil {
|
||||||
|
result <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||||
|
if err != nil {
|
||||||
|
result <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
*heapPeak = max(*heapPeak, heap)
|
||||||
|
*rssPeak = max(*rssPeak, rss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type consistencyResult struct {
|
||||||
|
PageSize int `json:"page_size"`
|
||||||
|
SeededConsistent int `json:"seeded_consistent"`
|
||||||
|
FreshApply reconciliationRun `json:"fresh_apply"`
|
||||||
|
DryRun reconciliationRun `json:"dry_run"`
|
||||||
|
Apply reconciliationRun `json:"apply"`
|
||||||
|
RepeatedApply reconciliationRun `json:"repeated_apply"`
|
||||||
|
FreshPreserved bool `json:"fresh_preserved"`
|
||||||
|
OrphanDeleted bool `json:"orphan_deleted"`
|
||||||
|
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
||||||
|
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
||||||
|
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
||||||
|
TotalDurationMillis int64 `json:"total_duration_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type reconciliationRun struct {
|
||||||
|
DurationMillis int64 `json:"duration_ms"`
|
||||||
|
Report reconcile.Report `json:"report"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func runConsistency(arguments []string, output io.Writer) error {
|
||||||
|
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(io.Discard)
|
||||||
|
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||||
|
pageSize := flags.Int("page-size", 10, "reconciliation page size")
|
||||||
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||||
|
return errors.New("invalid consistency experiment flags")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
settings, err := config.LoadReconciliation()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pool, err := database.Open(ctx, database.Config{
|
||||||
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||||
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||||
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer pool.Close()
|
||||||
|
objects, err := objectstore.New(objectstore.Config{
|
||||||
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||||
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||||
|
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
metadata := postgres.New(pool)
|
||||||
|
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||||
|
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
initial, err := clean.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if hasAnomalies(initial) {
|
||||||
|
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
|
||||||
|
}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
|
environment, err := prepare(ctx, *baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fixture, err := jpegFixture(320, 200)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
seededConsistent := *pageSize + 1
|
||||||
|
for index := range seededConsistent {
|
||||||
|
filename := fmt.Sprintf("consistent-%03d.jpg", index)
|
||||||
|
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
||||||
|
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
||||||
|
}
|
||||||
|
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
|
||||||
|
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||||
|
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
||||||
|
UnreferencedRetention: time.Hour, Apply: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
freshRun, err := measuredReconciliation(ctx, fresh)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
||||||
|
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
||||||
|
attachmentExists(ctx, pool, abandoned.ID)
|
||||||
|
if _, err := pool.Exec(ctx, `
|
||||||
|
UPDATE attachments
|
||||||
|
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
||||||
|
WHERE id = $1`, abandoned.ID); err != nil {
|
||||||
|
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||||
|
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dryRunResult, err := measuredReconciliation(ctx, dryRun)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||||
|
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
applyResult, err := measuredReconciliation(ctx, apply)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repeatedResult, err := measuredReconciliation(ctx, apply)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
result := consistencyResult{
|
||||||
|
PageSize: *pageSize, SeededConsistent: seededConsistent,
|
||||||
|
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||||
|
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||||
|
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||||
|
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||||
|
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||||
|
}
|
||||||
|
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||||
|
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
||||||
|
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
||||||
|
}
|
||||||
|
return writeJSON(output, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
|
||||||
|
started := time.Now()
|
||||||
|
report, err := reconciler.Run(ctx)
|
||||||
|
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasAnomalies(report reconcile.Report) bool {
|
||||||
|
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||||
|
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||||
|
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||||
|
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||||
|
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("query attachment keys: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
keys := make(map[string]string, len(ids))
|
||||||
|
for rows.Next() {
|
||||||
|
var id string
|
||||||
|
var key string
|
||||||
|
if err := rows.Scan(&id, &key); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan attachment key: %w", err)
|
||||||
|
}
|
||||||
|
keys[id] = key
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("iterate attachment keys: %w", err)
|
||||||
|
}
|
||||||
|
if len(keys) != len(ids) {
|
||||||
|
return nil, errors.New("not all attachment keys were found")
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
|
||||||
|
var exists bool
|
||||||
|
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
|
||||||
|
_, err := objects.Stat(ctx, key)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepare(ctx context.Context, baseURL string) (setup, error) {
|
||||||
|
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
|
||||||
|
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
|
||||||
|
var user struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
|
||||||
|
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
|
||||||
|
}, &user, ""); err != nil {
|
||||||
|
return setup{}, err
|
||||||
|
}
|
||||||
|
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
|
||||||
|
var session struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
}
|
||||||
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
|
||||||
|
return setup{}, err
|
||||||
|
}
|
||||||
|
client.token = session.Token
|
||||||
|
var chat struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
|
||||||
|
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
|
||||||
|
}, &chat, "Bearer "+client.token); err != nil {
|
||||||
|
return setup{}, err
|
||||||
|
}
|
||||||
|
return setup{client: client, chatID: chat.ID}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *apiClient) upload(
|
||||||
|
ctx context.Context,
|
||||||
|
chatID string,
|
||||||
|
filename string,
|
||||||
|
body io.Reader,
|
||||||
|
contentLength int64,
|
||||||
|
) (attachment, error) {
|
||||||
|
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
|
||||||
|
if err != nil {
|
||||||
|
return attachment{}, err
|
||||||
|
}
|
||||||
|
request.ContentLength = contentLength
|
||||||
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||||
|
request.Header.Set("Content-Type", "image/jpeg")
|
||||||
|
response, err := c.client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return attachment{}, err
|
||||||
|
}
|
||||||
|
defer func() { _ = response.Body.Close() }()
|
||||||
|
if response.StatusCode != http.StatusCreated {
|
||||||
|
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||||
|
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
|
||||||
|
}
|
||||||
|
var result attachment
|
||||||
|
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
||||||
|
return attachment{}, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
|
||||||
|
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
|
||||||
|
"text": "", "attachment_ids": []string{attachmentID},
|
||||||
|
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *apiClient) jsonRequest(
|
||||||
|
ctx context.Context,
|
||||||
|
method string,
|
||||||
|
path string,
|
||||||
|
payload any,
|
||||||
|
result any,
|
||||||
|
authorization string,
|
||||||
|
extraHeader ...string,
|
||||||
|
) error {
|
||||||
|
var body io.Reader
|
||||||
|
if payload != nil {
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body = bytes.NewReader(encoded)
|
||||||
|
}
|
||||||
|
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if payload != nil {
|
||||||
|
request.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
if authorization != "" {
|
||||||
|
request.Header.Set("Authorization", authorization)
|
||||||
|
}
|
||||||
|
for index := 0; index+1 < len(extraHeader); index += 2 {
|
||||||
|
request.Header.Set(extraHeader[index], extraHeader[index+1])
|
||||||
|
}
|
||||||
|
response, err := c.client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = response.Body.Close() }()
|
||||||
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||||
|
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||||
|
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return json.NewDecoder(response.Body).Decode(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||||
|
if err == nil && count == int64(expected) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func activeReplicasQuery() string {
|
||||||
|
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
||||||
|
}
|
||||||
|
|
||||||
|
func freshMetricQuery(metric string) string {
|
||||||
|
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
||||||
|
}
|
||||||
|
|
||||||
|
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
values := request.URL.Query()
|
||||||
|
values.Set("query", query)
|
||||||
|
request.URL.RawQuery = values.Encode()
|
||||||
|
response, err := http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer func() { _ = response.Body.Close() }()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||||
|
}
|
||||||
|
var payload struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Data struct {
|
||||||
|
Result []struct {
|
||||||
|
Value []json.RawMessage `json:"value"`
|
||||||
|
} `json:"result"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
|
||||||
|
return 0, errors.New("prometheus query returned no scalar value")
|
||||||
|
}
|
||||||
|
var raw string
|
||||||
|
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
value, err := strconv.ParseFloat(raw, 64)
|
||||||
|
return int64(value), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func jpegFixture(width, height int) ([]byte, error) {
|
||||||
|
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||||
|
state := uint64(0x9e3779b97f4a7c15)
|
||||||
|
for index := range pixels.Pix {
|
||||||
|
state ^= state << 13
|
||||||
|
state ^= state >> 7
|
||||||
|
state ^= state << 17
|
||||||
|
pixels.Pix[index] = byte(state >> 24)
|
||||||
|
}
|
||||||
|
var encoded bytes.Buffer
|
||||||
|
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return encoded.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type rateReader struct {
|
||||||
|
reader *bytes.Reader
|
||||||
|
rate int64
|
||||||
|
started time.Time
|
||||||
|
read int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRateReader(data []byte, rate int64) *rateReader {
|
||||||
|
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *rateReader) Read(buffer []byte) (int, error) {
|
||||||
|
for {
|
||||||
|
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
|
||||||
|
if allowed > 0 {
|
||||||
|
if int64(len(buffer)) > allowed {
|
||||||
|
buffer = buffer[:allowed]
|
||||||
|
}
|
||||||
|
count, err := r.reader.Read(buffer)
|
||||||
|
r.read += int64(count)
|
||||||
|
return count, err
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSON(output io.Writer, value any) error {
|
||||||
|
encoder := json.NewEncoder(output)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
return encoder.Encode(value)
|
||||||
|
}
|
||||||
+8
-109
@@ -1,132 +1,31 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
appLogger := logger.New()
|
appLogger := logger.New()
|
||||||
if err := run(appLogger, os.Args[1:]); err != nil {
|
if err := run(appLogger, os.Args[1:]); err != nil {
|
||||||
appLogger.Error("server stopped", "error", err)
|
appLogger.Error("application stopped", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func run(logger *slog.Logger, arguments []string) error {
|
func run(logger *slog.Logger, arguments []string) error {
|
||||||
if len(arguments) == 0 {
|
if len(arguments) == 0 {
|
||||||
return runServer(logger)
|
return bootstrap.RunServer(logger)
|
||||||
}
|
}
|
||||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||||
return runMigrations()
|
return bootstrap.RunMigrations()
|
||||||
}
|
}
|
||||||
return fmt.Errorf("usage: messenger [migrate]")
|
if arguments[0] == "reconcile-attachments" {
|
||||||
}
|
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
|
||||||
|
|
||||||
func runServer(logger *slog.Logger) error {
|
|
||||||
settings, err := config.Load()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
instanceID, err := os.Hostname()
|
return errors.New("usage: messenger [migrate|reconcile-attachments]")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
pool, err := database.Open(context.Background(), database.Config{
|
|
||||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
||||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
||||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer pool.Close()
|
|
||||||
sessions, err := session.New(settings.CacheURL)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() { _ = sessions.Close() }()
|
|
||||||
passwords, err := password.New(password.Config{
|
|
||||||
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
|
|
||||||
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
|
|
||||||
KeyLength: settings.PasswordKeyLength,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
app := messenger.NewServer(
|
|
||||||
logger,
|
|
||||||
postgres.New(pool),
|
|
||||||
sessions,
|
|
||||||
passwords,
|
|
||||||
settings.SessionTTL,
|
|
||||||
settings.IdempotencyTTL,
|
|
||||||
settings.RequestTimeout,
|
|
||||||
instanceID,
|
|
||||||
settings.SwaggerPort,
|
|
||||||
)
|
|
||||||
server := &http.Server{
|
|
||||||
Addr: settings.Address,
|
|
||||||
Handler: app.Handler(),
|
|
||||||
ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
|
|
||||||
ReadTimeout: settings.HTTPReadTimeout,
|
|
||||||
WriteTimeout: settings.HTTPWriteTimeout,
|
|
||||||
IdleTimeout: settings.HTTPIdleTimeout,
|
|
||||||
}
|
|
||||||
listener, err := net.Listen("tcp", server.Addr)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
serveResult := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
serveResult <- server.Serve(listener)
|
|
||||||
}()
|
|
||||||
logger.Info("server started", "address", server.Addr)
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
defer stop()
|
|
||||||
select {
|
|
||||||
case err := <-serveResult:
|
|
||||||
return err
|
|
||||||
case <-ctx.Done():
|
|
||||||
logger.Info("shutdown requested")
|
|
||||||
app.SetReady(false)
|
|
||||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
|
||||||
_ = server.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runMigrations() error {
|
|
||||||
settings, err := config.LoadMigration()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
pool, err := database.Open(context.Background(), database.Config{
|
|
||||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
||||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
||||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer pool.Close()
|
|
||||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ services:
|
|||||||
cpus: ${POSTGRES_CPUS}
|
cpus: ${POSTGRES_CPUS}
|
||||||
mem_limit: ${POSTGRES_MEMORY}
|
mem_limit: ${POSTGRES_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
valkey:
|
valkey:
|
||||||
image: valkey/valkey:8.1.4-alpine
|
image: valkey/valkey:8.1.4-alpine
|
||||||
@@ -31,6 +32,58 @@ services:
|
|||||||
cpus: ${VALKEY_CPUS}
|
cpus: ${VALKEY_CPUS}
|
||||||
mem_limit: ${VALKEY_MEMORY}
|
mem_limit: ${VALKEY_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
|
s3:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: infra/minio/server.Dockerfile
|
||||||
|
args:
|
||||||
|
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
|
||||||
|
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
|
||||||
|
command: ["server", "/data", "--address", ":9000"]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||||
|
MINIO_BROWSER: "off"
|
||||||
|
MINIO_PROMETHEUS_AUTH_TYPE: public
|
||||||
|
volumes:
|
||||||
|
- s3_data:/data
|
||||||
|
expose:
|
||||||
|
- "9000"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
cpus: ${S3_CPUS}
|
||||||
|
mem_limit: ${S3_MEMORY}
|
||||||
|
restart: unless-stopped
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
|
s3-bootstrap:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: infra/minio/client.Dockerfile
|
||||||
|
args:
|
||||||
|
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||||
|
image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z
|
||||||
|
entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||||
|
S3_BUCKET: ${S3_BUCKET}
|
||||||
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||||
|
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
cpus: ${S3_BOOTSTRAP_CPUS}
|
||||||
|
mem_limit: ${S3_BOOTSTRAP_MEMORY}
|
||||||
|
restart: "no"
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
migrate:
|
migrate:
|
||||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||||
@@ -52,6 +105,7 @@ services:
|
|||||||
cpus: ${MIGRATE_CPUS}
|
cpus: ${MIGRATE_CPUS}
|
||||||
mem_limit: ${MIGRATE_MEMORY}
|
mem_limit: ${MIGRATE_MEMORY}
|
||||||
restart: "no"
|
restart: "no"
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
api:
|
api:
|
||||||
build: .
|
build: .
|
||||||
@@ -82,6 +136,17 @@ services:
|
|||||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||||
|
S3_REGION: ${S3_REGION}
|
||||||
|
S3_BUCKET: ${S3_BUCKET}
|
||||||
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||||
|
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||||
|
S3_USE_TLS: ${S3_USE_TLS}
|
||||||
|
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||||
|
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||||
|
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||||
|
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
|
||||||
|
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
|
||||||
expose:
|
expose:
|
||||||
- "8080"
|
- "8080"
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -89,6 +154,8 @@ services:
|
|||||||
condition: service_completed_successfully
|
condition: service_completed_successfully
|
||||||
valkey:
|
valkey:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
|
||||||
interval: 2s
|
interval: 2s
|
||||||
@@ -101,6 +168,7 @@ services:
|
|||||||
mem_limit: ${API_MEMORY}
|
mem_limit: ${API_MEMORY}
|
||||||
stop_grace_period: 8s
|
stop_grace_period: 8s
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [application, data]
|
||||||
|
|
||||||
nginx:
|
nginx:
|
||||||
build:
|
build:
|
||||||
@@ -120,6 +188,7 @@ services:
|
|||||||
cpus: ${NGINX_CPUS}
|
cpus: ${NGINX_CPUS}
|
||||||
mem_limit: ${NGINX_MEMORY}
|
mem_limit: ${NGINX_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [edge, application, monitoring]
|
||||||
|
|
||||||
swagger:
|
swagger:
|
||||||
build:
|
build:
|
||||||
@@ -134,6 +203,7 @@ services:
|
|||||||
cpus: ${SWAGGER_CPUS}
|
cpus: ${SWAGGER_CPUS}
|
||||||
mem_limit: ${SWAGGER_MEMORY}
|
mem_limit: ${SWAGGER_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [edge]
|
||||||
|
|
||||||
tests:
|
tests:
|
||||||
profiles: ["tests"]
|
profiles: ["tests"]
|
||||||
@@ -150,6 +220,7 @@ services:
|
|||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
cpus: ${TEST_CPUS}
|
cpus: ${TEST_CPUS}
|
||||||
mem_limit: ${TEST_MEMORY}
|
mem_limit: ${TEST_MEMORY}
|
||||||
|
networks: [data]
|
||||||
|
|
||||||
postgres-exporter:
|
postgres-exporter:
|
||||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
|
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
|
||||||
@@ -164,6 +235,7 @@ services:
|
|||||||
cpus: ${EXPORTER_CPUS}
|
cpus: ${EXPORTER_CPUS}
|
||||||
mem_limit: ${EXPORTER_MEMORY}
|
mem_limit: ${EXPORTER_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [data, monitoring]
|
||||||
|
|
||||||
valkey-exporter:
|
valkey-exporter:
|
||||||
image: oliver006/redis_exporter:v1.89.0-alpine
|
image: oliver006/redis_exporter:v1.89.0-alpine
|
||||||
@@ -177,6 +249,7 @@ services:
|
|||||||
cpus: ${EXPORTER_CPUS}
|
cpus: ${EXPORTER_CPUS}
|
||||||
mem_limit: ${EXPORTER_MEMORY}
|
mem_limit: ${EXPORTER_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [data, monitoring]
|
||||||
|
|
||||||
nginx-exporter:
|
nginx-exporter:
|
||||||
image: nginx/nginx-prometheus-exporter:1.5.1
|
image: nginx/nginx-prometheus-exporter:1.5.1
|
||||||
@@ -187,6 +260,7 @@ services:
|
|||||||
cpus: ${EXPORTER_CPUS}
|
cpus: ${EXPORTER_CPUS}
|
||||||
mem_limit: ${EXPORTER_MEMORY}
|
mem_limit: ${EXPORTER_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [monitoring]
|
||||||
|
|
||||||
prometheus:
|
prometheus:
|
||||||
build:
|
build:
|
||||||
@@ -211,9 +285,11 @@ services:
|
|||||||
- nginx-exporter
|
- nginx-exporter
|
||||||
- postgres-exporter
|
- postgres-exporter
|
||||||
- valkey-exporter
|
- valkey-exporter
|
||||||
|
- s3
|
||||||
cpus: ${PROMETHEUS_CPUS}
|
cpus: ${PROMETHEUS_CPUS}
|
||||||
mem_limit: ${PROMETHEUS_MEMORY}
|
mem_limit: ${PROMETHEUS_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [edge, application, data, monitoring]
|
||||||
|
|
||||||
grafana:
|
grafana:
|
||||||
build:
|
build:
|
||||||
@@ -233,9 +309,20 @@ services:
|
|||||||
cpus: ${GRAFANA_CPUS}
|
cpus: ${GRAFANA_CPUS}
|
||||||
mem_limit: ${GRAFANA_MEMORY}
|
mem_limit: ${GRAFANA_MEMORY}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks: [edge, monitoring]
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
valkey_data:
|
valkey_data:
|
||||||
prometheus_data:
|
prometheus_data:
|
||||||
grafana_data:
|
grafana_data:
|
||||||
|
s3_data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
edge:
|
||||||
|
application:
|
||||||
|
internal: true
|
||||||
|
data:
|
||||||
|
internal: true
|
||||||
|
monitoring:
|
||||||
|
internal: true
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
services:
|
||||||
|
api:
|
||||||
|
image: ${API_IMAGE:?API_IMAGE is required}
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
image: ${API_IMAGE:?API_IMAGE is required}
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: ${NGINX_IMAGE:?NGINX_IMAGE is required}
|
||||||
|
|
||||||
|
swagger:
|
||||||
|
image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required}
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required}
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required}
|
||||||
|
|
||||||
|
s3:
|
||||||
|
image: ${S3_IMAGE:?S3_IMAGE is required}
|
||||||
|
|
||||||
|
s3-bootstrap:
|
||||||
|
image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
services:
|
||||||
|
nginx:
|
||||||
|
volumes:
|
||||||
|
- ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
{
|
||||||
|
"page_size": 10,
|
||||||
|
"seeded_consistent": 11,
|
||||||
|
"fresh_apply": {
|
||||||
|
"duration_ms": 4,
|
||||||
|
"report": {
|
||||||
|
"metadata_scanned": 14,
|
||||||
|
"objects_scanned": 13,
|
||||||
|
"consistent": 12,
|
||||||
|
"fresh_pending_attachments": 1,
|
||||||
|
"old_pending_attachments": 0,
|
||||||
|
"deleting_attachments": 0,
|
||||||
|
"fresh_unreferenced_attachments": 11,
|
||||||
|
"old_unreferenced_attachments": 0,
|
||||||
|
"deleted_attachment_objects": 0,
|
||||||
|
"fresh_orphan_objects": 1,
|
||||||
|
"old_orphan_objects": 0,
|
||||||
|
"deleted_orphan_objects": 0,
|
||||||
|
"fresh_missing_objects": 2,
|
||||||
|
"referenced_missing_objects": 0,
|
||||||
|
"old_unreferenced_metadata": 0,
|
||||||
|
"deleted_unreferenced_rows": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dry_run": {
|
||||||
|
"duration_ms": 5,
|
||||||
|
"report": {
|
||||||
|
"metadata_scanned": 14,
|
||||||
|
"objects_scanned": 13,
|
||||||
|
"consistent": 11,
|
||||||
|
"fresh_pending_attachments": 0,
|
||||||
|
"old_pending_attachments": 1,
|
||||||
|
"deleting_attachments": 0,
|
||||||
|
"fresh_unreferenced_attachments": 11,
|
||||||
|
"old_unreferenced_attachments": 0,
|
||||||
|
"deleted_attachment_objects": 0,
|
||||||
|
"fresh_orphan_objects": 0,
|
||||||
|
"old_orphan_objects": 1,
|
||||||
|
"deleted_orphan_objects": 0,
|
||||||
|
"fresh_missing_objects": 0,
|
||||||
|
"referenced_missing_objects": 1,
|
||||||
|
"old_unreferenced_metadata": 1,
|
||||||
|
"deleted_unreferenced_rows": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"apply": {
|
||||||
|
"duration_ms": 13,
|
||||||
|
"report": {
|
||||||
|
"metadata_scanned": 14,
|
||||||
|
"objects_scanned": 13,
|
||||||
|
"consistent": 11,
|
||||||
|
"fresh_pending_attachments": 0,
|
||||||
|
"old_pending_attachments": 1,
|
||||||
|
"deleting_attachments": 0,
|
||||||
|
"fresh_unreferenced_attachments": 11,
|
||||||
|
"old_unreferenced_attachments": 0,
|
||||||
|
"deleted_attachment_objects": 1,
|
||||||
|
"fresh_orphan_objects": 0,
|
||||||
|
"old_orphan_objects": 1,
|
||||||
|
"deleted_orphan_objects": 1,
|
||||||
|
"fresh_missing_objects": 0,
|
||||||
|
"referenced_missing_objects": 1,
|
||||||
|
"old_unreferenced_metadata": 1,
|
||||||
|
"deleted_unreferenced_rows": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"repeated_apply": {
|
||||||
|
"duration_ms": 1,
|
||||||
|
"report": {
|
||||||
|
"metadata_scanned": 12,
|
||||||
|
"objects_scanned": 11,
|
||||||
|
"consistent": 11,
|
||||||
|
"fresh_pending_attachments": 0,
|
||||||
|
"old_pending_attachments": 0,
|
||||||
|
"deleting_attachments": 0,
|
||||||
|
"fresh_unreferenced_attachments": 11,
|
||||||
|
"old_unreferenced_attachments": 0,
|
||||||
|
"deleted_attachment_objects": 0,
|
||||||
|
"fresh_orphan_objects": 0,
|
||||||
|
"old_orphan_objects": 0,
|
||||||
|
"deleted_orphan_objects": 0,
|
||||||
|
"fresh_missing_objects": 0,
|
||||||
|
"referenced_missing_objects": 1,
|
||||||
|
"old_unreferenced_metadata": 0,
|
||||||
|
"deleted_unreferenced_rows": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"fresh_preserved": true,
|
||||||
|
"orphan_deleted": true,
|
||||||
|
"unused_metadata_deleted": true,
|
||||||
|
"in_use_metadata_preserved": true,
|
||||||
|
"abandoned_attachment_deleted": true,
|
||||||
|
"total_duration_ms": 328
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"replicas": 2,
|
||||||
|
"concurrency": 12,
|
||||||
|
"rounds": 4,
|
||||||
|
"settle_time_ms": 3000,
|
||||||
|
"image_bytes": 3759914,
|
||||||
|
"upload_rate_bytes_per_second": 1572864,
|
||||||
|
"successful_uploads": 48,
|
||||||
|
"failed_uploads": 0,
|
||||||
|
"duration_ms": 12974,
|
||||||
|
"heap_baseline_bytes": 153985824,
|
||||||
|
"heap_peak_bytes": 202505208,
|
||||||
|
"heap_delta_bytes": 48519384,
|
||||||
|
"rss_baseline_bytes": 335298560,
|
||||||
|
"rss_peak_bytes": 345976832,
|
||||||
|
"rss_delta_bytes": 10678272
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
Date: 2026-10-09
|
||||||
|
Source commit: 37d93c340e394ed4a923972accc6c872abb96076
|
||||||
|
Result collection state: application, migrations, Compose and Nginx configuration matched the source commit; REPORT.md and evidence/ were refreshed afterward
|
||||||
|
Environment: Docker 29.2.1, linux/arm64, 2 CPUs, 2054299648 bytes RAM
|
||||||
|
Application topology: 2 API replicas, 512 MiB limit per replica
|
||||||
|
|
||||||
|
PASS gofmt for changed Go packages
|
||||||
|
PASS go test ./internal/...
|
||||||
|
PASS go build ./cmd/messenger and ./cmd/lab4experiment in Docker build
|
||||||
|
PASS make check
|
||||||
|
PASS docker compose config --quiet
|
||||||
|
PASS sh -n scripts/rolling-update.sh
|
||||||
|
PASS make test BASE_URL=http://127.0.0.1:28080
|
||||||
|
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
PASS PostgreSQL and Valkey integration suites
|
||||||
|
PASS MinIO bootstrap completed twice without an error
|
||||||
|
PASS anonymous bucket request returned HTTP 403
|
||||||
|
PASS real pending -> deleting -> delete on PostgreSQL
|
||||||
|
PASS stale pending candidate cannot claim an attachment already changed to ready
|
||||||
|
PASS 48 of 48 uploads succeeded in 4 waves with concurrency 12
|
||||||
|
PASS reconciler preserved fresh and referenced anomalies
|
||||||
|
PASS reconciler deleted an old orphan, old unreferenced missing metadata, and old pending data
|
||||||
|
PASS repeated reconciliation apply made no further changes
|
||||||
|
PASS Nginx accepted the candidate upstream with nginx -t and atomic reload
|
||||||
|
PASS 20 of 20 health requests returned HTTP 200 from only the two candidate API instances
|
||||||
|
|
||||||
|
CI additionally runs full OpenAPI validation, race tests, golangci-lint,
|
||||||
|
persistence and API-replica failover on the exact pushed commit. Production
|
||||||
|
deployment publishes verified images and passes immutable repository@sha256
|
||||||
|
references to the manually approved environment job.
|
||||||
@@ -5,37 +5,49 @@ go 1.26.0
|
|||||||
require (
|
require (
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/jackc/pgx/v5 v5.11.0
|
github.com/jackc/pgx/v5 v5.11.0
|
||||||
|
github.com/minio/minio-go/v7 v7.0.99
|
||||||
github.com/prometheus/client_golang v1.23.2
|
github.com/prometheus/client_golang v1.23.2
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
github.com/valkey-io/valkey-go v1.0.78
|
github.com/valkey-io/valkey-go v1.0.78
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
golang.org/x/sync v0.23.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
|
github.com/go-ini/ini v1.67.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
|
github.com/klauspost/compress v1.18.2 // indirect
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||||
|
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||||
|
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||||
|
github.com/minio/md5-simd v1.1.2 // indirect
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
|
github.com/philhofer/fwd v1.2.0 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/prometheus/client_model v0.6.2 // indirect
|
github.com/prometheus/client_model v0.6.2 // indirect
|
||||||
github.com/prometheus/common v0.66.1 // indirect
|
github.com/prometheus/common v0.66.1 // indirect
|
||||||
github.com/prometheus/procfs v0.16.1 // indirect
|
github.com/prometheus/procfs v0.16.1 // indirect
|
||||||
|
github.com/rs/xid v1.6.0 // indirect
|
||||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||||
github.com/spf13/afero v1.15.0 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
github.com/spf13/cast v1.10.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
|
github.com/tinylib/msgp v1.6.1 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/net v0.58.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.8 // indirect
|
google.golang.org/protobuf v1.36.8 // indirect
|
||||||
|
|||||||
@@ -5,10 +5,14 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
|||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
|
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||||
|
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
@@ -23,20 +27,33 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
|||||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||||
|
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||||
|
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
|
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||||
|
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||||
|
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||||
|
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
||||||
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||||
|
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||||
|
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||||
@@ -49,6 +66,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
|
|||||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||||
|
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||||
|
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||||
@@ -68,6 +87,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
|||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
|
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||||
|
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||||
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
||||||
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
alias_name=bootstrap
|
||||||
|
policy_name=messenger-attachments
|
||||||
|
policy_file=/tmp/messenger-attachments-policy.json
|
||||||
|
|
||||||
|
mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||||
|
mc mb --ignore-existing "$alias_name/$S3_BUCKET"
|
||||||
|
mc anonymous set none "$alias_name/$S3_BUCKET"
|
||||||
|
|
||||||
|
cat >"$policy_file" <<EOF
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||||
|
"Resource": ["arn:aws:s3:::$S3_BUCKET"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mc admin policy create "$alias_name" "$policy_name" "$policy_file"
|
||||||
|
mc admin user add "$alias_name" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||||
|
mc admin policy attach "$alias_name" "$policy_name" --user "$S3_ACCESS_KEY"
|
||||||
|
|
||||||
|
mc alias set application http://s3:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||||
|
mc ls "application/$S3_BUCKET" >/dev/null
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
FROM golang:1.24.8-alpine3.22 AS build
|
||||||
|
|
||||||
|
ARG MC_VERSION
|
||||||
|
RUN apk add --no-cache git
|
||||||
|
RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc
|
||||||
|
WORKDIR /src/mc
|
||||||
|
RUN go mod download
|
||||||
|
RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||||
|
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc .
|
||||||
|
|
||||||
|
FROM alpine:3.22.2
|
||||||
|
|
||||||
|
COPY --from=build /out/mc /usr/local/bin/mc
|
||||||
|
COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh
|
||||||
|
|
||||||
|
ENTRYPOINT ["mc"]
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
FROM golang:1.24.8-alpine3.22 AS build
|
||||||
|
|
||||||
|
ARG MINIO_VERSION
|
||||||
|
RUN apk add --no-cache git
|
||||||
|
RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio
|
||||||
|
WORKDIR /src/minio
|
||||||
|
RUN go mod download
|
||||||
|
RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||||
|
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio .
|
||||||
|
|
||||||
|
FROM alpine:3.22.2
|
||||||
|
|
||||||
|
RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data
|
||||||
|
COPY --from=build /out/minio /usr/local/bin/minio
|
||||||
|
|
||||||
|
USER minio
|
||||||
|
VOLUME ["/data"]
|
||||||
|
ENTRYPOINT ["minio"]
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
func RunMigrations() error {
|
||||||
|
settings, err := config.LoadMigration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pool, err := database.Open(context.Background(), database.Config{
|
||||||
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||||
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||||
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer pool.Close()
|
||||||
|
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"io"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||||
|
)
|
||||||
|
|
||||||
|
type reconciliationOutput struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
GracePeriod string `json:"grace_period"`
|
||||||
|
UnreferencedRetention string `json:"unreferenced_retention"`
|
||||||
|
PageSize int `json:"page_size"`
|
||||||
|
DurationMillis int64 `json:"duration_ms"`
|
||||||
|
Report reconcile.Report `json:"report"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func RunReconciliation(arguments []string, output io.Writer) error {
|
||||||
|
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(io.Discard)
|
||||||
|
apply := flags.Bool("apply", false, "apply safe repairs")
|
||||||
|
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
|
||||||
|
retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments")
|
||||||
|
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
|
||||||
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||||
|
return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]")
|
||||||
|
}
|
||||||
|
if *apply && *gracePeriod <= 0 {
|
||||||
|
return errors.New("grace-period must be positive in apply mode")
|
||||||
|
}
|
||||||
|
settings, err := config.LoadReconciliation()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
pool, err := database.Open(ctx, database.Config{
|
||||||
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||||
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||||
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer pool.Close()
|
||||||
|
objects, err := objectstore.New(objectstore.Config{
|
||||||
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||||
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||||
|
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||||
|
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
|
||||||
|
UnreferencedRetention: *retention, Apply: *apply,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
started := time.Now()
|
||||||
|
report, err := reconciler.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
mode := "dry-run"
|
||||||
|
if *apply {
|
||||||
|
mode = "apply"
|
||||||
|
}
|
||||||
|
return json.NewEncoder(output).Encode(reconciliationOutput{
|
||||||
|
Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(),
|
||||||
|
PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) {
|
||||||
|
err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard)
|
||||||
|
require.EqualError(t, err, "grace-period must be positive in apply mode")
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/httpapi"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||||
|
)
|
||||||
|
|
||||||
|
func RunServer(logger *slog.Logger) error {
|
||||||
|
settings, err := config.LoadServer()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
instanceID, err := os.Hostname()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pool, err := database.Open(context.Background(), databaseConfig(settings))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer pool.Close()
|
||||||
|
sessions, err := session.New(settings.CacheURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = sessions.Close() }()
|
||||||
|
passwords, err := password.New(password.Config{
|
||||||
|
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
|
||||||
|
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
|
||||||
|
KeyLength: settings.PasswordKeyLength,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
objects, err := objectstore.New(objectStoreConfig(settings))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repository := postgres.New(pool)
|
||||||
|
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
|
||||||
|
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||||
|
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
|
||||||
|
CleanupTimeout: settings.RequestTimeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
app := httpapi.NewServer(logger, httpapi.Dependencies{
|
||||||
|
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
|
||||||
|
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
|
||||||
|
}, httpapi.Config{
|
||||||
|
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
|
||||||
|
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
|
||||||
|
})
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
|
||||||
|
ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout,
|
||||||
|
IdleTimeout: settings.HTTPIdleTimeout,
|
||||||
|
}
|
||||||
|
listener, err := net.Listen("tcp", server.Addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
serveResult := make(chan error, 1)
|
||||||
|
go func() { serveResult <- server.Serve(listener) }()
|
||||||
|
logger.Info("server started", "address", server.Addr)
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
select {
|
||||||
|
case err := <-serveResult:
|
||||||
|
return err
|
||||||
|
case <-ctx.Done():
|
||||||
|
logger.Info("shutdown requested")
|
||||||
|
app.SetReady(false)
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||||
|
_ = server.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func databaseConfig(settings config.ServerConfig) database.Config {
|
||||||
|
return database.Config{
|
||||||
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||||
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||||
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStoreConfig(settings config.ServerConfig) objectstore.Config {
|
||||||
|
return objectstore.Config{
|
||||||
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||||
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||||
|
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||||
|
}
|
||||||
|
}
|
||||||
+185
-47
@@ -11,30 +11,41 @@ import (
|
|||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type ServerConfig struct {
|
||||||
Address string
|
Address string
|
||||||
DatabaseURL string
|
DatabaseURL string
|
||||||
CacheURL string
|
CacheURL string
|
||||||
SessionTTL time.Duration
|
SessionTTL time.Duration
|
||||||
IdempotencyTTL time.Duration
|
IdempotencyTTL time.Duration
|
||||||
DBMaxConns int32
|
DBMaxConns int32
|
||||||
DBMinConns int32
|
DBMinConns int32
|
||||||
DBMaxConnectionAge time.Duration
|
DBMaxConnectionAge time.Duration
|
||||||
DBMaxConnectionIdle time.Duration
|
DBMaxConnectionIdle time.Duration
|
||||||
DBHealthCheckPeriod time.Duration
|
DBHealthCheckPeriod time.Duration
|
||||||
DependencyTimeout time.Duration
|
DependencyTimeout time.Duration
|
||||||
RequestTimeout time.Duration
|
RequestTimeout time.Duration
|
||||||
HTTPReadHeaderTimeout time.Duration
|
HTTPReadHeaderTimeout time.Duration
|
||||||
HTTPReadTimeout time.Duration
|
HTTPReadTimeout time.Duration
|
||||||
HTTPWriteTimeout time.Duration
|
HTTPWriteTimeout time.Duration
|
||||||
HTTPIdleTimeout time.Duration
|
HTTPIdleTimeout time.Duration
|
||||||
HTTPShutdownTimeout time.Duration
|
HTTPShutdownTimeout time.Duration
|
||||||
PasswordMemoryKiB uint32
|
PasswordMemoryKiB uint32
|
||||||
PasswordIterations uint32
|
PasswordIterations uint32
|
||||||
PasswordParallelism uint8
|
PasswordParallelism uint8
|
||||||
PasswordSaltLength uint32
|
PasswordSaltLength uint32
|
||||||
PasswordKeyLength uint32
|
PasswordKeyLength uint32
|
||||||
SwaggerPort string
|
SwaggerPort string
|
||||||
|
S3Endpoint string
|
||||||
|
S3Region string
|
||||||
|
S3Bucket string
|
||||||
|
S3AccessKey string
|
||||||
|
S3SecretKey string
|
||||||
|
S3UseTLS bool
|
||||||
|
S3Timeout time.Duration
|
||||||
|
AttachmentMaxBytes int64
|
||||||
|
AttachmentMaxPixels int64
|
||||||
|
AttachmentBodyMemory int64
|
||||||
|
AttachmentDecodeMemory int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type MigrationConfig struct {
|
type MigrationConfig struct {
|
||||||
@@ -48,6 +59,23 @@ type MigrationConfig struct {
|
|||||||
MigrationTimeout time.Duration
|
MigrationTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ReconciliationConfig struct {
|
||||||
|
DatabaseURL string
|
||||||
|
DBMaxConns int32
|
||||||
|
DBMinConns int32
|
||||||
|
DBMaxConnectionAge time.Duration
|
||||||
|
DBMaxConnectionIdle time.Duration
|
||||||
|
DBHealthCheckPeriod time.Duration
|
||||||
|
DependencyTimeout time.Duration
|
||||||
|
S3Endpoint string
|
||||||
|
S3Region string
|
||||||
|
S3Bucket string
|
||||||
|
S3AccessKey string
|
||||||
|
S3SecretKey string
|
||||||
|
S3UseTLS bool
|
||||||
|
S3Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
type databaseConfig struct {
|
type databaseConfig struct {
|
||||||
url string
|
url string
|
||||||
maxConns int
|
maxConns int
|
||||||
@@ -58,92 +86,129 @@ type databaseConfig struct {
|
|||||||
dependencyTimeout time.Duration
|
dependencyTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load() (Config, error) {
|
type objectStorageConfig struct {
|
||||||
|
endpoint string
|
||||||
|
region string
|
||||||
|
bucket string
|
||||||
|
accessKey string
|
||||||
|
secretKey string
|
||||||
|
useTLS bool
|
||||||
|
timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadServer() (ServerConfig, error) {
|
||||||
values := viper.New()
|
values := viper.New()
|
||||||
values.AutomaticEnv()
|
values.AutomaticEnv()
|
||||||
|
|
||||||
host, err := requiredString(values, "APP_HOST")
|
host, err := requiredString(values, "APP_HOST")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
port, err := requiredPort(values, "APP_PORT")
|
port, err := requiredPort(values, "APP_PORT")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
database, err := loadDatabaseConfig(values)
|
database, err := loadDatabaseConfig(values)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
cacheURL, err := requiredString(values, "CACHE_URL")
|
cacheURL, err := requiredString(values, "CACHE_URL")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
store, err := requiredString(values, "SESSION_STORE")
|
store, err := requiredString(values, "SESSION_STORE")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
if store != "valkey" {
|
if store != "valkey" {
|
||||||
return Config{}, errors.New("SESSION_STORE must be valkey")
|
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
|
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
|
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
|
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
|
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
|
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
|
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
|
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
|
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
|
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
|
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
|
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
|
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
}
|
}
|
||||||
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
|
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
storage, err := loadObjectStorageConfig(values)
|
||||||
|
if err != nil {
|
||||||
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||||
|
if err != nil {
|
||||||
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||||
|
if err != nil {
|
||||||
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||||
|
if err != nil {
|
||||||
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
|
||||||
|
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
|
||||||
|
}
|
||||||
|
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||||
|
if err != nil {
|
||||||
|
return ServerConfig{}, err
|
||||||
|
}
|
||||||
|
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
|
||||||
|
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||||
|
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||||
}
|
}
|
||||||
|
|
||||||
return Config{
|
return ServerConfig{
|
||||||
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
||||||
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
|
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
|
||||||
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||||
@@ -155,7 +220,12 @@ func Load() (Config, error) {
|
|||||||
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
|
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
|
||||||
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
|
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
|
||||||
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
|
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
|
||||||
SwaggerPort: swaggerPort,
|
SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||||
|
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
|
||||||
|
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
|
||||||
|
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||||
|
AttachmentBodyMemory: int64(attachmentBodyMemory),
|
||||||
|
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +248,62 @@ func LoadMigration() (MigrationConfig, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func LoadReconciliation() (ReconciliationConfig, error) {
|
||||||
|
values := viper.New()
|
||||||
|
values.AutomaticEnv()
|
||||||
|
database, err := loadDatabaseConfig(values)
|
||||||
|
if err != nil {
|
||||||
|
return ReconciliationConfig{}, err
|
||||||
|
}
|
||||||
|
storage, err := loadObjectStorageConfig(values)
|
||||||
|
if err != nil {
|
||||||
|
return ReconciliationConfig{}, err
|
||||||
|
}
|
||||||
|
return ReconciliationConfig{
|
||||||
|
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||||
|
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||||
|
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||||
|
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||||
|
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
|
||||||
|
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||||
|
endpoint, err := requiredString(values, "S3_ENDPOINT")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
region, err := requiredString(values, "S3_REGION")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
bucket, err := requiredString(values, "S3_BUCKET")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
secretKey, err := requiredString(values, "S3_SECRET_KEY")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
useTLS, err := boolean(values, "S3_USE_TLS")
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
|
||||||
|
if err != nil {
|
||||||
|
return objectStorageConfig{}, err
|
||||||
|
}
|
||||||
|
return objectStorageConfig{
|
||||||
|
endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey,
|
||||||
|
secretKey: secretKey, useTLS: useTLS, timeout: timeout,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
||||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -260,3 +386,15 @@ func integerValue(value string, minimum, maximum int) (int, error) {
|
|||||||
}
|
}
|
||||||
return parsed, nil
|
return parsed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func boolean(values *viper.Viper, name string) (bool, error) {
|
||||||
|
value, err := requiredString(values, name)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
parsed, err := strconv.ParseBool(value)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("%s: must be true or false", name)
|
||||||
|
}
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||||
|
values := map[string]string{
|
||||||
|
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||||
|
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
|
||||||
|
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
|
||||||
|
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
|
||||||
|
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||||
|
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||||
|
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
|
||||||
|
}
|
||||||
|
for name, value := range values {
|
||||||
|
t.Setenv(name, value)
|
||||||
|
}
|
||||||
|
t.Setenv("APP_HOST", "")
|
||||||
|
t.Setenv("CACHE_URL", "")
|
||||||
|
t.Setenv("SWAGGER_PORT", "")
|
||||||
|
|
||||||
|
settings, err := LoadReconciliation()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "attachments", settings.S3Bucket)
|
||||||
|
}
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
package attachment
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"image"
|
||||||
|
_ "image/jpeg"
|
||||||
|
_ "image/png"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"math"
|
||||||
|
"mime"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"golang.org/x/sync/semaphore"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Repository interface {
|
||||||
|
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||||
|
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
|
||||||
|
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
|
||||||
|
DeletePendingAttachment(context.Context, string, string) error
|
||||||
|
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
|
||||||
|
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ObjectStore interface {
|
||||||
|
Put(context.Context, string, string, []byte) error
|
||||||
|
Get(context.Context, string) (io.ReadCloser, error)
|
||||||
|
Delete(context.Context, string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
MaxBytes int64
|
||||||
|
MaxPixels int64
|
||||||
|
BodyMemoryBytes int64
|
||||||
|
DecodeMemoryBytes int64
|
||||||
|
CleanupTimeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type Upload struct {
|
||||||
|
ChatID string
|
||||||
|
OwnerID string
|
||||||
|
Filename string
|
||||||
|
ContentType string
|
||||||
|
ContentLength int64
|
||||||
|
Body io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
type Content struct {
|
||||||
|
Attachment model.Attachment
|
||||||
|
Body io.ReadCloser
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service struct {
|
||||||
|
repository Repository
|
||||||
|
objects ObjectStore
|
||||||
|
config Config
|
||||||
|
bodyMemory *semaphore.Weighted
|
||||||
|
decodeMemory *semaphore.Weighted
|
||||||
|
logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
|
||||||
|
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
|
||||||
|
return nil, errors.New("invalid attachment service configuration")
|
||||||
|
}
|
||||||
|
if config.BodyMemoryBytes < config.MaxBytes {
|
||||||
|
return nil, errors.New("attachment body memory must fit one maximum upload")
|
||||||
|
}
|
||||||
|
if config.DecodeMemoryBytes < config.MaxPixels*8 {
|
||||||
|
return nil, errors.New("attachment decode memory must fit one maximum image")
|
||||||
|
}
|
||||||
|
return &Service{
|
||||||
|
repository: repository, objects: objects, config: config,
|
||||||
|
bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes),
|
||||||
|
decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) {
|
||||||
|
if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil {
|
||||||
|
return model.Attachment{}, mapDependencyError(err)
|
||||||
|
}
|
||||||
|
data, contentType, release, err := s.readImage(input)
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
attachmentID := uuid.NewString()
|
||||||
|
objectKey := "attachments/" + uuid.NewString() + "/original"
|
||||||
|
digest := sha256.Sum256(data)
|
||||||
|
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||||
|
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
|
||||||
|
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||||
|
ObjectKey: objectKey, Status: model.AttachmentPending,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, mapDependencyError(err)
|
||||||
|
}
|
||||||
|
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
|
||||||
|
s.deletePending(pending.ID, objectKey)
|
||||||
|
return model.Attachment{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
|
||||||
|
if err == nil {
|
||||||
|
return attachment, nil
|
||||||
|
}
|
||||||
|
return s.resolveReadyOutcome(pending.ID, objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
|
||||||
|
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
|
||||||
|
if err == nil && stored.Status == model.AttachmentReady {
|
||||||
|
return stored, nil
|
||||||
|
}
|
||||||
|
if err == nil && stored.Status == model.AttachmentPending {
|
||||||
|
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||||
|
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||||
|
return model.Attachment{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
|
||||||
|
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
|
||||||
|
}
|
||||||
|
return model.Attachment{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
if errors.Is(err, model.ErrNotFound) {
|
||||||
|
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||||
|
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||||
|
}
|
||||||
|
return model.Attachment{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
|
||||||
|
return model.Attachment{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) deletePending(attachmentID, objectKey string) {
|
||||||
|
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
|
||||||
|
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
|
||||||
|
attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID)
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, mapDependencyError(err)
|
||||||
|
}
|
||||||
|
return attachment, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
|
||||||
|
attachment, err := s.Metadata(ctx, attachmentID, userID)
|
||||||
|
if err != nil {
|
||||||
|
return Content{}, err
|
||||||
|
}
|
||||||
|
body, err := s.objects.Get(ctx, attachment.ObjectKey)
|
||||||
|
if err != nil {
|
||||||
|
return Content{}, model.ErrUnavailable
|
||||||
|
}
|
||||||
|
return Content{Attachment: attachment, Body: body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
|
||||||
|
mediaType, parameters, err := mime.ParseMediaType(input.ContentType)
|
||||||
|
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
|
||||||
|
return nil, "", nil, model.ErrUnsupportedMedia
|
||||||
|
}
|
||||||
|
if input.ContentLength > s.config.MaxBytes {
|
||||||
|
return nil, "", nil, model.ErrPayloadTooLarge
|
||||||
|
}
|
||||||
|
reservation := s.config.MaxBytes
|
||||||
|
if !s.bodyMemory.TryAcquire(reservation) {
|
||||||
|
return nil, "", nil, model.ErrCapacity
|
||||||
|
}
|
||||||
|
releaseBody := func() { s.bodyMemory.Release(reservation) }
|
||||||
|
data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrInvalidInput
|
||||||
|
}
|
||||||
|
if int64(len(data)) > s.config.MaxBytes {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrPayloadTooLarge
|
||||||
|
}
|
||||||
|
if len(data) == 0 {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrUnsupportedMedia
|
||||||
|
}
|
||||||
|
detected := http.DetectContentType(data)
|
||||||
|
if detected != mediaType {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrUnsupportedMedia
|
||||||
|
}
|
||||||
|
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||||
|
if err != nil || imageContentType(format) != mediaType {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrUnsupportedMedia
|
||||||
|
}
|
||||||
|
if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrPayloadTooLarge
|
||||||
|
}
|
||||||
|
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
|
||||||
|
decodeBytes := pixels * 8
|
||||||
|
if !s.decodeMemory.TryAcquire(decodeBytes) {
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrCapacity
|
||||||
|
}
|
||||||
|
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
|
||||||
|
s.decodeMemory.Release(decodeBytes)
|
||||||
|
releaseBody()
|
||||||
|
return nil, "", nil, model.ErrUnsupportedMedia
|
||||||
|
}
|
||||||
|
release := func() {
|
||||||
|
s.decodeMemory.Release(decodeBytes)
|
||||||
|
releaseBody()
|
||||||
|
}
|
||||||
|
return data, mediaType, release, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func imageContentType(format string) string {
|
||||||
|
switch format {
|
||||||
|
case "png":
|
||||||
|
return "image/png"
|
||||||
|
case "jpeg":
|
||||||
|
return "image/jpeg"
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapDependencyError(err error) error {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, model.ErrNotFound):
|
||||||
|
return model.ErrNotFound
|
||||||
|
case errors.Is(err, model.ErrConflict):
|
||||||
|
return model.ErrConflict
|
||||||
|
default:
|
||||||
|
return model.ErrUnavailable
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
package attachment
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"image"
|
||||||
|
"image/color"
|
||||||
|
"image/png"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUploadLifecycle(t *testing.T) {
|
||||||
|
data := pngFixture(t, 2, 2)
|
||||||
|
events := make([]string, 0, 3)
|
||||||
|
repository := &repositoryStub{events: &events}
|
||||||
|
objects := &objectStub{events: &events}
|
||||||
|
service := newTestService(t, repository, objects, 1024, 4)
|
||||||
|
|
||||||
|
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, model.AttachmentReady, result.Status)
|
||||||
|
require.Equal(t, []string{"pending", "put", "ready"}, events)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadValidation(t *testing.T) {
|
||||||
|
data := pngFixture(t, 2, 2)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contentType string
|
||||||
|
data []byte
|
||||||
|
maxBytes int64
|
||||||
|
maxPixels int64
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
|
||||||
|
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge},
|
||||||
|
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge},
|
||||||
|
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels)
|
||||||
|
input := upload(test.data, test.contentType)
|
||||||
|
_, err := service.Upload(context.Background(), input)
|
||||||
|
require.ErrorIs(t, err, test.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
|
||||||
|
data := pngFixture(t, 2, 2)
|
||||||
|
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
|
||||||
|
require.True(t, service.bodyMemory.TryAcquire(1024))
|
||||||
|
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||||
|
require.ErrorIs(t, err, model.ErrCapacity)
|
||||||
|
service.bodyMemory.Release(1024)
|
||||||
|
|
||||||
|
require.True(t, service.decodeMemory.TryAcquire(32))
|
||||||
|
_, err = service.Upload(context.Background(), upload(data, "image/png"))
|
||||||
|
require.ErrorIs(t, err, model.ErrCapacity)
|
||||||
|
service.decodeMemory.Release(32)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) {
|
||||||
|
oversized := pngFixture(t, 3, 2)
|
||||||
|
valid := pngFixture(t, 2, 2)
|
||||||
|
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
|
||||||
|
|
||||||
|
_, err := service.Upload(context.Background(), upload(oversized, "image/png"))
|
||||||
|
require.ErrorIs(t, err, model.ErrPayloadTooLarge)
|
||||||
|
_, err = service.Upload(context.Background(), upload(valid, "image/png"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
|
||||||
|
_, err := NewService(
|
||||||
|
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
|
||||||
|
Config{
|
||||||
|
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
|
||||||
|
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
|
||||||
|
data := pngFixture(t, 2, 2)
|
||||||
|
repository := &repositoryStub{}
|
||||||
|
objects := &objectStub{putErr: errors.New("s3 unavailable")}
|
||||||
|
service := newTestService(t, repository, objects, 1024, 4)
|
||||||
|
|
||||||
|
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||||
|
require.ErrorIs(t, err, model.ErrUnavailable)
|
||||||
|
require.True(t, repository.pendingDeleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
|
||||||
|
data := pngFixture(t, 2, 2)
|
||||||
|
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
|
||||||
|
objects := &objectStub{}
|
||||||
|
service := newTestService(t, repository, objects, 1024, 4)
|
||||||
|
|
||||||
|
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, model.AttachmentReady, result.Status)
|
||||||
|
require.False(t, objects.deleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
|
||||||
|
t.Helper()
|
||||||
|
service, err := NewService(
|
||||||
|
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
|
||||||
|
Config{
|
||||||
|
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
|
||||||
|
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return service
|
||||||
|
}
|
||||||
|
|
||||||
|
func upload(data []byte, contentType string) Upload {
|
||||||
|
return Upload{
|
||||||
|
ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType,
|
||||||
|
ContentLength: int64(len(data)), Body: bytes.NewReader(data),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type repositoryStub struct {
|
||||||
|
events *[]string
|
||||||
|
markErr error
|
||||||
|
lookupReady bool
|
||||||
|
pendingDeleted bool
|
||||||
|
pending model.Attachment
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) {
|
||||||
|
return model.Chat{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) {
|
||||||
|
attachment.Status = model.AttachmentPending
|
||||||
|
s.pending = attachment
|
||||||
|
s.record("pending")
|
||||||
|
return attachment, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
|
||||||
|
if s.markErr != nil {
|
||||||
|
return model.Attachment{}, s.markErr
|
||||||
|
}
|
||||||
|
s.pending.Status = model.AttachmentReady
|
||||||
|
s.record("ready")
|
||||||
|
return s.pending, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error {
|
||||||
|
s.pendingDeleted = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
|
||||||
|
if s.lookupReady {
|
||||||
|
s.pending.Status = model.AttachmentReady
|
||||||
|
return s.pending, nil
|
||||||
|
}
|
||||||
|
return model.Attachment{}, model.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
|
||||||
|
return s.pending, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *repositoryStub) record(event string) {
|
||||||
|
if s.events != nil {
|
||||||
|
*s.events = append(*s.events, event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectStub struct {
|
||||||
|
events *[]string
|
||||||
|
putErr error
|
||||||
|
deleted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *objectStub) Put(context.Context, string, string, []byte) error {
|
||||||
|
if s.events != nil {
|
||||||
|
*s.events = append(*s.events, "put")
|
||||||
|
}
|
||||||
|
return s.putErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) {
|
||||||
|
return io.NopCloser(bytes.NewReader(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *objectStub) Delete(context.Context, string) error {
|
||||||
|
s.deleted = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pngFixture(t *testing.T, width, height int) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var data bytes.Buffer
|
||||||
|
value := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||||
|
value.Set(0, 0, color.White)
|
||||||
|
require.NoError(t, png.Encode(&data, value))
|
||||||
|
return data.Bytes()
|
||||||
|
}
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
|
||||||
|
started := time.Now()
|
||||||
|
result := "client_error"
|
||||||
|
defer func() {
|
||||||
|
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
|
||||||
|
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
|
||||||
|
}()
|
||||||
|
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
return methodNotAllowed(w, http.MethodPost)
|
||||||
|
}
|
||||||
|
user, _, authErr := s.authenticate(r)
|
||||||
|
if authErr != nil {
|
||||||
|
return authErr
|
||||||
|
}
|
||||||
|
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
|
||||||
|
if !valid {
|
||||||
|
return invalidRequest()
|
||||||
|
}
|
||||||
|
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||||
|
if err != nil {
|
||||||
|
return invalidRequest()
|
||||||
|
}
|
||||||
|
filename, valid := attachmentFilename(query)
|
||||||
|
if !valid {
|
||||||
|
return invalidRequest()
|
||||||
|
}
|
||||||
|
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||||
|
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||||
|
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, model.ErrCapacity) {
|
||||||
|
result = "capacity_rejected"
|
||||||
|
w.Header().Set("Retry-After", "1")
|
||||||
|
} else if errors.Is(err, model.ErrUnavailable) {
|
||||||
|
result = "dependency_error"
|
||||||
|
}
|
||||||
|
return attachmentError(err)
|
||||||
|
}
|
||||||
|
result = "success"
|
||||||
|
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||||
|
writeJSON(w, http.StatusCreated, uploaded)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentFilename(query url.Values) (string, bool) {
|
||||||
|
if len(query) != 1 || len(query["filename"]) != 1 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
filename := query["filename"][0]
|
||||||
|
length := utf8.RuneCountInString(filename)
|
||||||
|
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
for _, char := range filename {
|
||||||
|
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return filename, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
return methodNotAllowed(w, http.MethodGet)
|
||||||
|
}
|
||||||
|
user, _, authErr := s.authenticate(r)
|
||||||
|
if authErr != nil {
|
||||||
|
return authErr
|
||||||
|
}
|
||||||
|
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||||
|
if apiErr != nil {
|
||||||
|
return apiErr
|
||||||
|
}
|
||||||
|
value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID)
|
||||||
|
if err != nil {
|
||||||
|
return attachmentError(err)
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
return methodNotAllowed(w, http.MethodGet)
|
||||||
|
}
|
||||||
|
user, _, authErr := s.authenticate(r)
|
||||||
|
if authErr != nil {
|
||||||
|
return authErr
|
||||||
|
}
|
||||||
|
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||||
|
if err != nil || !originalVariant(query) {
|
||||||
|
return invalidRequest()
|
||||||
|
}
|
||||||
|
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||||
|
if apiErr != nil {
|
||||||
|
return apiErr
|
||||||
|
}
|
||||||
|
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
|
||||||
|
if err != nil {
|
||||||
|
return attachmentError(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = content.Body.Close() }()
|
||||||
|
w.Header().Set("Content-Type", content.Attachment.ContentType)
|
||||||
|
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
|
||||||
|
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
if _, err := io.Copy(w, content.Body); err != nil {
|
||||||
|
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func originalVariant(query url.Values) bool {
|
||||||
|
if len(query) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
|
||||||
|
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||||
|
attachmentID, valid := canonicalID(parts[3])
|
||||||
|
if !valid {
|
||||||
|
return "", invalidRequest()
|
||||||
|
}
|
||||||
|
return attachmentID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentError(err error) *apiError {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, model.ErrNotFound):
|
||||||
|
return notFound()
|
||||||
|
case errors.Is(err, model.ErrInvalidInput):
|
||||||
|
return invalidRequest()
|
||||||
|
case errors.Is(err, model.ErrUnsupportedMedia):
|
||||||
|
return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
|
||||||
|
case errors.Is(err, model.ErrPayloadTooLarge):
|
||||||
|
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||||
|
case errors.Is(err, model.ErrCapacity):
|
||||||
|
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
|
||||||
|
default:
|
||||||
|
return unavailable()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -7,8 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
||||||
@@ -23,8 +21,8 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError
|
|||||||
if !ok {
|
if !ok {
|
||||||
return unauthorized()
|
return unauthorized()
|
||||||
}
|
}
|
||||||
credentials, err := s.messengerStore.UserByUsername(r.Context(), username)
|
credentials, err := s.userRepository.UserByUsername(r.Context(), username)
|
||||||
if errors.Is(err, postgres.ErrNotFound) {
|
if errors.Is(err, model.ErrNotFound) {
|
||||||
if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil {
|
if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil {
|
||||||
return internalError()
|
return internalError()
|
||||||
}
|
}
|
||||||
@@ -69,7 +67,7 @@ func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError
|
|||||||
if authErr != nil {
|
if authErr != nil {
|
||||||
return authErr
|
return authErr
|
||||||
}
|
}
|
||||||
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, session.ErrNotFound) {
|
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) {
|
||||||
return unauthorized()
|
return unauthorized()
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return unavailable()
|
return unavailable()
|
||||||
@@ -88,14 +86,14 @@ func (s *Server) authenticate(r *http.Request) (User, string, *apiError) {
|
|||||||
return User{}, "", unauthorized()
|
return User{}, "", unauthorized()
|
||||||
}
|
}
|
||||||
userID, err := s.sessions.UserID(r.Context(), token)
|
userID, err := s.sessions.UserID(r.Context(), token)
|
||||||
if errors.Is(err, session.ErrNotFound) || errors.Is(err, session.ErrInvalidToken) {
|
if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) {
|
||||||
return User{}, "", unauthorized()
|
return User{}, "", unauthorized()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return User{}, "", unavailable()
|
return User{}, "", unavailable()
|
||||||
}
|
}
|
||||||
user, err := s.messengerStore.UserByID(r.Context(), userID)
|
user, err := s.userRepository.UserByID(r.Context(), userID)
|
||||||
if errors.Is(err, postgres.ErrNotFound) {
|
if errors.Is(err, model.ErrNotFound) {
|
||||||
_ = s.sessions.Delete(r.Context(), token)
|
_ = s.sessions.Delete(r.Context(), token)
|
||||||
return User{}, "", unauthorized()
|
return User{}, "", unauthorized()
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -14,6 +14,7 @@ const (
|
|||||||
errorCodePayloadTooLarge = "payload_too_large"
|
errorCodePayloadTooLarge = "payload_too_large"
|
||||||
errorCodeUnavailable = "unavailable"
|
errorCodeUnavailable = "unavailable"
|
||||||
errorCodeConflict = "conflict"
|
errorCodeConflict = "conflict"
|
||||||
|
errorCodeRateLimited = "rate_limited"
|
||||||
)
|
)
|
||||||
|
|
||||||
type apiError struct {
|
type apiError struct {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||||
@@ -32,12 +31,15 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if err := s.messengerStore.Ping(ctx); err != nil {
|
if err := s.database.Ping(ctx); err != nil {
|
||||||
return unavailable()
|
return unavailable()
|
||||||
}
|
}
|
||||||
if err := s.sessions.Ping(ctx); err != nil {
|
if err := s.sessions.Ping(ctx); err != nil {
|
||||||
return unavailable()
|
return unavailable()
|
||||||
}
|
}
|
||||||
|
if err := s.objects.Ping(ctx); err != nil {
|
||||||
|
return unavailable()
|
||||||
|
}
|
||||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -61,8 +63,8 @@ func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return internalError()
|
return internalError()
|
||||||
}
|
}
|
||||||
user, err := s.messengerStore.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
|
user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
|
||||||
if errors.Is(err, postgres.ErrConflict) {
|
if errors.Is(err, model.ErrConflict) {
|
||||||
return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"}
|
return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -125,8 +127,8 @@ func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
if _, included := members[user.ID]; !included {
|
if _, included := members[user.ID]; !included {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
chat, err := s.messengerStore.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
|
chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
|
||||||
if errors.Is(err, postgres.ErrInvalidMembers) {
|
if errors.Is(err, model.ErrInvalidMembers) {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -141,8 +143,8 @@ func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *a
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
items, hasNext, listErr := s.messengerStore.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
|
items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
|
||||||
if errors.Is(listErr, postgres.ErrInvalidCursor) {
|
if errors.Is(listErr, model.ErrInvalidCursor) {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
if listErr != nil {
|
if listErr != nil {
|
||||||
@@ -176,8 +178,8 @@ func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
if !valid {
|
if !valid {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
chat, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID)
|
chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID)
|
||||||
if errors.Is(err, postgres.ErrNotFound) {
|
if errors.Is(err, model.ErrNotFound) {
|
||||||
return notFound()
|
return notFound()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -207,21 +209,22 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
var input struct {
|
var input struct {
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
|
AttachmentIDs []string `json:"attachment_ids"`
|
||||||
}
|
}
|
||||||
if err := readJSON(w, r, &input); err != nil {
|
if err := readJSON(w, r, &input); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !validText(input.Text, 4000) {
|
if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
message, replay, err := s.messengerStore.CreateMessage(
|
message, replay, err := s.messageRepository.CreateMessage(
|
||||||
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
|
r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL,
|
||||||
)
|
)
|
||||||
if errors.Is(err, postgres.ErrNotFound) {
|
if errors.Is(err, model.ErrNotFound) {
|
||||||
return notFound()
|
return notFound()
|
||||||
}
|
}
|
||||||
if errors.Is(err, postgres.ErrIdempotencyConflict) {
|
if errors.Is(err, model.ErrIdempotencyConflict) {
|
||||||
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
|
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -234,16 +237,42 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validMessage(text string, attachmentIDs []string) bool {
|
||||||
|
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return len(attachmentIDs) > 0 || strings.TrimSpace(text) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalAttachmentIDs(attachmentIDs []string) bool {
|
||||||
|
if len(attachmentIDs) > 10 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(attachmentIDs))
|
||||||
|
for index, attachmentID := range attachmentIDs {
|
||||||
|
canonical, valid := canonicalID(attachmentID)
|
||||||
|
if !valid {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[canonical]; duplicate {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen[canonical] = struct{}{}
|
||||||
|
attachmentIDs[index] = canonical
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
|
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
|
||||||
limit, cursor, err := pageParams(r, "message", chatID)
|
limit, cursor, err := pageParams(r, "message", chatID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
items, hasNext, listErr := s.messengerStore.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
|
items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
|
||||||
if errors.Is(listErr, postgres.ErrNotFound) {
|
if errors.Is(listErr, model.ErrNotFound) {
|
||||||
return notFound()
|
return notFound()
|
||||||
}
|
}
|
||||||
if errors.Is(listErr, postgres.ErrInvalidCursor) {
|
if errors.Is(listErr, model.ErrInvalidCursor) {
|
||||||
return invalidRequest()
|
return invalidRequest()
|
||||||
}
|
}
|
||||||
if listErr != nil {
|
if listErr != nil {
|
||||||
+21
-5
@@ -1,26 +1,42 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
type MessengerStore interface {
|
type UserRepository interface {
|
||||||
Ping(context.Context) error
|
|
||||||
CreateUser(context.Context, string, string, string) (model.User, error)
|
CreateUser(context.Context, string, string, string) (model.User, error)
|
||||||
UserByID(context.Context, string) (model.User, error)
|
UserByID(context.Context, string) (model.User, error)
|
||||||
UserByUsername(context.Context, string) (model.UserCredentials, error)
|
UserByUsername(context.Context, string) (model.UserCredentials, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChatRepository interface {
|
||||||
CreateChat(context.Context, string, []string, string) (model.Chat, error)
|
CreateChat(context.Context, string, []string, string) (model.Chat, error)
|
||||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||||
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
|
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
|
||||||
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
|
}
|
||||||
|
|
||||||
|
type MessageRepository interface {
|
||||||
|
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
|
||||||
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
|
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type SessionStore interface {
|
type AttachmentService interface {
|
||||||
|
Upload(context.Context, attachment.Upload) (model.Attachment, error)
|
||||||
|
Metadata(context.Context, string, string) (model.Attachment, error)
|
||||||
|
Content(context.Context, string, string) (attachment.Content, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type HealthChecker interface {
|
||||||
Ping(context.Context) error
|
Ping(context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type SessionStore interface {
|
||||||
|
HealthChecker
|
||||||
Create(context.Context, string, time.Duration) (string, time.Time, error)
|
Create(context.Context, string, time.Duration) (string, time.Time, error)
|
||||||
UserID(context.Context, string) (string, error)
|
UserID(context.Context, string) (string, error)
|
||||||
Delete(context.Context, string) error
|
Delete(context.Context, string) error
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -12,11 +12,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type metrics struct {
|
type metrics struct {
|
||||||
requests *prometheus.CounterVec
|
requests *prometheus.CounterVec
|
||||||
duration *prometheus.HistogramVec
|
duration *prometheus.HistogramVec
|
||||||
inFlight prometheus.Gauge
|
inFlight prometheus.Gauge
|
||||||
replays prometheus.Counter
|
replays prometheus.Counter
|
||||||
handler http.Handler
|
attachmentUploads *prometheus.CounterVec
|
||||||
|
attachmentDuration prometheus.Histogram
|
||||||
|
attachmentBytes prometheus.Counter
|
||||||
|
handler http.Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMetrics() *metrics {
|
func newMetrics() *metrics {
|
||||||
@@ -47,12 +50,31 @@ func newMetrics() *metrics {
|
|||||||
Name: "idempotency_replays_total",
|
Name: "idempotency_replays_total",
|
||||||
Help: "Message requests served from an idempotency record.",
|
Help: "Message requests served from an idempotency record.",
|
||||||
}),
|
}),
|
||||||
|
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Namespace: "messenger",
|
||||||
|
Name: "attachment_uploads_total",
|
||||||
|
Help: "Completed attachment uploads by result.",
|
||||||
|
}, []string{"result"}),
|
||||||
|
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||||
|
Namespace: "messenger",
|
||||||
|
Name: "attachment_upload_duration_seconds",
|
||||||
|
Help: "Attachment upload duration in seconds.",
|
||||||
|
Buckets: prometheus.DefBuckets,
|
||||||
|
}),
|
||||||
|
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Namespace: "messenger",
|
||||||
|
Name: "attachment_upload_bytes_total",
|
||||||
|
Help: "Bytes accepted by successful attachment uploads.",
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
registry.MustRegister(
|
registry.MustRegister(
|
||||||
result.requests,
|
result.requests,
|
||||||
result.duration,
|
result.duration,
|
||||||
result.inFlight,
|
result.inFlight,
|
||||||
result.replays,
|
result.replays,
|
||||||
|
result.attachmentUploads,
|
||||||
|
result.attachmentDuration,
|
||||||
|
result.attachmentBytes,
|
||||||
collectors.NewGoCollector(),
|
collectors.NewGoCollector(),
|
||||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||||
)
|
)
|
||||||
@@ -79,5 +101,14 @@ func routeTemplate(path string) string {
|
|||||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||||
return "/api/v1/chats/{chat_id}/messages"
|
return "/api/v1/chats/{chat_id}/messages"
|
||||||
}
|
}
|
||||||
|
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||||
|
return "/api/v1/chats/{chat_id}/attachments"
|
||||||
|
}
|
||||||
|
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||||
|
return "/api/v1/attachments/{attachment_id}"
|
||||||
|
}
|
||||||
|
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||||
|
return "/api/v1/attachments/{attachment_id}/content"
|
||||||
|
}
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
type User = model.User
|
type User = model.User
|
||||||
type Chat = model.Chat
|
type Chat = model.Chat
|
||||||
type Message = model.Message
|
type Message = model.Message
|
||||||
|
type Attachment = model.Attachment
|
||||||
|
|
||||||
func newID() (string, error) {
|
func newID() (string, error) {
|
||||||
id, err := uuid.NewRandom()
|
id, err := uuid.NewRandom()
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -33,6 +33,12 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError {
|
|||||||
handler = s.chat
|
handler = s.chat
|
||||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||||
handler = s.messages
|
handler = s.messages
|
||||||
|
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||||
|
handler = s.uploadAttachment
|
||||||
|
} else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||||
|
handler = s.attachmentMetadata
|
||||||
|
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||||
|
handler = s.attachmentContent
|
||||||
} else {
|
} else {
|
||||||
return notFound()
|
return notFound()
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -8,18 +8,42 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type Dependencies struct {
|
||||||
|
Users UserRepository
|
||||||
|
Chats ChatRepository
|
||||||
|
Messages MessageRepository
|
||||||
|
Attachments AttachmentService
|
||||||
|
Sessions SessionStore
|
||||||
|
Passwords PasswordService
|
||||||
|
Database HealthChecker
|
||||||
|
Objects HealthChecker
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
SessionTTL time.Duration
|
||||||
|
IdempotencyTTL time.Duration
|
||||||
|
RequestTTL time.Duration
|
||||||
|
InstanceID string
|
||||||
|
SwaggerPort string
|
||||||
|
}
|
||||||
|
|
||||||
type Server struct {
|
type Server struct {
|
||||||
messengerStore MessengerStore
|
userRepository UserRepository
|
||||||
sessions SessionStore
|
chatRepository ChatRepository
|
||||||
passwords PasswordService
|
messageRepository MessageRepository
|
||||||
sessionTTL time.Duration
|
attachments AttachmentService
|
||||||
idempotencyTTL time.Duration
|
sessions SessionStore
|
||||||
requestTTL time.Duration
|
passwords PasswordService
|
||||||
instanceID string
|
database HealthChecker
|
||||||
logger *slog.Logger
|
objects HealthChecker
|
||||||
swaggerPort string
|
sessionTTL time.Duration
|
||||||
metrics *metrics
|
idempotencyTTL time.Duration
|
||||||
ready atomic.Bool
|
requestTTL time.Duration
|
||||||
|
instanceID string
|
||||||
|
logger *slog.Logger
|
||||||
|
swaggerPort string
|
||||||
|
metrics *metrics
|
||||||
|
ready atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type statusWriter struct {
|
type statusWriter struct {
|
||||||
@@ -34,19 +58,16 @@ func (w *statusWriter) WriteHeader(status int) {
|
|||||||
|
|
||||||
func NewServer(
|
func NewServer(
|
||||||
logger *slog.Logger,
|
logger *slog.Logger,
|
||||||
messages MessengerStore,
|
dependencies Dependencies,
|
||||||
sessions SessionStore,
|
config Config,
|
||||||
passwords PasswordService,
|
|
||||||
sessionTTL time.Duration,
|
|
||||||
idempotencyTTL time.Duration,
|
|
||||||
requestTTL time.Duration,
|
|
||||||
instanceID string,
|
|
||||||
swaggerPort string,
|
|
||||||
) *Server {
|
) *Server {
|
||||||
s := &Server{
|
s := &Server{
|
||||||
messengerStore: messages, sessions: sessions, passwords: passwords,
|
userRepository: dependencies.Users, chatRepository: dependencies.Chats,
|
||||||
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
|
messageRepository: dependencies.Messages,
|
||||||
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
|
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
|
||||||
|
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
|
||||||
|
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
|
||||||
|
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
|
||||||
}
|
}
|
||||||
s.ready.Store(true)
|
s.ready.Store(true)
|
||||||
return s
|
return s
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package messenger
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -14,8 +14,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var errDependency = errors.New("dependency unavailable")
|
var errDependency = errors.New("dependency unavailable")
|
||||||
@@ -63,13 +63,34 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
|
|||||||
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
|
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
|
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) {
|
||||||
return s.message, s.messageReplay, s.messageErr
|
return s.message, s.messageReplay, s.messageErr
|
||||||
}
|
}
|
||||||
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
|
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type objectStoreStub struct {
|
||||||
|
pingErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
|
||||||
|
|
||||||
|
type attachmentServiceStub struct {
|
||||||
|
uploaded model.Attachment
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) {
|
||||||
|
return s.uploaded, s.err
|
||||||
|
}
|
||||||
|
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
|
||||||
|
return s.uploaded, s.err
|
||||||
|
}
|
||||||
|
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
|
||||||
|
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
|
||||||
|
}
|
||||||
|
|
||||||
type sessionStub struct {
|
type sessionStub struct {
|
||||||
pingErr error
|
pingErr error
|
||||||
userID string
|
userID string
|
||||||
@@ -100,14 +121,16 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
|
|||||||
name string
|
name string
|
||||||
store messengerStoreStub
|
store messengerStoreStub
|
||||||
sessions sessionStub
|
sessions sessionStub
|
||||||
|
objects objectStoreStub
|
||||||
}{
|
}{
|
||||||
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
|
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
|
||||||
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
|
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
|
||||||
|
{name: "s3", objects: objectStoreStub{pingErr: errDependency}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
|
server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{})
|
||||||
response := serveRequest(server, http.MethodGet, "/health/ready", "")
|
response := serveRequest(server, http.MethodGet, "/health/ready", "")
|
||||||
if response.Code != http.StatusServiceUnavailable {
|
if response.Code != http.StatusServiceUnavailable {
|
||||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
|
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
|
||||||
@@ -160,6 +183,18 @@ func (s *MessengerSuite) TestAuthenticationChallenges() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() {
|
||||||
|
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||||
|
response := serveRequest(
|
||||||
|
server,
|
||||||
|
http.MethodGet,
|
||||||
|
"/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
s.Equal(http.StatusUnauthorized, response.Code)
|
||||||
|
s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate"))
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
|
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
|
||||||
t := s.T()
|
t := s.T()
|
||||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||||
@@ -228,7 +263,7 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
|
|||||||
t := s.T()
|
t := s.T()
|
||||||
store := messengerStoreStub{
|
store := messengerStoreStub{
|
||||||
userByID: model.User{ID: "user-id"},
|
userByID: model.User{ID: "user-id"},
|
||||||
messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict),
|
messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict),
|
||||||
}
|
}
|
||||||
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
|
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
|
||||||
@@ -242,8 +277,49 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
|
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
|
||||||
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
|
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
|
||||||
|
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
|
||||||
|
s.False(valid, filename)
|
||||||
|
}
|
||||||
|
filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}})
|
||||||
|
s.True(valid)
|
||||||
|
s.Equal("фото.png", filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MessengerSuite) TestUploadRejectsMalformedQuery() {
|
||||||
|
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
|
||||||
|
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
|
||||||
|
request := httptest.NewRequest(
|
||||||
|
http.MethodPost,
|
||||||
|
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG",
|
||||||
|
strings.NewReader("image"),
|
||||||
|
)
|
||||||
|
request.Header.Set("Authorization", "Bearer valid-token")
|
||||||
|
request.Header.Set("Content-Type", "image/png")
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
server.Handler().ServeHTTP(response, request)
|
||||||
|
s.Equal(http.StatusBadRequest, response.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server {
|
||||||
|
return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MessengerSuite) server(
|
||||||
|
store messengerStoreStub,
|
||||||
|
sessions SessionStore,
|
||||||
|
passwords PasswordService,
|
||||||
|
objects objectStoreStub,
|
||||||
|
attachments AttachmentService,
|
||||||
|
) *Server {
|
||||||
|
return NewServer(s.logger, Dependencies{
|
||||||
|
Users: store, Chats: store, Messages: store, Attachments: attachments,
|
||||||
|
Sessions: sessions, Passwords: passwords, Database: store, Objects: objects,
|
||||||
|
}, Config{
|
||||||
|
SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second,
|
||||||
|
InstanceID: "test-instance", SwaggerPort: "8081",
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
|
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrNotFound = errors.New("not found")
|
||||||
|
ErrConflict = errors.New("conflict")
|
||||||
|
ErrInvalidMembers = errors.New("invalid members")
|
||||||
|
ErrInvalidCursor = errors.New("invalid cursor")
|
||||||
|
ErrIdempotencyConflict = errors.New("idempotency conflict")
|
||||||
|
ErrInvalidToken = errors.New("invalid token")
|
||||||
|
ErrSessionNotFound = errors.New("session not found")
|
||||||
|
ErrInvalidInput = errors.New("invalid input")
|
||||||
|
ErrUnsupportedMedia = errors.New("unsupported media")
|
||||||
|
ErrPayloadTooLarge = errors.New("payload too large")
|
||||||
|
ErrCapacity = errors.New("capacity exhausted")
|
||||||
|
ErrUnavailable = errors.New("dependency unavailable")
|
||||||
|
)
|
||||||
+82
-11
@@ -5,6 +5,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
AttachmentPending = "pending"
|
||||||
|
AttachmentReady = "ready"
|
||||||
|
AttachmentDeleting = "deleting"
|
||||||
|
)
|
||||||
|
|
||||||
const timestampLayout = "2006-01-02T15:04:05.000000Z"
|
const timestampLayout = "2006-01-02T15:04:05.000000Z"
|
||||||
|
|
||||||
type User struct {
|
type User struct {
|
||||||
@@ -27,11 +33,45 @@ type Chat struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Message struct {
|
type Message struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
ChatID string `json:"chat_id"`
|
ChatID string `json:"chat_id"`
|
||||||
SenderID string `json:"sender_id"`
|
SenderID string `json:"sender_id"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
AttachmentIDs []string `json:"attachment_ids"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Attachment struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
ChatID string `json:"chat_id"`
|
||||||
|
OwnerID string `json:"owner_id"`
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Variants []Variant `json:"variants"`
|
||||||
|
ErrorCode *string `json:"error_code"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
ObjectKey string `json:"-"`
|
||||||
|
ReadyAt *time.Time `json:"-"`
|
||||||
|
Referenced bool `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type StoredObject struct {
|
||||||
|
Key string
|
||||||
|
Size int64
|
||||||
|
LastModified time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type Variant struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Width int `json:"width"`
|
||||||
|
Height int `json:"height"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Position struct {
|
type Position struct {
|
||||||
@@ -54,11 +94,27 @@ type chatJSON struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type messageJSON struct {
|
type messageJSON struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
ChatID string `json:"chat_id"`
|
ChatID string `json:"chat_id"`
|
||||||
SenderID string `json:"sender_id"`
|
SenderID string `json:"sender_id"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
CreatedAt string `json:"created_at"`
|
CreatedAt string `json:"created_at"`
|
||||||
|
AttachmentIDs []string `json:"attachment_ids"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type attachmentJSON struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
ChatID string `json:"chat_id"`
|
||||||
|
OwnerID string `json:"owner_id"`
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Variants []Variant `json:"variants"`
|
||||||
|
ErrorCode *string `json:"error_code"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
UpdatedAt string `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u User) MarshalJSON() ([]byte, error) {
|
func (u User) MarshalJSON() ([]byte, error) {
|
||||||
@@ -70,7 +126,22 @@ func (c Chat) MarshalJSON() ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m Message) MarshalJSON() ([]byte, error) {
|
func (m Message) MarshalJSON() ([]byte, error) {
|
||||||
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt)})
|
attachments := m.AttachmentIDs
|
||||||
|
if attachments == nil {
|
||||||
|
attachments = []string{}
|
||||||
|
}
|
||||||
|
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a Attachment) MarshalJSON() ([]byte, error) {
|
||||||
|
variants := a.Variants
|
||||||
|
if variants == nil {
|
||||||
|
variants = []Variant{}
|
||||||
|
}
|
||||||
|
return json.Marshal(attachmentJSON{
|
||||||
|
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
|
||||||
|
a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func FormatTime(value time.Time) string {
|
func FormatTime(value time.Time) string {
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
package objectstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrNotFound = errors.New("object not found")
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Endpoint string
|
||||||
|
Region string
|
||||||
|
Bucket string
|
||||||
|
AccessKey string
|
||||||
|
SecretKey string
|
||||||
|
UseTLS bool
|
||||||
|
Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type Store struct {
|
||||||
|
client *minio.Client
|
||||||
|
bucket string
|
||||||
|
timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(config Config) (*Store, error) {
|
||||||
|
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
|
||||||
|
return nil, errors.New("invalid object store configuration")
|
||||||
|
}
|
||||||
|
client, err := minio.New(endpoint, &minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
|
||||||
|
Secure: config.UseTLS,
|
||||||
|
Region: config.Region,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create S3 client: %w", err)
|
||||||
|
}
|
||||||
|
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func endpointHost(value string, useTLS bool) (string, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if !strings.Contains(value, "://") {
|
||||||
|
if value == "" {
|
||||||
|
return "", errors.New("S3 endpoint is required")
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(value)
|
||||||
|
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
|
||||||
|
return "", errors.New("invalid S3 endpoint")
|
||||||
|
}
|
||||||
|
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||||
|
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
|
||||||
|
}
|
||||||
|
return parsed.Host, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Ping(ctx context.Context) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
defer cancel()
|
||||||
|
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("check S3 bucket: %w", err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return errors.New("S3 bucket does not exist")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
defer cancel()
|
||||||
|
_, err := s.client.PutObject(
|
||||||
|
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("put S3 object: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
return nil, objectError("get S3 object", err)
|
||||||
|
}
|
||||||
|
if _, err := object.Stat(); err != nil {
|
||||||
|
cancel()
|
||||||
|
_ = object.Close()
|
||||||
|
return nil, objectError("stat S3 object", err)
|
||||||
|
}
|
||||||
|
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
defer cancel()
|
||||||
|
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return model.StoredObject{}, objectError("stat S3 object", err)
|
||||||
|
}
|
||||||
|
return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||||
|
return objectError("delete S3 object", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
|
||||||
|
if limit < 1 || limit > 1000 {
|
||||||
|
return nil, "", errors.New("object list limit must be between 1 and 1000")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||||
|
defer cancel()
|
||||||
|
objects := make([]model.StoredObject, 0, limit+1)
|
||||||
|
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
|
||||||
|
for info := range s.client.ListObjects(ctx, s.bucket, options) {
|
||||||
|
if info.Err != nil {
|
||||||
|
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
|
||||||
|
}
|
||||||
|
objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
|
||||||
|
if len(objects) == limit+1 {
|
||||||
|
cancel()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var next string
|
||||||
|
if len(objects) > limit {
|
||||||
|
objects = objects[:limit]
|
||||||
|
next = objects[len(objects)-1].Key
|
||||||
|
}
|
||||||
|
return objects, next, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type timeoutReader struct {
|
||||||
|
io.ReadCloser
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *timeoutReader) Close() error {
|
||||||
|
err := r.ReadCloser.Close()
|
||||||
|
r.cancel()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectError(action string, err error) error {
|
||||||
|
response := minio.ToErrorResponse(err)
|
||||||
|
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
|
||||||
|
return ErrNotFound
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s: %w", action, err)
|
||||||
|
}
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
const attachmentColumns = `
|
||||||
|
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||||
|
object_key, status, created_at, updated_at, ready_at`
|
||||||
|
|
||||||
|
type rowScanner interface {
|
||||||
|
Scan(...any) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
|
||||||
|
query := `
|
||||||
|
INSERT INTO attachments (
|
||||||
|
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at
|
||||||
|
)
|
||||||
|
SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
|
||||||
|
)
|
||||||
|
RETURNING ` + attachmentColumns
|
||||||
|
|
||||||
|
err := scanAttachment(s.pool.QueryRow(
|
||||||
|
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
|
||||||
|
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
|
||||||
|
), &attachment)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return model.Attachment{}, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err)
|
||||||
|
}
|
||||||
|
return publicAttachment(attachment), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||||
|
query := `
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||||
|
WHERE id = $1 AND object_key = $2 AND status = 'pending'
|
||||||
|
RETURNING ` + attachmentColumns
|
||||||
|
|
||||||
|
var attachment model.Attachment
|
||||||
|
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return model.Attachment{}, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
|
||||||
|
}
|
||||||
|
return publicAttachment(attachment), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error {
|
||||||
|
const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'`
|
||||||
|
if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil {
|
||||||
|
return fmt.Errorf("delete pending attachment: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
|
||||||
|
query := `
|
||||||
|
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||||
|
FROM attachments a
|
||||||
|
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
|
||||||
|
WHERE a.id = $1 AND a.status = 'ready'`
|
||||||
|
|
||||||
|
var attachment model.Attachment
|
||||||
|
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return model.Attachment{}, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
|
||||||
|
}
|
||||||
|
return publicAttachment(attachment), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||||
|
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
|
||||||
|
|
||||||
|
var attachment model.Attachment
|
||||||
|
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return model.Attachment{}, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
|
||||||
|
}
|
||||||
|
return publicAttachment(attachment), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||||
|
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
|
||||||
|
var exists bool
|
||||||
|
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
|
||||||
|
return false, fmt.Errorf("check attachment by object key: %w", err)
|
||||||
|
}
|
||||||
|
return exists, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
|
||||||
|
if limit < 1 || limit > 1000 {
|
||||||
|
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
|
||||||
|
}
|
||||||
|
query := `
|
||||||
|
SELECT ` + attachmentColumnsWithAlias("a") + `,
|
||||||
|
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
|
||||||
|
FROM attachments a
|
||||||
|
WHERE a.object_key > $1
|
||||||
|
ORDER BY a.object_key
|
||||||
|
LIMIT $2`
|
||||||
|
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("list attachments: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
|
||||||
|
var attachment model.Attachment
|
||||||
|
err := scanAttachmentWithReference(row, &attachment)
|
||||||
|
return publicAttachment(attachment), err
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("scan attachments: %w", err)
|
||||||
|
}
|
||||||
|
var next string
|
||||||
|
if len(attachments) > limit {
|
||||||
|
attachments = attachments[:limit]
|
||||||
|
next = attachments[len(attachments)-1].ObjectKey
|
||||||
|
}
|
||||||
|
return attachments, next, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
|
||||||
|
return row.Scan(
|
||||||
|
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||||
|
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||||
|
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||||
|
&attachment.Referenced,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
|
||||||
|
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
|
||||||
|
var referenced bool
|
||||||
|
if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil {
|
||||||
|
return false, fmt.Errorf("check attachment references: %w", err)
|
||||||
|
}
|
||||||
|
return referenced, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) {
|
||||||
|
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||||
|
|
||||||
|
var status string
|
||||||
|
var createdAt time.Time
|
||||||
|
var readyAt *time.Time
|
||||||
|
err = tx.QueryRow(
|
||||||
|
ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID,
|
||||||
|
).Scan(&status, &createdAt, &readyAt)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
|
||||||
|
}
|
||||||
|
if status != candidate.Status {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var age time.Time
|
||||||
|
switch status {
|
||||||
|
case model.AttachmentPending:
|
||||||
|
age = createdAt
|
||||||
|
case model.AttachmentReady:
|
||||||
|
if readyAt == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
age = *readyAt
|
||||||
|
default:
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if age.After(createdBefore) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var referenced bool
|
||||||
|
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil {
|
||||||
|
return false, fmt.Errorf("recheck attachment references: %w", err)
|
||||||
|
}
|
||||||
|
if referenced {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(
|
||||||
|
ctx,
|
||||||
|
`UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`,
|
||||||
|
candidate.ID, candidate.Status,
|
||||||
|
); err != nil {
|
||||||
|
return false, fmt.Errorf("mark attachment deleting: %w", err)
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return false, fmt.Errorf("commit attachment deletion claim: %w", err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) {
|
||||||
|
const query = `
|
||||||
|
DELETE FROM attachments a
|
||||||
|
WHERE a.id = $1
|
||||||
|
AND a.status = 'deleting'
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
|
||||||
|
result, err := s.pool.Exec(ctx, query, attachmentID)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("delete claimed attachment: %w", err)
|
||||||
|
}
|
||||||
|
return result.RowsAffected() == 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
|
||||||
|
return row.Scan(
|
||||||
|
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||||
|
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||||
|
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentColumnsWithAlias(alias string) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
|
||||||
|
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
|
||||||
|
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
|
||||||
|
alias,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicAttachment(attachment model.Attachment) model.Attachment {
|
||||||
|
attachment.Variants = []model.Variant{}
|
||||||
|
return attachment
|
||||||
|
}
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
package postgres
|
package postgres
|
||||||
|
|
||||||
import "errors"
|
import "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrConflict = errors.New("conflict")
|
ErrConflict = model.ErrConflict
|
||||||
ErrNotFound = errors.New("not found")
|
ErrNotFound = model.ErrNotFound
|
||||||
ErrInvalidMembers = errors.New("invalid members")
|
ErrInvalidMembers = model.ErrInvalidMembers
|
||||||
ErrInvalidCursor = errors.New("invalid cursor")
|
ErrInvalidCursor = model.ErrInvalidCursor
|
||||||
ErrIdempotencyConflict = errors.New("idempotency conflict")
|
ErrIdempotencyConflict = model.ErrIdempotencyConflict
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
@@ -21,9 +22,13 @@ func (s *Store) CreateMessage(
|
|||||||
chatID string,
|
chatID string,
|
||||||
senderID string,
|
senderID string,
|
||||||
text string,
|
text string,
|
||||||
|
attachmentIDs []string,
|
||||||
idempotencyKey string,
|
idempotencyKey string,
|
||||||
idempotencyTTL time.Duration,
|
idempotencyTTL time.Duration,
|
||||||
) (model.Message, bool, error) {
|
) (model.Message, bool, error) {
|
||||||
|
if attachmentIDs == nil {
|
||||||
|
attachmentIDs = []string{}
|
||||||
|
}
|
||||||
var (
|
var (
|
||||||
tx pgx.Tx
|
tx pgx.Tx
|
||||||
err error
|
err error
|
||||||
@@ -46,13 +51,23 @@ func (s *Store) CreateMessage(
|
|||||||
if !visible {
|
if !visible {
|
||||||
return model.Message{}, false, ErrNotFound
|
return model.Message{}, false, ErrNotFound
|
||||||
}
|
}
|
||||||
|
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||||
|
if err != nil {
|
||||||
|
return model.Message{}, false, err
|
||||||
|
}
|
||||||
|
if !validAttachments {
|
||||||
|
return model.Message{}, false, ErrNotFound
|
||||||
|
}
|
||||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||||
return model.Message{}, false, err
|
return model.Message{}, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
requestHash = sha256.Sum256([]byte(text))
|
requestHash, err = messageRequestHash(text, attachmentIDs)
|
||||||
|
if err != nil {
|
||||||
|
return model.Message{}, false, err
|
||||||
|
}
|
||||||
message, replay, err = createIdempotentMessage(
|
message, replay, err = createIdempotentMessage(
|
||||||
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
|
ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return model.Message{}, false, err
|
return model.Message{}, false, err
|
||||||
@@ -63,6 +78,44 @@ func (s *Store) CreateMessage(
|
|||||||
return message, replay, nil
|
return message, replay, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) {
|
||||||
|
if attachmentIDs == nil {
|
||||||
|
attachmentIDs = []string{}
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
AttachmentIDs []string `json:"attachment_ids"`
|
||||||
|
}{text, attachmentIDs})
|
||||||
|
if err != nil {
|
||||||
|
return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err)
|
||||||
|
}
|
||||||
|
return sha256.Sum256(payload), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
|
||||||
|
if len(attachmentIDs) == 0 {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
const query = `
|
||||||
|
SELECT id
|
||||||
|
FROM attachments
|
||||||
|
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
|
||||||
|
FOR UPDATE`
|
||||||
|
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
count := 0
|
||||||
|
for rows.Next() {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||||
|
}
|
||||||
|
return count == len(attachmentIDs), nil
|
||||||
|
}
|
||||||
|
|
||||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||||
const query = `
|
const query = `
|
||||||
SELECT EXISTS (
|
SELECT EXISTS (
|
||||||
@@ -84,6 +137,7 @@ func createIdempotentMessage(
|
|||||||
chatID string,
|
chatID string,
|
||||||
senderID string,
|
senderID string,
|
||||||
text string,
|
text string,
|
||||||
|
attachmentIDs []string,
|
||||||
idempotencyKey string,
|
idempotencyKey string,
|
||||||
requestHash []byte,
|
requestHash []byte,
|
||||||
idempotencyTTL time.Duration,
|
idempotencyTTL time.Duration,
|
||||||
@@ -126,7 +180,7 @@ func createIdempotentMessage(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
|
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs)
|
||||||
return message, false, err
|
return message, false, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -235,6 +289,7 @@ func insertMessage(
|
|||||||
chatID string,
|
chatID string,
|
||||||
senderID string,
|
senderID string,
|
||||||
text string,
|
text string,
|
||||||
|
attachmentIDs []string,
|
||||||
) (model.Message, error) {
|
) (model.Message, error) {
|
||||||
const query = `
|
const query = `
|
||||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||||
@@ -255,14 +310,37 @@ func insertMessage(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return model.Message{}, fmt.Errorf("insert message: %w", err)
|
return model.Message{}, fmt.Errorf("insert message: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil {
|
||||||
|
return model.Message{}, err
|
||||||
|
}
|
||||||
|
message.AttachmentIDs = make([]string, len(attachmentIDs))
|
||||||
|
copy(message.AttachmentIDs, attachmentIDs)
|
||||||
return message, nil
|
return message, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error {
|
||||||
|
const query = `
|
||||||
|
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||||
|
VALUES ($1, $2, $3, $4)`
|
||||||
|
for position, attachmentID := range attachmentIDs {
|
||||||
|
if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil {
|
||||||
|
return fmt.Errorf("insert message attachment: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
|
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
|
||||||
const query = `
|
const query = `
|
||||||
SELECT id, chat_id, sender_id, text, created_at
|
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||||
FROM messages
|
ARRAY(
|
||||||
WHERE id = $1`
|
SELECT ma.attachment_id::text
|
||||||
|
FROM message_attachments ma
|
||||||
|
WHERE ma.message_id = m.id
|
||||||
|
ORDER BY ma.position
|
||||||
|
)
|
||||||
|
FROM messages m
|
||||||
|
WHERE m.id = $1`
|
||||||
|
|
||||||
var (
|
var (
|
||||||
message model.Message
|
message model.Message
|
||||||
@@ -274,6 +352,7 @@ func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Messag
|
|||||||
&message.SenderID,
|
&message.SenderID,
|
||||||
&message.Text,
|
&message.Text,
|
||||||
&message.CreatedAt,
|
&message.CreatedAt,
|
||||||
|
&message.AttachmentIDs,
|
||||||
)
|
)
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return model.Message{}, fmt.Errorf("idempotency result message not found")
|
return model.Message{}, fmt.Errorf("idempotency result message not found")
|
||||||
|
|||||||
@@ -344,11 +344,17 @@ func (s *Store) ListMessages(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const query = `
|
const query = `
|
||||||
SELECT id, chat_id, sender_id, text, created_at
|
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||||
FROM messages
|
ARRAY(
|
||||||
WHERE chat_id = $1
|
SELECT ma.attachment_id::text
|
||||||
AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid))
|
FROM message_attachments ma
|
||||||
ORDER BY created_at, id
|
WHERE ma.message_id = m.id
|
||||||
|
ORDER BY ma.position
|
||||||
|
)
|
||||||
|
FROM messages m
|
||||||
|
WHERE m.chat_id = $1
|
||||||
|
AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid))
|
||||||
|
ORDER BY m.created_at, m.id
|
||||||
LIMIT $4`
|
LIMIT $4`
|
||||||
|
|
||||||
var afterTime any
|
var afterTime any
|
||||||
@@ -372,6 +378,7 @@ func (s *Store) ListMessages(
|
|||||||
&message.SenderID,
|
&message.SenderID,
|
||||||
&message.Text,
|
&message.Text,
|
||||||
&message.CreatedAt,
|
&message.CreatedAt,
|
||||||
|
&message.AttachmentIDs,
|
||||||
)
|
)
|
||||||
return message, err
|
return message, err
|
||||||
}))
|
}))
|
||||||
|
|||||||
@@ -100,18 +100,16 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
|||||||
owner, chatID := createOwnerAndChat(t, store)
|
owner, chatID := createOwnerAndChat(t, store)
|
||||||
key := uuid.NewString()
|
key := uuid.NewString()
|
||||||
|
|
||||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour)
|
||||||
if err != nil || replay {
|
if err != nil || replay {
|
||||||
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
|
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
|
||||||
}
|
}
|
||||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour)
|
||||||
if err != nil || !replay {
|
if err != nil || !replay {
|
||||||
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
|
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
|
||||||
}
|
}
|
||||||
if first != second {
|
s.Equal(first, second)
|
||||||
t.Fatalf("replay message = %v, want %v", second, first)
|
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||||
}
|
|
||||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
|
||||||
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
|
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
|
||||||
}
|
}
|
||||||
if count := messageCount(t, pool, chatID); count != 1 {
|
if count := messageCount(t, pool, chatID); count != 1 {
|
||||||
@@ -119,6 +117,99 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||||
|
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||||
|
outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash")
|
||||||
|
s.Require().NoError(err)
|
||||||
|
attachment := model.Attachment{
|
||||||
|
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||||
|
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
|
||||||
|
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||||
|
}
|
||||||
|
pending, err := s.store.CreatePendingAttachment(context.Background(), attachment)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal("ready", created.Status)
|
||||||
|
s.Empty(created.Variants)
|
||||||
|
s.Nil(created.ErrorCode)
|
||||||
|
|
||||||
|
visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(created, visible)
|
||||||
|
_, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID)
|
||||||
|
s.ErrorIs(err, postgres.ErrNotFound)
|
||||||
|
|
||||||
|
message, replay, err := s.store.CreateMessage(
|
||||||
|
context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour,
|
||||||
|
)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.False(replay)
|
||||||
|
s.Equal([]string{created.ID}, message.AttachmentIDs)
|
||||||
|
|
||||||
|
messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Len(messages, 1)
|
||||||
|
s.Equal([]string{created.ID}, messages[0].AttachmentIDs)
|
||||||
|
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.True(referenced)
|
||||||
|
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour))
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.False(deleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
|
||||||
|
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||||
|
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||||
|
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png",
|
||||||
|
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64),
|
||||||
|
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||||
|
})
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.True(claimed)
|
||||||
|
deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.True(deleted)
|
||||||
|
|
||||||
|
pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||||
|
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png",
|
||||||
|
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64),
|
||||||
|
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||||
|
})
|
||||||
|
s.Require().NoError(err)
|
||||||
|
ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.False(claimed)
|
||||||
|
visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(model.AttachmentReady, visible.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
|
||||||
|
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
|
||||||
|
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||||
|
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||||
|
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
|
||||||
|
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||||
|
})
|
||||||
|
s.Require().NoError(err)
|
||||||
|
attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
_, _, err = s.store.CreateMessage(
|
||||||
|
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||||
|
)
|
||||||
|
s.ErrorIs(err, postgres.ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||||
t := s.T()
|
t := s.T()
|
||||||
pool := s.pool
|
pool := s.pool
|
||||||
@@ -139,7 +230,7 @@ func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
|||||||
go func() {
|
go func() {
|
||||||
defer group.Done()
|
defer group.Done()
|
||||||
message, replay, err := stores[worker%len(stores)].CreateMessage(
|
message, replay, err := stores[worker%len(stores)].CreateMessage(
|
||||||
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
|
context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour,
|
||||||
)
|
)
|
||||||
results <- result{message: message, replay: replay, err: err}
|
results <- result{message: message, replay: replay, err: err}
|
||||||
}()
|
}()
|
||||||
@@ -186,13 +277,13 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
|||||||
}
|
}
|
||||||
key := uuid.NewString()
|
key := uuid.NewString()
|
||||||
|
|
||||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
|
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil {
|
||||||
t.Fatalf("owner CreateMessage() error = %v", err)
|
t.Fatalf("owner CreateMessage() error = %v", err)
|
||||||
}
|
}
|
||||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
|
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil {
|
||||||
t.Fatalf("second user CreateMessage() error = %v", err)
|
t.Fatalf("second user CreateMessage() error = %v", err)
|
||||||
}
|
}
|
||||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
|
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil {
|
||||||
t.Fatalf("other chat CreateMessage() error = %v", err)
|
t.Fatalf("other chat CreateMessage() error = %v", err)
|
||||||
}
|
}
|
||||||
if _, err := pool.Exec(context.Background(), `
|
if _, err := pool.Exec(context.Background(), `
|
||||||
@@ -202,7 +293,7 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
|||||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
|
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
|
||||||
t.Fatalf("expire idempotency key: %v", err)
|
t.Fatalf("expire idempotency key: %v", err)
|
||||||
}
|
}
|
||||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
|
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour)
|
||||||
if err != nil || replay || message.Text != "after expiry" {
|
if err != nil || replay || message.Text != "after expiry" {
|
||||||
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
|
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
|
||||||
}
|
}
|
||||||
@@ -212,7 +303,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
|||||||
t := s.T()
|
t := s.T()
|
||||||
owner, chatID := createOwnerAndChat(t, s.store)
|
owner, chatID := createOwnerAndChat(t, s.store)
|
||||||
expiredKey := uuid.NewString()
|
expiredKey := uuid.NewString()
|
||||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
|
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil {
|
||||||
t.Fatalf("create expired message: %v", err)
|
t.Fatalf("create expired message: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := s.pool.Exec(context.Background(), `
|
if _, err := s.pool.Exec(context.Background(), `
|
||||||
@@ -223,7 +314,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
|||||||
t.Fatalf("expire idempotency key: %v", err)
|
t.Fatalf("expire idempotency key: %v", err)
|
||||||
}
|
}
|
||||||
if _, _, err := s.store.CreateMessage(
|
if _, _, err := s.store.CreateMessage(
|
||||||
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
|
context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
t.Fatalf("create new message: %v", err)
|
t.Fatalf("create new message: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,350 @@
|
|||||||
|
package reconcile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MetadataStore interface {
|
||||||
|
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
|
||||||
|
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
|
||||||
|
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
|
||||||
|
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ObjectStore interface {
|
||||||
|
List(context.Context, string, string, int) ([]model.StoredObject, string, error)
|
||||||
|
Delete(context.Context, string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Prefix string
|
||||||
|
PageSize int
|
||||||
|
GraceTime time.Duration
|
||||||
|
UnreferencedRetention time.Duration
|
||||||
|
Apply bool
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type Report struct {
|
||||||
|
MetadataScanned int `json:"metadata_scanned"`
|
||||||
|
ObjectsScanned int `json:"objects_scanned"`
|
||||||
|
Consistent int `json:"consistent"`
|
||||||
|
FreshPendingAttachments int `json:"fresh_pending_attachments"`
|
||||||
|
OldPendingAttachments int `json:"old_pending_attachments"`
|
||||||
|
DeletingAttachments int `json:"deleting_attachments"`
|
||||||
|
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
|
||||||
|
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
|
||||||
|
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
|
||||||
|
FreshOrphanObjects int `json:"fresh_orphan_objects"`
|
||||||
|
OldOrphanObjects int `json:"old_orphan_objects"`
|
||||||
|
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
|
||||||
|
FreshMissingObjects int `json:"fresh_missing_objects"`
|
||||||
|
ReferencedMissingObjects int `json:"referenced_missing_objects"`
|
||||||
|
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
|
||||||
|
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Reconciler struct {
|
||||||
|
metadata MetadataStore
|
||||||
|
objects ObjectStore
|
||||||
|
config Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
|
||||||
|
if config.PageSize < 1 || config.PageSize > 1000 {
|
||||||
|
return nil, errors.New("page size must be between 1 and 1000")
|
||||||
|
}
|
||||||
|
if config.GraceTime < 0 {
|
||||||
|
return nil, errors.New("grace time must not be negative")
|
||||||
|
}
|
||||||
|
if config.UnreferencedRetention <= 0 {
|
||||||
|
return nil, errors.New("unreferenced retention must be positive")
|
||||||
|
}
|
||||||
|
if config.Now == nil {
|
||||||
|
config.Now = time.Now
|
||||||
|
}
|
||||||
|
return &Reconciler{metadata: metadata, objects: objects, config: config}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) Run(ctx context.Context) (Report, error) {
|
||||||
|
metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize)
|
||||||
|
if err != nil {
|
||||||
|
return Report{}, err
|
||||||
|
}
|
||||||
|
objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize)
|
||||||
|
if err != nil {
|
||||||
|
return Report{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
now := r.config.Now()
|
||||||
|
anomalyCutoff := now.Add(-r.config.GraceTime)
|
||||||
|
retentionCutoff := now.Add(-r.config.UnreferencedRetention)
|
||||||
|
var report Report
|
||||||
|
for metadata.current != nil || objects.current != nil {
|
||||||
|
switch {
|
||||||
|
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
|
||||||
|
if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
if err := objects.advance(ctx); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
|
||||||
|
if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
if err := metadata.advance(ctx); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
if err := metadata.advance(ctx); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
if err := objects.advance(ctx); err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) handlePair(
|
||||||
|
ctx context.Context,
|
||||||
|
metadata model.Attachment,
|
||||||
|
retentionCutoff time.Time,
|
||||||
|
anomalyCutoff time.Time,
|
||||||
|
report *Report,
|
||||||
|
) error {
|
||||||
|
report.MetadataScanned++
|
||||||
|
report.ObjectsScanned++
|
||||||
|
switch metadata.Status {
|
||||||
|
case model.AttachmentPending:
|
||||||
|
if metadata.CreatedAt.After(anomalyCutoff) {
|
||||||
|
report.FreshPendingAttachments++
|
||||||
|
report.Consistent++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
report.OldPendingAttachments++
|
||||||
|
return r.deletePair(ctx, metadata, anomalyCutoff, report)
|
||||||
|
case model.AttachmentDeleting:
|
||||||
|
report.DeletingAttachments++
|
||||||
|
return r.deleteClaimedPair(ctx, metadata, report)
|
||||||
|
case model.AttachmentReady:
|
||||||
|
if metadata.Referenced {
|
||||||
|
report.Consistent++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
age := metadata.CreatedAt
|
||||||
|
if metadata.ReadyAt != nil {
|
||||||
|
age = *metadata.ReadyAt
|
||||||
|
}
|
||||||
|
if age.After(retentionCutoff) {
|
||||||
|
report.FreshUnreferencedAttachments++
|
||||||
|
report.Consistent++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
report.OldUnreferencedAttachments++
|
||||||
|
return r.deletePair(ctx, metadata, retentionCutoff, report)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error {
|
||||||
|
if !r.config.Apply {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
|
||||||
|
}
|
||||||
|
if !claimed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return r.deleteClaimedPair(ctx, attachment, report)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error {
|
||||||
|
if !r.config.Apply {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
|
||||||
|
return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err)
|
||||||
|
}
|
||||||
|
report.DeletedAttachmentObjects++
|
||||||
|
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err)
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
report.DeletedUnreferencedRows++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) handleOrphanObject(
|
||||||
|
ctx context.Context,
|
||||||
|
object model.StoredObject,
|
||||||
|
cutoff time.Time,
|
||||||
|
report *Report,
|
||||||
|
) error {
|
||||||
|
report.ObjectsScanned++
|
||||||
|
if object.LastModified.After(cutoff) {
|
||||||
|
report.FreshOrphanObjects++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
report.OldOrphanObjects++
|
||||||
|
if !r.config.Apply {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := r.objects.Delete(ctx, object.Key); err != nil {
|
||||||
|
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
|
||||||
|
}
|
||||||
|
report.DeletedOrphanObjects++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) handleMissingObject(
|
||||||
|
ctx context.Context,
|
||||||
|
metadata model.Attachment,
|
||||||
|
cutoff time.Time,
|
||||||
|
report *Report,
|
||||||
|
) error {
|
||||||
|
report.MetadataScanned++
|
||||||
|
if metadata.Status == model.AttachmentDeleting {
|
||||||
|
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||||
|
}
|
||||||
|
if metadata.CreatedAt.After(cutoff) {
|
||||||
|
report.FreshMissingObjects++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if metadata.Referenced {
|
||||||
|
report.ReferencedMissingObjects++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
report.OldUnreferencedMetadata++
|
||||||
|
if !r.config.Apply {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
|
||||||
|
}
|
||||||
|
if !claimed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error {
|
||||||
|
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
report.DeletedUnreferencedRows++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type metadataIterator struct {
|
||||||
|
store MetadataStore
|
||||||
|
pageSize int
|
||||||
|
after string
|
||||||
|
page []model.Attachment
|
||||||
|
index int
|
||||||
|
current *model.Attachment
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) {
|
||||||
|
iterator := &metadataIterator{store: store, pageSize: pageSize}
|
||||||
|
if err := iterator.advance(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return iterator, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *metadataIterator) advance(ctx context.Context) error {
|
||||||
|
for {
|
||||||
|
if i.index < len(i.page) {
|
||||||
|
i.current = &i.page[i.index]
|
||||||
|
i.index++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if i.done {
|
||||||
|
i.current = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list attachment metadata: %w", err)
|
||||||
|
}
|
||||||
|
i.page = page
|
||||||
|
i.index = 0
|
||||||
|
i.after = next
|
||||||
|
i.done = next == ""
|
||||||
|
if len(page) == 0 && !i.done {
|
||||||
|
return errors.New("metadata page is empty but has continuation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectIterator struct {
|
||||||
|
store ObjectStore
|
||||||
|
prefix string
|
||||||
|
pageSize int
|
||||||
|
after string
|
||||||
|
page []model.StoredObject
|
||||||
|
index int
|
||||||
|
current *model.StoredObject
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) {
|
||||||
|
iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize}
|
||||||
|
if err := iterator.advance(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return iterator, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *objectIterator) advance(ctx context.Context) error {
|
||||||
|
for {
|
||||||
|
if i.index < len(i.page) {
|
||||||
|
i.current = &i.page[i.index]
|
||||||
|
i.index++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if i.done {
|
||||||
|
i.current = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list objects: %w", err)
|
||||||
|
}
|
||||||
|
i.page = page
|
||||||
|
i.index = 0
|
||||||
|
i.after = next
|
||||||
|
i.done = next == ""
|
||||||
|
if len(page) == 0 && !i.done {
|
||||||
|
return errors.New("object page is empty but has continuation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package reconcile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||||
|
old := now.Add(-48 * time.Hour)
|
||||||
|
fresh := now.Add(-10 * time.Minute)
|
||||||
|
metadata := &metadataStub{items: map[string]model.Attachment{
|
||||||
|
"attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false),
|
||||||
|
"attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true),
|
||||||
|
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
|
||||||
|
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
|
||||||
|
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
|
||||||
|
}}
|
||||||
|
objects := &objectStub{items: map[string]model.StoredObject{
|
||||||
|
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
|
||||||
|
"attachments/b/original": {Key: "attachments/b/original", LastModified: old},
|
||||||
|
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
|
||||||
|
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
|
||||||
|
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
|
||||||
|
}}
|
||||||
|
reconciler, err := New(metadata, objects, Config{
|
||||||
|
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
|
||||||
|
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
report, err := reconciler.Run(context.Background())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 1, report.OldPendingAttachments)
|
||||||
|
require.Equal(t, 1, report.OldUnreferencedAttachments)
|
||||||
|
require.Equal(t, 1, report.FreshUnreferencedAttachments)
|
||||||
|
require.Equal(t, 1, report.DeletedOrphanObjects)
|
||||||
|
require.Equal(t, 2, report.DeletedAttachmentObjects)
|
||||||
|
require.Equal(t, 3, report.DeletedUnreferencedRows)
|
||||||
|
require.Contains(t, metadata.items, "attachments/b/original")
|
||||||
|
require.Contains(t, metadata.items, "attachments/d/original")
|
||||||
|
require.NotContains(t, metadata.items, "attachments/a/original")
|
||||||
|
require.NotContains(t, metadata.items, "attachments/c/original")
|
||||||
|
require.NotContains(t, metadata.items, "attachments/missing/original")
|
||||||
|
require.NotContains(t, objects.items, "attachments/orphan/original")
|
||||||
|
|
||||||
|
repeated, err := reconciler.Run(context.Background())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Zero(t, repeated.DeletedOrphanObjects)
|
||||||
|
require.Zero(t, repeated.DeletedAttachmentObjects)
|
||||||
|
require.Zero(t, repeated.DeletedUnreferencedRows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}}
|
||||||
|
objects := &objectStub{items: map[string]model.StoredObject{
|
||||||
|
"attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)},
|
||||||
|
}}
|
||||||
|
reconciler, err := New(metadata, objects, Config{
|
||||||
|
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
|
||||||
|
UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
report, err := reconciler.Run(context.Background())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Zero(t, report.DeletedOrphanObjects)
|
||||||
|
require.Contains(t, objects.items, "attachments/a/original")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRejectsInvalidRetention(t *testing.T) {
|
||||||
|
_, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1})
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment {
|
||||||
|
readyAt := createdAt
|
||||||
|
return model.Attachment{
|
||||||
|
ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt,
|
||||||
|
ReadyAt: &readyAt, Referenced: referenced,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type metadataStub struct {
|
||||||
|
items map[string]model.Attachment
|
||||||
|
appeared map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) {
|
||||||
|
keys := make([]string, 0, len(s.items))
|
||||||
|
for key := range s.items {
|
||||||
|
if key > after {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var next string
|
||||||
|
if len(keys) > limit {
|
||||||
|
keys = keys[:limit]
|
||||||
|
next = keys[len(keys)-1]
|
||||||
|
}
|
||||||
|
items := make([]model.Attachment, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
items = append(items, s.items[key])
|
||||||
|
}
|
||||||
|
return items, next, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) {
|
||||||
|
if s.appeared[key] {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
_, exists := s.items[key]
|
||||||
|
return exists, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
|
||||||
|
for key, item := range s.items {
|
||||||
|
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
age := item.CreatedAt
|
||||||
|
if item.Status == model.AttachmentReady && item.ReadyAt != nil {
|
||||||
|
age = *item.ReadyAt
|
||||||
|
}
|
||||||
|
if age.After(cutoff) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
item.Status = model.AttachmentDeleting
|
||||||
|
s.items[key] = item
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) {
|
||||||
|
for key, item := range s.items {
|
||||||
|
if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced {
|
||||||
|
delete(s.items, key)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectStub struct {
|
||||||
|
items map[string]model.StoredObject
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
|
||||||
|
keys := make([]string, 0, len(s.items))
|
||||||
|
for key := range s.items {
|
||||||
|
if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var next string
|
||||||
|
if len(keys) > limit {
|
||||||
|
keys = keys[:limit]
|
||||||
|
next = keys[len(keys)-1]
|
||||||
|
}
|
||||||
|
items := make([]model.StoredObject, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
items = append(items, s.items[key])
|
||||||
|
}
|
||||||
|
return items, next, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *objectStub) Delete(_ context.Context, key string) error {
|
||||||
|
delete(s.items, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -13,6 +13,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/valkey-io/valkey-go"
|
"github.com/valkey-io/valkey-go"
|
||||||
|
|
||||||
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -21,12 +23,12 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrNotFound = errors.New("session not found")
|
ErrNotFound = model.ErrSessionNotFound
|
||||||
ErrInvalidToken = errors.New("invalid session token")
|
ErrInvalidToken = model.ErrInvalidToken
|
||||||
ErrInvalidRecord = errors.New("invalid session record")
|
ErrInvalidRecord = errors.New("invalid session record")
|
||||||
ErrInvalidTTL = errors.New("invalid session TTL")
|
ErrInvalidTTL = errors.New("invalid session TTL")
|
||||||
ErrTokenGeneration = errors.New("session token generation failed")
|
ErrTokenGeneration = errors.New("session token generation failed")
|
||||||
ErrStore = errors.New("session store failure")
|
ErrStore = model.ErrUnavailable
|
||||||
)
|
)
|
||||||
|
|
||||||
type record struct {
|
type record struct {
|
||||||
|
|||||||
+22
-1
@@ -26,6 +26,23 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge
|
|||||||
VALKEY_PASSWORD=local-valkey-password
|
VALKEY_PASSWORD=local-valkey-password
|
||||||
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
||||||
|
|
||||||
|
# Административные данные MinIO используются только при bootstrap.
|
||||||
|
MINIO_ROOT_USER=local-minio-admin
|
||||||
|
MINIO_ROOT_PASSWORD=local-minio-admin-password
|
||||||
|
|
||||||
|
# API использует отдельного пользователя с доступом только к bucket вложений.
|
||||||
|
S3_ENDPOINT=s3:9000
|
||||||
|
S3_REGION=us-east-1
|
||||||
|
S3_BUCKET=messenger-attachments
|
||||||
|
S3_ACCESS_KEY=messenger-app
|
||||||
|
S3_SECRET_KEY=local-messenger-s3-password
|
||||||
|
S3_USE_TLS=false
|
||||||
|
S3_TIMEOUT_SECONDS=5
|
||||||
|
ATTACHMENT_MAX_BYTES=10485760
|
||||||
|
ATTACHMENT_MAX_PIXELS=40000000
|
||||||
|
ATTACHMENT_BODY_MEMORY_BYTES=67108864
|
||||||
|
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
|
||||||
|
|
||||||
# Размер пула и обслуживание соединений PostgreSQL.
|
# Размер пула и обслуживание соединений PostgreSQL.
|
||||||
DB_MIN_CONNS=1
|
DB_MIN_CONNS=1
|
||||||
DB_MAX_CONNS=10
|
DB_MAX_CONNS=10
|
||||||
@@ -55,7 +72,7 @@ SWAGGER_PORT=8081
|
|||||||
|
|
||||||
# Ограничения ресурсов контейнеров.
|
# Ограничения ресурсов контейнеров.
|
||||||
API_CPUS=1
|
API_CPUS=1
|
||||||
API_MEMORY=256m
|
API_MEMORY=512m
|
||||||
MIGRATE_CPUS=0.5
|
MIGRATE_CPUS=0.5
|
||||||
MIGRATE_MEMORY=128m
|
MIGRATE_MEMORY=128m
|
||||||
NGINX_CPUS=0.5
|
NGINX_CPUS=0.5
|
||||||
@@ -74,6 +91,10 @@ POSTGRES_CPUS=2
|
|||||||
POSTGRES_MEMORY=1g
|
POSTGRES_MEMORY=1g
|
||||||
VALKEY_CPUS=1
|
VALKEY_CPUS=1
|
||||||
VALKEY_MEMORY=256m
|
VALKEY_MEMORY=256m
|
||||||
|
S3_CPUS=1
|
||||||
|
S3_MEMORY=512m
|
||||||
|
S3_BOOTSTRAP_CPUS=0.25
|
||||||
|
S3_BOOTSTRAP_MEMORY=256m
|
||||||
|
|
||||||
# Пароль локального администратора Grafana.
|
# Пароль локального администратора Grafana.
|
||||||
GRAFANA_ADMIN_PASSWORD=local-grafana-password
|
GRAFANA_ADMIN_PASSWORD=local-grafana-password
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
ALTER TABLE messages
|
||||||
|
DROP CONSTRAINT messages_text_length,
|
||||||
|
DROP CONSTRAINT messages_text_not_blank;
|
||||||
|
|
||||||
|
ALTER TABLE messages
|
||||||
|
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000),
|
||||||
|
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
|
||||||
|
|
||||||
|
CREATE TABLE attachments (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
chat_id UUID NOT NULL,
|
||||||
|
owner_id UUID NOT NULL,
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
content_type TEXT NOT NULL,
|
||||||
|
size_bytes BIGINT NOT NULL,
|
||||||
|
sha256 TEXT NOT NULL,
|
||||||
|
object_key TEXT NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
|
||||||
|
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
|
||||||
|
CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id)
|
||||||
|
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
|
||||||
|
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||||
|
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||||
|
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||||
|
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||||
|
CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE message_attachments (
|
||||||
|
message_id UUID NOT NULL,
|
||||||
|
chat_id UUID NOT NULL,
|
||||||
|
attachment_id UUID NOT NULL,
|
||||||
|
position SMALLINT NOT NULL,
|
||||||
|
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
|
||||||
|
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
|
||||||
|
CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9),
|
||||||
|
CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id)
|
||||||
|
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
|
||||||
|
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
|
||||||
|
REFERENCES attachments (id, chat_id) ON DELETE RESTRICT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
ALTER TABLE attachments
|
||||||
|
ADD COLUMN status TEXT NOT NULL DEFAULT 'ready',
|
||||||
|
ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp();
|
||||||
|
|
||||||
|
UPDATE attachments
|
||||||
|
SET ready_at = created_at;
|
||||||
|
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')),
|
||||||
|
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||||
|
(status = 'pending' AND ready_at IS NULL) OR
|
||||||
|
(status IN ('ready', 'deleting') AND ready_at IS NOT NULL)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX attachments_status_created_at_object_key_idx
|
||||||
|
ON attachments (status, created_at, object_key);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
ALTER TABLE attachments
|
||||||
|
DROP CONSTRAINT attachments_ready_at;
|
||||||
|
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||||
|
(status = 'pending' AND ready_at IS NULL) OR
|
||||||
|
(status = 'ready' AND ready_at IS NOT NULL) OR
|
||||||
|
status = 'deleting'
|
||||||
|
);
|
||||||
@@ -85,6 +85,50 @@
|
|||||||
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
|
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
|
||||||
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
|
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Attachment uploads by result", "id": 15,
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 56},
|
||||||
|
"targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Attachment upload latency p95", "id": 16,
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 56},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||||
|
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Attachment upload traffic", "id": 17,
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 64},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
|
||||||
|
"targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "MinIO objects", "id": 18,
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 64},
|
||||||
|
"targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "MinIO requests and errors", "id": 19,
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 64},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"},
|
||||||
|
{"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "MinIO availability", "id": 20,
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 72},
|
||||||
|
"targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "MinIO storage", "id": 21,
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 72},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"},
|
||||||
|
{"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "5s",
|
"refresh": "5s",
|
||||||
@@ -93,7 +137,7 @@
|
|||||||
"templating": {"list": []},
|
"templating": {"list": []},
|
||||||
"time": {"from": "now-15m", "to": "now"},
|
"time": {"from": "now-15m", "to": "now"},
|
||||||
"timezone": "browser",
|
"timezone": "browser",
|
||||||
"title": "Messenger lab3 overview",
|
"title": "Messenger lab4 overview",
|
||||||
"uid": "messenger-overview",
|
"uid": "messenger-overview",
|
||||||
"version": 2
|
"version": 3
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
global:
|
global:
|
||||||
scrape_interval: 5s
|
scrape_interval: 1s
|
||||||
evaluation_interval: 5s
|
evaluation_interval: 1s
|
||||||
|
|
||||||
scrape_configs:
|
scrape_configs:
|
||||||
- job_name: messenger
|
- job_name: messenger
|
||||||
@@ -21,3 +21,8 @@ scrape_configs:
|
|||||||
- job_name: valkey
|
- job_name: valkey
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["valkey-exporter:9121"]
|
- targets: ["valkey-exporter:9121"]
|
||||||
|
|
||||||
|
- job_name: minio
|
||||||
|
metrics_path: /minio/v2/metrics/cluster
|
||||||
|
static_configs:
|
||||||
|
- targets: ["s3:9000"]
|
||||||
|
|||||||
+3
-1
@@ -20,12 +20,13 @@ http {
|
|||||||
|
|
||||||
upstream messenger {
|
upstream messenger {
|
||||||
zone messenger 64k;
|
zone messenger 64k;
|
||||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
include /etc/nginx/conf.d/upstream.conf;
|
||||||
keepalive 32;
|
keepalive 32;
|
||||||
}
|
}
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 8080;
|
listen 8080;
|
||||||
|
client_max_body_size 11m;
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
proxy_pass http://messenger;
|
proxy_pass http://messenger;
|
||||||
@@ -34,6 +35,7 @@ http {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
proxy_connect_timeout 1s;
|
proxy_connect_timeout 1s;
|
||||||
proxy_send_timeout 10s;
|
proxy_send_timeout 10s;
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||||
+3
-2
@@ -2,7 +2,8 @@
|
|||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
docker compose stop nginx api valkey postgres
|
docker compose stop nginx api valkey postgres s3
|
||||||
docker compose start postgres valkey
|
docker compose start postgres valkey s3
|
||||||
|
docker compose run --rm s3-bootstrap
|
||||||
docker compose run --rm migrate
|
docker compose run --rm migrate
|
||||||
docker compose up -d --wait --wait-timeout 90
|
docker compose up -d --wait --wait-timeout 90
|
||||||
|
|||||||
+194
-28
@@ -2,56 +2,222 @@
|
|||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
wait_for_replicas() {
|
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
||||||
expected=$1
|
printf 'usage: %s <image-manifest>\n' "$0" >&2
|
||||||
deadline=$(($(date +%s) + 90))
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
manifest=$1
|
||||||
|
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"}
|
||||||
|
current_manifest=$deploy_dir/current-images.env
|
||||||
|
active_containers=$deploy_dir/active-api-containers
|
||||||
|
previous_containers=$deploy_dir/previous-api-containers
|
||||||
|
upstream_file=$deploy_dir/upstream.conf
|
||||||
|
upstream_backup=$deploy_dir/upstream.previous.conf
|
||||||
|
candidate_file=$deploy_dir/candidate-api-containers
|
||||||
|
replicas=${API_REPLICAS:-2}
|
||||||
|
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4}
|
||||||
|
switched=false
|
||||||
|
nginx_created=false
|
||||||
|
upstream_changed=false
|
||||||
|
old_pool=""
|
||||||
|
|
||||||
|
mkdir -p "$deploy_dir"
|
||||||
|
: > "$candidate_file"
|
||||||
|
export NGINX_UPSTREAM_FILE=$upstream_file
|
||||||
|
|
||||||
|
load_manifest() {
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$1"
|
||||||
|
set +a
|
||||||
|
}
|
||||||
|
|
||||||
|
compose() {
|
||||||
|
docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
container_name() {
|
||||||
|
name=$(docker inspect --format '{{.Name}}' "$1")
|
||||||
|
printf '%s\n' "${name#/}"
|
||||||
|
}
|
||||||
|
|
||||||
|
running_container() {
|
||||||
|
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
|
||||||
|
}
|
||||||
|
|
||||||
|
write_upstream() {
|
||||||
|
output=$1
|
||||||
|
containers=$2
|
||||||
|
: > "$output"
|
||||||
|
for container in $containers; do
|
||||||
|
name=$(container_name "$container")
|
||||||
|
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_containers() {
|
||||||
|
containers=$1
|
||||||
|
deadline=$(($(date +%s) + 120))
|
||||||
|
|
||||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||||
containers=$(docker compose ps -q api)
|
|
||||||
count=0
|
count=0
|
||||||
healthy=0
|
healthy=0
|
||||||
for container in $containers; do
|
for container in $containers; do
|
||||||
count=$((count + 1))
|
count=$((count + 1))
|
||||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
|
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
|
||||||
if [ "$status" = "healthy" ]; then
|
if [ "$status" = healthy ]; then
|
||||||
healthy=$((healthy + 1))
|
healthy=$((healthy + 1))
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
|
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
||||||
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
|
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
old_containers=$(docker compose ps -q api)
|
smoke_check() {
|
||||||
old_count=0
|
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||||
for container in $old_containers; do
|
}
|
||||||
old_count=$((old_count + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
docker compose build
|
remove_containers() {
|
||||||
docker compose up -d postgres valkey
|
containers=$1
|
||||||
docker compose run --rm migrate
|
for container in $containers; do
|
||||||
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
if [ "$old_count" -lt 2 ]; then
|
restore_upstream() {
|
||||||
docker compose up -d --scale api=2 --wait --wait-timeout 120
|
if [ -s "$upstream_backup" ]; then
|
||||||
exit 0
|
cp "$upstream_backup" "$upstream_file"
|
||||||
|
compose exec -T nginx nginx -t
|
||||||
|
compose exec -T nginx nginx -s reload
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
rollback() {
|
||||||
|
status=$?
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
candidates=$(cat "$candidate_file" 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
|
||||||
|
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
|
||||||
|
restore_upstream || true
|
||||||
|
if [ "$switched" = true ]; then
|
||||||
|
smoke_check || true
|
||||||
|
fi
|
||||||
|
elif [ "$nginx_created" = true ]; then
|
||||||
|
compose stop nginx >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
remove_containers "$candidates"
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap rollback EXIT
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
|
load_manifest "$manifest"
|
||||||
|
new_api_image=$API_IMAGE
|
||||||
|
|
||||||
|
if [ -f "$previous_containers" ]; then
|
||||||
|
remove_containers "$(cat "$previous_containers")"
|
||||||
|
rm -f "$previous_containers"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
if [ -s "$active_containers" ]; then
|
||||||
wait_for_replicas 3
|
for container in $(cat "$active_containers"); do
|
||||||
|
if running_container "$container"; then
|
||||||
|
old_pool="$old_pool $container"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
if [ -z "$old_pool" ]; then
|
||||||
|
old_pool=$(compose ps -q api)
|
||||||
|
fi
|
||||||
|
|
||||||
for container in $old_containers; do
|
if [ -s "$upstream_file" ]; then
|
||||||
docker stop --time 6 "$container" >/dev/null
|
:
|
||||||
docker rm "$container" >/dev/null
|
elif [ -n "$old_pool" ]; then
|
||||||
docker compose up -d --no-deps --no-recreate --scale api=3 api
|
write_upstream "$upstream_file" "$old_pool"
|
||||||
wait_for_replicas 3
|
else
|
||||||
|
cp nginx/upstream.conf "$upstream_file"
|
||||||
|
fi
|
||||||
|
cp "$upstream_file" "$upstream_backup"
|
||||||
|
|
||||||
|
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||||
|
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
|
||||||
|
compose run --rm --no-deps s3-bootstrap
|
||||||
|
compose run --rm --no-deps migrate
|
||||||
|
|
||||||
|
nginx_container=$(compose ps -q nginx)
|
||||||
|
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
|
||||||
|
compose up -d --no-build --no-deps --force-recreate nginx
|
||||||
|
nginx_container=$(compose ps -q nginx)
|
||||||
|
fi
|
||||||
|
if [ -n "$nginx_container" ]; then
|
||||||
|
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
|
||||||
|
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
|
||||||
|
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
API_IMAGE=$new_api_image
|
||||||
|
export API_IMAGE
|
||||||
|
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
|
||||||
|
if [ -z "$short_digest" ]; then
|
||||||
|
short_digest=$(date +%s)
|
||||||
|
fi
|
||||||
|
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
|
||||||
|
|
||||||
|
candidate_pool=""
|
||||||
|
index=1
|
||||||
|
while [ "$index" -le "$replicas" ]; do
|
||||||
|
name="${project}-api-${short_digest}-${generation}-${index}"
|
||||||
|
container=$(compose run -d --no-deps --name "$name" api)
|
||||||
|
candidate_pool="$candidate_pool $container"
|
||||||
|
printf '%s\n' "$container" >> "$candidate_file"
|
||||||
|
index=$((index + 1))
|
||||||
done
|
done
|
||||||
|
wait_for_containers "$candidate_pool"
|
||||||
|
|
||||||
docker compose up -d --no-deps --no-recreate --scale api=2 api
|
write_upstream "$upstream_file.next" "$candidate_pool"
|
||||||
wait_for_replicas 2
|
cp "$upstream_file.next" "$upstream_file"
|
||||||
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
|
upstream_changed=true
|
||||||
|
|
||||||
|
if [ -n "$(compose ps -q nginx)" ]; then
|
||||||
|
compose exec -T nginx nginx -t
|
||||||
|
switched=true
|
||||||
|
compose exec -T nginx nginx -s reload
|
||||||
|
else
|
||||||
|
nginx_created=true
|
||||||
|
compose up -d --no-build --no-deps nginx
|
||||||
|
switched=true
|
||||||
|
fi
|
||||||
|
smoke_check
|
||||||
|
|
||||||
|
application_network="${project}_application"
|
||||||
|
for container in $candidate_pool; do
|
||||||
|
docker network disconnect "$application_network" "$container"
|
||||||
|
docker network connect --alias api "$application_network" "$container"
|
||||||
|
done
|
||||||
|
smoke_check
|
||||||
|
|
||||||
|
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
|
||||||
|
|
||||||
|
printf '%s\n' "$candidate_pool" > "$active_containers"
|
||||||
|
cp "$manifest" "$current_manifest"
|
||||||
|
rm -f "$candidate_file" "$upstream_file.next"
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
|
||||||
|
if [ -n "$old_pool" ]; then
|
||||||
|
for container in $old_pool; do
|
||||||
|
docker stop --time 6 "$container" >/dev/null || true
|
||||||
|
done
|
||||||
|
printf '%s\n' "$old_pool" > "$previous_containers"
|
||||||
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user