From 69ec859cae4867429e01859c180d7f3ccc305a74 Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:59:39 +0300 Subject: [PATCH 1/5] chore: import lab6 baseline --- .dockerignore | 1 + .gitea/workflows/ci.yml | 101 ++ .gitea/workflows/deploy.yml | 200 +++ .gitea/workflows/incidents.yml | 73 + .gitignore | 2 + Dockerfile | 45 + cmd/dependencycheck/main.go | 330 ++++ cmd/lab5experiment/api_scenario.go | 298 ++++ cmd/lab5experiment/main.go | 607 +++++++ cmd/messenger/main.go | 40 + compose.yaml | 526 ++++++ config/messenger.yaml | 76 + deploy/compose.images.yaml | 30 + deploy/compose.runtime.yaml | 4 + go.mod | 83 + go.sum | 184 ++ infra/loki/loki.yaml | 48 + infra/minio/bootstrap.sh | 36 + infra/minio/client.Dockerfile | 16 + infra/minio/server.Dockerfile | 18 + infra/otel/collector.yaml | 73 + infra/tempo/tempo.yaml | 48 + internal/bootstrap/migration.go | 25 + internal/bootstrap/pipeline.go | 222 +++ internal/bootstrap/reconciliation.go | 86 + internal/bootstrap/reconciliation_test.go | 13 + internal/bootstrap/replay.go | 39 + internal/bootstrap/server.go | 123 ++ internal/bootstrap/telemetry.go | 23 + internal/broker/redisstream/broker.go | 309 ++++ internal/broker/redisstream/broker_test.go | 93 + internal/broker/redisstream/envelope.go | 147 ++ internal/broker/redisstream/envelope_test.go | 74 + internal/config/config.go | 416 +++++ internal/config/config_test.go | 131 ++ internal/config/loader.go | 47 + internal/config/pipeline.go | 307 ++++ internal/database/database.go | 94 + internal/database/migrations.go | 150 ++ internal/database/telemetry.go | 28 + internal/logger/logger.go | 10 + .../messenger/attachment/pipeline/metrics.go | 106 ++ internal/messenger/attachment/service.go | 237 +++ internal/messenger/attachment/service_test.go | 220 +++ .../messenger/attachment/transform/exif.go | 169 ++ .../attachment/transform/processor.go | 272 +++ .../attachment/transform/processor_test.go | 261 +++ .../messenger/attachment/transform/resize.go | 128 ++ .../messenger/attachment/worker/worker.go | 543 ++++++ .../attachment/worker/worker_test.go | 109 ++ internal/messenger/httpapi/attachments.go | 198 +++ internal/messenger/httpapi/auth.go | 111 ++ internal/messenger/httpapi/errors.go | 61 + internal/messenger/httpapi/handlers.go | 300 ++++ .../messenger/httpapi/messenger_interface.go | 49 + internal/messenger/httpapi/metrics.go | 90 + internal/messenger/httpapi/model.go | 30 + internal/messenger/httpapi/request.go | 147 ++ internal/messenger/httpapi/routes.go | 47 + internal/messenger/httpapi/server.go | 158 ++ internal/messenger/httpapi/server_test.go | 378 +++++ internal/model/errors.go | 24 + internal/model/model.go | 223 +++ internal/model/model_test.go | 24 + internal/objectstore/store.go | 225 +++ internal/outbox/dispatcher.go | 233 +++ internal/outbox/dispatcher_test.go | 207 +++ internal/password/password.go | 133 ++ internal/password/password_test.go | 44 + internal/postgres/attachment_jobs.go | 563 ++++++ internal/postgres/attachment_lease.go | 32 + internal/postgres/attachment_recovery.go | 210 +++ internal/postgres/attachments.go | 393 +++++ internal/postgres/errors.go | 17 + internal/postgres/idempotency.go | 374 ++++ internal/postgres/outbox.go | 136 ++ internal/postgres/store.go | 442 +++++ internal/postgres/store_integration_test.go | 711 ++++++++ internal/reconcile/reconcile.go | 410 +++++ internal/reconcile/reconcile_test.go | 340 ++++ internal/session/session.go | 163 ++ internal/session/session_integration_test.go | 72 + internal/telemetry/database.go | 39 + internal/telemetry/logging.go | 74 + internal/telemetry/propagation.go | 70 + internal/telemetry/propagation_test.go | 33 + internal/telemetry/runtime.go | 252 +++ internal/telemetry/runtime_test.go | 59 + lab6.env.example | 93 + migrations/0001_initial.sql | 50 + migrations/0002_message_idempotency.sql | 19 + migrations/0003_attachments.sql | 47 + migrations/0004_attachment_lifecycle.sql | 16 + migrations/0005_allow_pending_deletion.sql | 9 + migrations/0006_async_attachment_pipeline.sql | 135 ++ migrations/0007_outbox_retention.sql | 3 + migrations/embed.go | 8 + .../grafana/dashboards/messenger-lab6.json | 1512 +++++++++++++++++ .../grafana/provisioning/alerting/.gitkeep | 1 + .../provisioning/dashboards/messenger.yml | 10 + .../datasources/observability.yml | 74 + .../grafana/provisioning/plugins/.gitkeep | 1 + monitoring/postgres-queries.yml | 39 + monitoring/prometheus.yml | 85 + monitoring/rules/messenger.yml | 178 ++ monitoring/runbooks/api-availability.md | 72 + monitoring/runbooks/attachment-pipeline.md | 85 + nginx/nginx.conf | 57 + nginx/upstream.conf | 1 + scripts/incident-database.sh | 91 + scripts/incident-telemetry.sh | 73 + scripts/incident-worker.sh | 107 ++ scripts/lab6-alert.py | 67 + scripts/lab6-correlation.py | 148 ++ scripts/lab6-observe.py | 86 + scripts/lab6-traffic.py | 261 +++ scripts/restart.sh | 9 + scripts/rolling-update.sh | 223 +++ scripts/stop-one.sh | 16 + scripts/test-lab5-acceptance.sh | 18 + scripts/test-lab5.sh | 11 + scripts/test-lab6.sh | 73 + scripts/validate-lab6-evidence.py | 101 ++ 123 files changed, 18072 insertions(+) create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitea/workflows/deploy.yml create mode 100644 .gitea/workflows/incidents.yml create mode 100644 Dockerfile create mode 100644 cmd/dependencycheck/main.go create mode 100644 cmd/lab5experiment/api_scenario.go create mode 100644 cmd/lab5experiment/main.go create mode 100644 cmd/messenger/main.go create mode 100644 compose.yaml create mode 100644 config/messenger.yaml create mode 100644 deploy/compose.images.yaml create mode 100644 deploy/compose.runtime.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/loki/loki.yaml create mode 100644 infra/minio/bootstrap.sh create mode 100644 infra/minio/client.Dockerfile create mode 100644 infra/minio/server.Dockerfile create mode 100644 infra/otel/collector.yaml create mode 100644 infra/tempo/tempo.yaml create mode 100644 internal/bootstrap/migration.go create mode 100644 internal/bootstrap/pipeline.go create mode 100644 internal/bootstrap/reconciliation.go create mode 100644 internal/bootstrap/reconciliation_test.go create mode 100644 internal/bootstrap/replay.go create mode 100644 internal/bootstrap/server.go create mode 100644 internal/bootstrap/telemetry.go create mode 100644 internal/broker/redisstream/broker.go create mode 100644 internal/broker/redisstream/broker_test.go create mode 100644 internal/broker/redisstream/envelope.go create mode 100644 internal/broker/redisstream/envelope_test.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/config/loader.go create mode 100644 internal/config/pipeline.go create mode 100644 internal/database/database.go create mode 100644 internal/database/migrations.go create mode 100644 internal/database/telemetry.go create mode 100644 internal/logger/logger.go create mode 100644 internal/messenger/attachment/pipeline/metrics.go create mode 100644 internal/messenger/attachment/service.go create mode 100644 internal/messenger/attachment/service_test.go create mode 100644 internal/messenger/attachment/transform/exif.go create mode 100644 internal/messenger/attachment/transform/processor.go create mode 100644 internal/messenger/attachment/transform/processor_test.go create mode 100644 internal/messenger/attachment/transform/resize.go create mode 100644 internal/messenger/attachment/worker/worker.go create mode 100644 internal/messenger/attachment/worker/worker_test.go create mode 100644 internal/messenger/httpapi/attachments.go create mode 100644 internal/messenger/httpapi/auth.go create mode 100644 internal/messenger/httpapi/errors.go create mode 100644 internal/messenger/httpapi/handlers.go create mode 100644 internal/messenger/httpapi/messenger_interface.go create mode 100644 internal/messenger/httpapi/metrics.go create mode 100644 internal/messenger/httpapi/model.go create mode 100644 internal/messenger/httpapi/request.go create mode 100644 internal/messenger/httpapi/routes.go create mode 100644 internal/messenger/httpapi/server.go create mode 100644 internal/messenger/httpapi/server_test.go create mode 100644 internal/model/errors.go create mode 100644 internal/model/model.go create mode 100644 internal/model/model_test.go create mode 100644 internal/objectstore/store.go create mode 100644 internal/outbox/dispatcher.go create mode 100644 internal/outbox/dispatcher_test.go create mode 100644 internal/password/password.go create mode 100644 internal/password/password_test.go create mode 100644 internal/postgres/attachment_jobs.go create mode 100644 internal/postgres/attachment_lease.go create mode 100644 internal/postgres/attachment_recovery.go create mode 100644 internal/postgres/attachments.go create mode 100644 internal/postgres/errors.go create mode 100644 internal/postgres/idempotency.go create mode 100644 internal/postgres/outbox.go create mode 100644 internal/postgres/store.go create mode 100644 internal/postgres/store_integration_test.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 internal/reconcile/reconcile_test.go create mode 100644 internal/session/session.go create mode 100644 internal/session/session_integration_test.go create mode 100644 internal/telemetry/database.go create mode 100644 internal/telemetry/logging.go create mode 100644 internal/telemetry/propagation.go create mode 100644 internal/telemetry/propagation_test.go create mode 100644 internal/telemetry/runtime.go create mode 100644 internal/telemetry/runtime_test.go create mode 100644 lab6.env.example create mode 100644 migrations/0001_initial.sql create mode 100644 migrations/0002_message_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_attachment_lifecycle.sql create mode 100644 migrations/0005_allow_pending_deletion.sql create mode 100644 migrations/0006_async_attachment_pipeline.sql create mode 100644 migrations/0007_outbox_retention.sql create mode 100644 migrations/embed.go create mode 100644 monitoring/grafana/dashboards/messenger-lab6.json create mode 100644 monitoring/grafana/provisioning/alerting/.gitkeep create mode 100644 monitoring/grafana/provisioning/dashboards/messenger.yml create mode 100644 monitoring/grafana/provisioning/datasources/observability.yml create mode 100644 monitoring/grafana/provisioning/plugins/.gitkeep create mode 100644 monitoring/postgres-queries.yml create mode 100644 monitoring/prometheus.yml create mode 100644 monitoring/rules/messenger.yml create mode 100644 monitoring/runbooks/api-availability.md create mode 100644 monitoring/runbooks/attachment-pipeline.md create mode 100644 nginx/nginx.conf create mode 100644 nginx/upstream.conf create mode 100755 scripts/incident-database.sh create mode 100755 scripts/incident-telemetry.sh create mode 100755 scripts/incident-worker.sh create mode 100755 scripts/lab6-alert.py create mode 100755 scripts/lab6-correlation.py create mode 100644 scripts/lab6-observe.py create mode 100755 scripts/lab6-traffic.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/test-lab5-acceptance.sh create mode 100755 scripts/test-lab5.sh create mode 100755 scripts/test-lab6.sh create mode 100755 scripts/validate-lab6-evidence.py diff --git a/.dockerignore b/.dockerignore index bc16e55..ddd2c73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +/messenger .DS_Store diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..33018c7 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,101 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab6.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + docker compose up -d --scale api=2 --wait --wait-timeout 90 + make test-lab5-acceptance + make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..b1b9a29 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,200 @@ +name: Deploy production + +on: + workflow_dispatch: + +concurrency: + group: production + cancel-in-progress: false + +jobs: + verify: + name: Verify selected commit + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + api_image: ${{ steps.publish.outputs.api_image }} + nginx_image: ${{ steps.publish.outputs.nginx_image }} + swagger_image: ${{ steps.publish.outputs.swagger_image }} + prometheus_image: ${{ steps.publish.outputs.prometheus_image }} + grafana_image: ${{ steps.publish.outputs.grafana_image }} + s3_image: ${{ steps.publish.outputs.s3_image }} + s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} + env: + COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }} + COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }} + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab6.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + make test-lab5-acceptance + make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + + - name: Publish verified images + id: publish + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin + + publish() { + output=$1 + image=$2 + push_output=$(docker push "$image") + printf '%s\n' "$push_output" + digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1) + test -n "$digest" + repository=${image%:*} + printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT" + } + + publish api_image "$API_IMAGE" + publish nginx_image "$NGINX_IMAGE" + publish swagger_image "$SWAGGER_IMAGE" + publish prometheus_image "$PROMETHEUS_IMAGE" + publish grafana_image "$GRAFANA_IMAGE" + publish s3_image "$S3_IMAGE" + publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE" + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi + + deploy: + name: Deploy selected commit + needs: verify + runs-on: ubuntu-latest + environment: production + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + DEPLOY_SHA: ${{ github.sha }} + steps: + - name: Configure SSH + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + + - name: Prepare immutable image manifest + run: | + cat > deploy-images.env </dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker compose down -v --remove-orphans diff --git a/.gitignore b/.gitignore index b5a7189..7913498 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +evidence/runtime/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab37655 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM golang:1.26.5-alpine3.24 AS build + +WORKDIR /src +ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go build -p=1 -o /out/messenger ./cmd/messenger +RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment + +FROM build AS test + +CMD ["go", "test", "./internal/postgres", "./internal/session"] + +FROM swaggerapi/swagger-ui:v5.31.2 AS swagger + +COPY contracts/openapi.json /spec/openapi.json +ENV SWAGGER_JSON=/spec/openapi.json + +FROM prom/prometheus:v3.13.3 AS prometheus + +COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml +COPY monitoring/rules /etc/prometheus/rules + +FROM grafana/grafana:13.2.4 AS grafana + +COPY monitoring/grafana/provisioning /etc/grafana/provisioning +COPY monitoring/grafana/dashboards /etc/grafana/dashboards + +FROM nginx:1.30.5-alpine AS loadbalancer + +COPY nginx/nginx.conf /etc/nginx/nginx.conf +COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf + +FROM alpine:3.24.2 + +RUN addgroup -S app && adduser -S -G app -u 10001 app +COPY --from=build --chown=app:app /out/messenger /app/messenger +COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment + +USER app +WORKDIR /app +HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 +CMD ["/app/messenger"] diff --git a/cmd/dependencycheck/main.go b/cmd/dependencycheck/main.go new file mode 100644 index 0000000..eaa3d64 --- /dev/null +++ b/cmd/dependencycheck/main.go @@ -0,0 +1,330 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +type apiClient struct { + baseURL string + client *http.Client + token string + userID string +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println("PASS: replica idempotency and dependency recovery") +} + +func run() error { + baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + flag.Parse() + client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}} + if err := client.registerAndLogin(); err != nil { + return err + } + if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil { + return err + } + if err := client.verifyConcurrentIdempotency(); err != nil { + return err + } + if err := verifyOutage(client, "valkey"); err != nil { + return err + } + return verifyOutage(client, "postgres") +} + +func (c *apiClient) registerAndLogin() error { + username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] + password := "T3st-" + uuid.NewString() + body := map[string]string{"username": username, "display_name": "Dependency check", "password": password} + payload, err := json.Marshal(body) + if err != nil { + return err + } + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("register: %w", err) + } + registrationBody := response.Body + defer func() { _ = registrationBody.Close() }() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("register status = %d", response.StatusCode) + } + var user struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil { + return fmt.Errorf("decode user: %w", err) + } + if user.ID == "" { + return fmt.Errorf("register returned an empty user id") + } + c.userID = user.ID + + request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return err + } + request.SetBasicAuth(username, password) + response, err = c.client.Do(request) + if err != nil { + return fmt.Errorf("login: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("login status = %d", response.StatusCode) + } + var session struct { + Token string `json:"token"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return fmt.Errorf("decode session: %w", err) + } + if session.Token == "" { + return fmt.Errorf("login returned an empty token") + } + c.token = session.Token + return nil +} + +type messageResult struct { + messageID string + instanceID string + err error +} + +func (c *apiClient) verifyConcurrentIdempotency() error { + chatID, err := c.createChat() + if err != nil { + return err + } + + const workers = 16 + var ( + key = uuid.NewString() + start = make(chan struct{}) + results = make(chan messageResult, workers) + group sync.WaitGroup + instances = make(map[string]struct{}) + messageID string + ) + transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers} + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + defer transport.CloseIdleConnections() + + for range workers { + group.Add(1) + go func() { + defer group.Done() + <-start + id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message") + results <- messageResult{messageID: id, instanceID: instance, err: requestErr} + }() + } + close(start) + group.Wait() + close(results) + + for result := range results { + if result.err != nil { + return result.err + } + if result.instanceID == "" { + return fmt.Errorf("message response has an empty X-Instance-Id") + } + instances[result.instanceID] = struct{}{} + if messageID == "" { + messageID = result.messageID + } + if result.messageID != messageID { + return fmt.Errorf("idempotent responses returned different message ids") + } + } + if len(instances) < 2 { + return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances)) + } + return c.verifyMessageHistory(chatID, messageID) +} + +func (c *apiClient) createChat() (string, error) { + payload, err := json.Marshal(map[string]any{ + "title": "Replica check", "member_ids": []string{c.userID}, + }) + if err != nil { + return "", err + } + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload)) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + response, err := c.client.Do(request) + if err != nil { + return "", fmt.Errorf("create chat: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create chat status = %d", response.StatusCode) + } + var chat struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil { + return "", fmt.Errorf("decode chat: %w", err) + } + return chat.ID, nil +} + +func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) { + payload, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", "", err + } + request, err := http.NewRequest( + http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload), + ) + if err != nil { + return "", "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", key) + response, err := client.Do(request) + if err != nil { + return "", "", fmt.Errorf("create message: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode) + } + var message struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err) + } + return message.ID, response.Header.Get("X-Instance-Id"), nil +} + +func (c *apiClient) verifyMessageHistory(chatID, messageID string) error { + request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+c.token) + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("list messages: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("list messages status = %d", response.StatusCode) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil { + return fmt.Errorf("decode messages: %w", err) + } + if len(page.Items) != 1 || page.Items[0].ID != messageID { + return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items)) + } + return nil +} + +func verifyOutage(client *apiClient, service string) (returnErr error) { + if err := compose("stop", service); err != nil { + return err + } + defer func() { + if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil { + returnErr = err + } + }() + if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil { + return fmt.Errorf("%s readiness outage: %w", service, err) + } + if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil { + return fmt.Errorf("%s business outage: %w", service, err) + } + if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil { + return err + } + if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil { + return fmt.Errorf("%s readiness recovery: %w", service, err) + } + if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil { + return fmt.Errorf("%s session recovery: %w", service, err) + } + return nil +} + +func compose(arguments ...string) error { + command := exec.Command("docker", append([]string{"compose"}, arguments...)...) + if output, err := command.CombinedOutput(); err != nil { + return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output)) + } + return nil +} + +func (c *apiClient) expectEventually(path string, status int, authenticated bool) error { + deadline := time.Now().Add(20 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + lastErr = c.expect(http.MethodGet, path, status, authenticated) + if lastErr == nil { + return nil + } + time.Sleep(200 * time.Millisecond) + } + return lastErr +} + +func (c *apiClient) expect(method, path string, status int, authenticated bool) error { + request, err := http.NewRequest(method, c.baseURL+path, nil) + if err != nil { + return err + } + if authenticated { + request.Header.Set("Authorization", "Bearer "+c.token) + } + response, err := c.client.Do(request) + if err != nil { + return err + } + return closeResponse(response, status) +} + +func closeResponse(response *http.Response, status int) error { + defer func() { _ = response.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + if response.StatusCode != status { + return fmt.Errorf("status = %d, want %d", response.StatusCode, status) + } + return nil +} diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go new file mode 100644 index 0000000..048a121 --- /dev/null +++ b/cmd/lab5experiment/api_scenario.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type apiState struct { + BaseURL string `json:"base_url"` + Token string `json:"token"` + AttachmentID string `json:"attachment_id"` +} + +type apiUser struct { + ID string `json:"id"` +} + +type apiSession struct { + Token string `json:"token"` +} + +type apiChat struct { + ID string `json:"id"` +} + +func runAPIProduce(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-produce [-base-url URL]") + } + client := &http.Client{Timeout: 10 * time.Second} + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + password := "Lab5-" + uuid.NewString() + owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password) + if err != nil { + return err + } + token, err := createAPISession(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + var attachment model.Attachment + response, err := apiRequest( + client, http.MethodPost, + *baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png", + token, "image/png", fixture, &attachment, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued { + return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status) + } + messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}}) + if err != nil { + return err + } + response, err = apiRequestWithHeaders( + client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages", + token, "application/json", messageBody, nil, + map[string]string{"Idempotency-Key": uuid.NewString()}, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("link queued attachment returned %d", response.StatusCode) + } + response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil) + if err != nil { + return err + } + if response.StatusCode != http.StatusConflict { + return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) + } + return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) +} + +func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { + flags := flag.NewFlagSet("api-verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]") + } + var state apiState + if err := json.NewDecoder(input).Decode(&state); err != nil { + return fmt.Errorf("decode API scenario state: %w", err) + } + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) + defer cancel() + var attachment model.Attachment + for { + response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("attachment metadata returned %d", response.StatusCode) + } + if attachment.Status == model.AttachmentReady { + break + } + if attachment.Status == model.AttachmentFailed { + return fmt.Errorf("attachment failed with %v", attachment.ErrorCode) + } + if err := wait(ctx, 250*time.Millisecond); err != nil { + return err + } + } + if len(attachment.Variants) != 3 { + return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants)) + } + response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) { + return errors.New("original changed after worker and broker restart") + } + response, thumbnail, err := apiBytes( + client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" { + return errors.New("thumbnail response is invalid") + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail)) + if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 { + return errors.New("thumbnail dimensions are invalid") + } + return json.NewEncoder(output).Encode(map[string]any{ + "accepted_while_worker_stopped": true, + "accepted_before_outbox_publish": true, + "linked_while_queued": true, + "content_conflict_while_queued": true, + "ready_after_api_broker_restart": true, + "original_preserved": true, + "thumbnail_dimensions": "256x160", + }) +} + +func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": username, "password": password, + }) + if err != nil { + return apiUser{}, err + } + var user apiUser + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user) + if err != nil { + return apiUser{}, err + } + if response.StatusCode != http.StatusCreated { + return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode) + } + return user, nil +} + +func createAPISession(client *http.Client, baseURL, username, password string) (string, error) { + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Basic "+credentials) + response, err := client.Do(request) + if err != nil { + return "", err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create session returned %d", response.StatusCode) + } + var session apiSession + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return "", err + } + return session.Token, nil +} + +func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) { + body, err := json.Marshal(map[string]any{ + "title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID}, + }) + if err != nil { + return apiChat{}, err + } + var chat apiChat + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat) + if err != nil { + return apiChat{}, err + } + if response.StatusCode != http.StatusCreated { + return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode) + } + return chat, nil +} + +func apiRequest( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, +) (*http.Response, error) { + return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil) +} + +func apiRequestWithHeaders( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, + headers map[string]string, +) (*http.Response, error) { + request, err := http.NewRequest(method, requestURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + for name, value := range headers { + request.Header.Set(name, value) + } + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if result != nil && response.StatusCode < http.StatusBadRequest { + if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil { + return nil, err + } + } + return response, nil +} + +func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) { + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + return nil, nil, err + } + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + return response, body, err +} diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go new file mode 100644 index 0000000..35c5241 --- /dev/null +++ b/cmd/lab5experiment/main.go @@ -0,0 +1,607 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" +) + +const ( + experimentLease = 2 * time.Second + maxAttempts = 5 +) + +type result struct { + StartedAt string `json:"started_at"` + DurationMillis int64 `json:"duration_ms"` + PipelineVersion int `json:"pipeline_version"` + FaultScenarios []scenario `json:"fault_scenarios"` + DuplicateDeliveries int `json:"duplicate_deliveries"` + V1ObjectCount int `json:"v1_object_count"` + V2ObjectCount int `json:"v2_object_count"` + AcceptedJobs int `json:"accepted_jobs"` + TerminalJobs int `json:"terminal_jobs"` + ReadyJobs int `json:"ready_jobs"` + FailedJobs int `json:"failed_jobs"` + PermanentErrorCode string `json:"permanent_error_code"` + ReplayAttempt int `json:"replay_attempt"` + InvariantResults map[string]bool `json:"invariant_results"` +} + +type scenario struct { + Name string `json:"name"` + AttemptAfterRecovery int `json:"attempt_after_recovery"` + ObjectsBeforeRecovery int `json:"objects_before_recovery"` + ObjectsAfterRecovery int `json:"objects_after_recovery"` + FinalStatus string `json:"final_status"` +} + +type experiment struct { + settings config.WorkerConfig + repository *postgres.Store + objects *objectstore.Store + broker *redisstream.Broker + dispatcher *outbox.Dispatcher + worker *worker.Worker + ownerID string + chatID string + fixture []byte +} + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(arguments []string, output io.Writer) error { + if len(arguments) > 0 { + switch arguments[0] { + case "api-produce": + return runAPIProduce(arguments[1:], output) + case "api-verify": + return runAPIVerify(arguments[1:], os.Stdin, output) + } + } + flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) + flags.SetOutput(io.Discard) + timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { + return errors.New("usage: lab5experiment [-timeout 2m]") + } + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + experiment, closeExperiment, err := newExperiment(ctx) + if err != nil { + return err + } + defer closeExperiment() + + report, err := experiment.run(ctx) + if err != nil { + return err + } + report.StartedAt = started.UTC().Format(time.RFC3339) + report.DurationMillis = time.Since(started).Milliseconds() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func newExperiment(ctx context.Context) (*experiment, func(), error) { + settings, err := config.LoadWorker() + if err != nil { + return nil, nil, err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return nil, nil, err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions, err := redis.ParseURL(settings.BrokerURL) + if err != nil { + pool.Close() + return nil, nil, err + } + client := redis.NewClient(redisOptions) + if err := client.Ping(ctx).Err(); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + consumer := "lab5experiment-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, + ClaimMinIdle: experimentLease, ReadCount: 32, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + if err := broker.Ensure(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + repository := postgres.New(pool) + dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ + BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, + RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processorWorker, err := worker.New( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, + worker.Config{ + WorkerID: consumer, Concurrency: 1, + LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, + ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + fixture, err := pngFixture(320, 200) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + value := &experiment{ + settings: settings, repository: repository, objects: objects, broker: broker, + dispatcher: dispatcher, worker: processorWorker, fixture: fixture, + } + if err := value.seed(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + closeExperiment := func() { + pool.Close() + _ = client.Close() + } + return value, closeExperiment, nil +} + +func (e *experiment) seed(ctx context.Context) error { + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") + if err != nil { + return err + } + member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") + if err != nil { + return err + } + chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) + if err != nil { + return err + } + e.ownerID = owner.ID + e.chatID = chat.ID + return nil +} + +func (e *experiment) run(ctx context.Context) (result, error) { + report := result{ + PipelineVersion: 1, + InvariantResults: map[string]bool{ + "accepted_jobs_reach_terminal_state": false, + "duplicate_effect_is_unique": false, + "old_pipeline_cannot_replace_new": false, + }, + } + for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { + observed, err := e.runLeaseRecovery(ctx, fault) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, observed) + report.AcceptedJobs++ + if observed.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + } + permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) + if err != nil { + return result{}, err + } + report.AcceptedJobs++ + report.TerminalJobs++ + report.FailedJobs++ + report.PermanentErrorCode = permanentCode + report.ReplayAttempt = replayAttempt + + commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, commitScenario) + report.AcceptedJobs++ + if commitScenario.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + + payload, err := redisstream.EncodeEnvelope(envelope) + if err != nil { + return result{}, err + } + for range 2 { + if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { + return result{}, err + } + } + duplicateDeliveries, err := e.readDeliveries(ctx, 2) + if err != nil { + return result{}, err + } + for _, delivery := range duplicateDeliveries { + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return result{}, err + } + } + report.DuplicateDeliveries = len(duplicateDeliveries) + report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V1ObjectCount != 4 { + return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) + } + report.InvariantResults["duplicate_effect_is_unique"] = true + + oldPayload := payload + v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} + if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { + return result{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return result{}, err + } + v2Delivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { + return result{}, err + } + if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { + return result{}, err + } + staleDelivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { + return result{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return result{}, err + } + if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { + return result{}, errors.New("new pipeline version was not preserved after stale delivery") + } + report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V2ObjectCount != 8 { + return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) + } + report.InvariantResults["old_pipeline_cannot_replace_new"] = true + report.AcceptedJobs++ + report.ReadyJobs++ + report.TerminalJobs++ + report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs + return report, nil +} + +func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { + attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") + if err != nil { + return "", 0, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { + return "", 0, errors.New("corrupted staging object did not reach failed") + } + firstCode := *metadata.ErrorCode + replayed, err := e.repository.ReplayFailedAttachment( + ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", + ) + if err != nil { + return "", 0, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", 0, err + } + replayDelivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { + return "", 0, errors.New("replayed permanent failure changed terminal classification") + } + return firstCode, replayed.Attempt, nil +} + +func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + if fault == worker.FaultBeforeSideEffect && before != 0 { + return scenario{}, errors.New("objects were written before the injected side effect") + } + if fault == worker.FaultAfterSideEffect && before != 4 { + return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, err + } + recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) + if err != nil { + return scenario{}, err + } + if recovered.Requeued != 1 { + return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return scenario{}, err + } + retry, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { + return scenario{}, err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) + if err != nil { + return scenario{}, err + } + return scenario{ + Name: string(fault), AttemptAfterRecovery: job.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, nil +} + +func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + return scenario{ + Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, delivery.Envelope, attachmentID, nil +} + +func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { + attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") + return attachmentID, job, err +} + +func (e *experiment) queueAttachmentData( + ctx context.Context, + data []byte, + contentType string, +) (string, string, model.AttachmentJob, error) { + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/experiment" + digest := sha256.Sum256(data) + _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, + }) + if err != nil { + return "", "", model.AttachmentJob{}, err + } + if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { + return "", "", model.AttachmentJob{}, err + } + job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} + if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { + return "", "", model.AttachmentJob{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", "", model.AttachmentJob{}, err + } + return attachmentID, stagingKey, job, nil +} + +func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { + deliveries, err := e.readDeliveries(ctx, 1) + if err != nil { + return redisstream.Delivery{}, err + } + return deliveries[0], nil +} + +func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { + var result []redisstream.Delivery + for len(result) < expected { + deliveries, err := e.broker.Read(ctx) + if err != nil { + return nil, err + } + result = append(result, deliveries...) + } + if len(result) != expected { + return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) + } + return result, nil +} + +func (e *experiment) ackReclaimed(ctx context.Context) error { + deliveries, _, err := e.broker.Reclaim(ctx, "0-0") + if err != nil { + return err + } + if len(deliveries) != 1 { + return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) + } + return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) +} + +func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { + objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) + if err != nil { + return 0, err + } + if next != "" { + return 0, errors.New("unexpected object pagination") + } + return len(objects), nil +} + +func pngFixture(width, height int) ([]byte, error) { + value := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, value); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func wait(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go new file mode 100644 index 0000000..c2c6c96 --- /dev/null +++ b/cmd/messenger/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "errors" + "log/slog" + "os" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger" +) + +func main() { + appLogger := logger.New() + if err := run(appLogger, os.Args[1:]); err != nil { + appLogger.Error("application stopped", "error", err) + os.Exit(1) + } +} + +func run(logger *slog.Logger, arguments []string) error { + if len(arguments) == 0 { + return bootstrap.RunServer(logger) + } + if len(arguments) == 1 && arguments[0] == "migrate" { + return bootstrap.RunMigrations() + } + if len(arguments) == 1 && arguments[0] == "dispatch" { + return bootstrap.RunDispatcher(logger) + } + if len(arguments) == 1 && arguments[0] == "worker" { + return bootstrap.RunWorker(logger) + } + if arguments[0] == "reconcile-attachments" { + return bootstrap.RunReconciliation(arguments[1:], os.Stdout) + } + if arguments[0] == "replay-attachment" { + return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout) + } + return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]") +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d21c2eb --- /dev/null +++ b/compose.yaml @@ -0,0 +1,526 @@ +x-messenger-config-volume: &messenger-config-volume + type: bind + source: ./config/messenger.yaml + target: /etc/messenger/messenger.yaml + read_only: true + +x-telemetry-environment: &telemetry-environment + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL} + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION} + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${POSTGRES_CPUS} + mem_limit: ${POSTGRES_MEMORY} + restart: unless-stopped + networks: [data] + + valkey: + image: valkey/valkey:8.1.4-alpine + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"] + environment: + VALKEY_PASSWORD: ${VALKEY_PASSWORD} + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${VALKEY_CPUS} + mem_limit: ${VALKEY_MEMORY} + restart: unless-stopped + networks: [data] + + redis-broker: + image: redis:8.2.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - always + - --maxmemory-policy + - noeviction + - --requirepass + - ${REDIS_BROKER_PASSWORD} + environment: + REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD} + volumes: + - redis_broker_data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${REDIS_BROKER_CPUS} + mem_limit: ${REDIS_BROKER_MEMORY} + restart: unless-stopped + networks: [data] + + s3: + build: + context: . + dockerfile: infra/minio/server.Dockerfile + args: + MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z + image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_BROWSER: "off" + MINIO_PROMETHEUS_AUTH_TYPE: public + volumes: + - s3_data:/data + expose: + - "9000" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${S3_CPUS} + mem_limit: ${S3_MEMORY} + restart: unless-stopped + networks: [data] + + s3-bootstrap: + build: + context: . + dockerfile: infra/minio/client.Dockerfile + args: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + depends_on: + s3: + condition: service_healthy + healthcheck: + disable: true + cpus: ${S3_BOOTSTRAP_CPUS} + mem_limit: ${S3_BOOTSTRAP_MEMORY} + restart: "no" + networks: [data] + + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "migrate"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + volumes: + - *messenger-config-volume + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: ${MIGRATE_CPUS} + mem_limit: ${MIGRATE_MEMORY} + restart: "no" + networks: [data] + + api: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab6} + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-api + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + APP_HOST: ${APP_HOST} + APP_PORT: ${APP_PORT} + DATABASE_URL: ${DATABASE_URL} + CACHE_URL: ${CACHE_URL} + SWAGGER_PORT: ${SWAGGER_PORT} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + volumes: + - *messenger-config-volume + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + interval: 2s + timeout: 1s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS} + mem_limit: ${API_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [application, data] + + dispatcher: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "dispatch"] + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-dispatcher + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${DISPATCHER_CPUS} + mem_limit: ${DISPATCHER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + worker: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "worker"] + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-worker + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${WORKER_CPUS} + mem_limit: ${WORKER_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [data, monitoring] + + nginx: + build: + context: . + target: loadbalancer + image: ${NGINX_IMAGE:-mtusi-nginx:lab6} + ports: + - "${API_BIND_ADDRESS}:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"] + interval: 2s + timeout: 1s + retries: 10 + cpus: ${NGINX_CPUS} + mem_limit: ${NGINX_MEMORY} + restart: unless-stopped + networks: [edge, application, monitoring] + + swagger: + build: + context: . + target: swagger + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6} + ports: + - "${SWAGGER_BIND_ADDRESS}:8080" + depends_on: + nginx: + condition: service_healthy + cpus: ${SWAGGER_CPUS} + mem_limit: ${SWAGGER_MEMORY} + restart: unless-stopped + networks: [edge] + + tests: + profiles: ["tests"] + build: + context: . + target: test + environment: + TEST_DATABASE_URL: ${DATABASE_URL} + TEST_CACHE_URL: ${CACHE_URL} + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [data] + + lab5-experiment: + profiles: ["experiments"] + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/lab5experiment"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [application, data] + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 + environment: + DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable + DATA_SOURCE_USER: ${POSTGRES_USER} + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_COLLECTION_TIMEOUT: 5s + PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml + volumes: + - ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro + depends_on: + postgres: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + valkey-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://valkey:6379 + REDIS_USER: default + REDIS_PASSWORD: ${VALKEY_PASSWORD} + depends_on: + valkey: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + redis-broker-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://redis-broker:6379 + REDIS_USER: default + REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed + depends_on: + redis-broker: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] + depends_on: + nginx: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [monitoring] + + prometheus: + build: + context: . + target: prometheus + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6} + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver + volumes: + - prometheus_data:/prometheus + ports: + - "${PROMETHEUS_BIND_ADDRESS}:9090" + healthcheck: + test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"] + interval: 5s + timeout: 3s + retries: 20 + depends_on: + - api + - nginx-exporter + - postgres-exporter + - valkey-exporter + - redis-broker-exporter + - dispatcher + - worker + - s3 + - otel-collector + - loki + - tempo + cpus: ${PROMETHEUS_CPUS} + mem_limit: ${PROMETHEUS_MEMORY} + restart: unless-stopped + networks: [edge, application, data, monitoring] + + grafana: + build: + context: . + target: grafana + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6} + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_EXPLORE_ENABLED: "true" + volumes: + - grafana_data:/var/lib/grafana + ports: + - "${GRAFANA_BIND_ADDRESS}:3000" + depends_on: + - prometheus + - loki + - tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${GRAFANA_CPUS} + mem_limit: ${GRAFANA_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + ports: + - "${LOKI_BIND_ADDRESS}:3100" + healthcheck: + test: ["CMD", "/usr/bin/loki", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${LOKI_CPUS} + mem_limit: ${LOKI_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + tempo: + image: grafana/tempo:2.10.10 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + ports: + - "${TEMPO_BIND_ADDRESS}:3200" + healthcheck: + test: ["CMD", "/tempo", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${TEMPO_CPUS} + mem_limit: ${TEMPO_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.160.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + expose: + - "4317" + - "4318" + - "8888" + - "8889" + - "13133" + depends_on: + - loki + - tempo + healthcheck: + test: ["CMD", "/otelcol-contrib", "components"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${OTEL_COLLECTOR_CPUS} + mem_limit: ${OTEL_COLLECTOR_MEMORY} + restart: unless-stopped + networks: [application, data, monitoring] + +volumes: + postgres_data: + valkey_data: + redis_broker_data: + prometheus_data: + grafana_data: + loki_data: + tempo_data: + s3_data: + +networks: + edge: + application: + internal: true + data: + internal: true + monitoring: + internal: true diff --git a/config/messenger.yaml b/config/messenger.yaml new file mode 100644 index 0000000..031e152 --- /dev/null +++ b/config/messenger.yaml @@ -0,0 +1,76 @@ +version: 1 + +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 + +database: + min_connections: 1 + max_connections: 10 + max_connection_age: 30m + max_connection_idle: 5m + health_check_period: 30s + dependency_timeout: 5s + +migration: + timeout: 30s + +object_storage: + use_tls: false + timeout: 5s + +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 + +pipeline: + version: 1 + +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 + +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 + +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 + +monitoring: + metrics_address: 0.0.0.0:9091 diff --git a/deploy/compose.images.yaml b/deploy/compose.images.yaml new file mode 100644 index 0000000..7d98319 --- /dev/null +++ b/deploy/compose.images.yaml @@ -0,0 +1,30 @@ +services: + api: + image: ${API_IMAGE:?API_IMAGE is required} + + migrate: + image: ${API_IMAGE:?API_IMAGE is required} + + dispatcher: + image: ${API_IMAGE:?API_IMAGE is required} + + worker: + image: ${API_IMAGE:?API_IMAGE is required} + + nginx: + image: ${NGINX_IMAGE:?NGINX_IMAGE is required} + + swagger: + image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required} + + prometheus: + image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required} + + grafana: + image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required} + + s3: + image: ${S3_IMAGE:?S3_IMAGE is required} + + s3-bootstrap: + image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required} diff --git a/deploy/compose.runtime.yaml b/deploy/compose.runtime.yaml new file mode 100644 index 0000000..b9dcd47 --- /dev/null +++ b/deploy/compose.runtime.yaml @@ -0,0 +1,4 @@ +services: + nginx: + volumes: + - ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..37c0a6e --- /dev/null +++ b/go.mod @@ -0,0 +1,83 @@ +module gitea.privcloud.tech/dizi/mtusi2026lab6 + +go 1.26.0 + +require ( + github.com/deepteams/webp v1.2.8 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.0.99 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + github.com/valkey-io/valkey-go v1.0.78 + go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel/exporters/prometheus v0.60.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/otlptranslator v0.0.2 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..63eb5b2 --- /dev/null +++ b/go.sum @@ -0,0 +1,184 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA= +github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= +github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= +github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4c04d5d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,48 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + volume_enabled: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.name + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 0000000..68a2888 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -eu + +alias_name=bootstrap +policy_name=messenger-attachments +policy_file=/tmp/messenger-attachments-policy.json + +mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +mc mb --ignore-existing "$alias_name/$S3_BUCKET" +mc anonymous set none "$alias_name/$S3_BUCKET" + +cat >"$policy_file" </dev/null diff --git a/infra/minio/client.Dockerfile b/infra/minio/client.Dockerfile new file mode 100644 index 0000000..fe8ee0a --- /dev/null +++ b/infra/minio/client.Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MC_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc +WORKDIR /src/mc +RUN go mod download +RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc . + +FROM alpine:3.22.2 + +COPY --from=build /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh + +ENTRYPOINT ["mc"] diff --git a/infra/minio/server.Dockerfile b/infra/minio/server.Dockerfile new file mode 100644 index 0000000..44b75c4 --- /dev/null +++ b/infra/minio/server.Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MINIO_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio +WORKDIR /src/minio +RUN go mod download +RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio . + +FROM alpine:3.22.2 + +RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data +COPY --from=build /out/minio /usr/local/bin/minio + +USER minio +VOLUME ["/data"] +ENTRYPOINT ["minio"] diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e912195 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,73 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_constant_labels: + included: + - service.name + - service.version + - deployment.environment.name + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..0d8a229 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,48 @@ +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +metrics_generator: + registry: + external_labels: + source: tempo + deployment_environment: lab + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + +usage_report: + reporting_enabled: false diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go new file mode 100644 index 0000000..0e7dd49 --- /dev/null +++ b/internal/bootstrap/migration.go @@ -0,0 +1,25 @@ +package bootstrap + +import ( + "context" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" +) + +func RunMigrations() error { + settings, err := config.LoadMigration() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + return database.Migrate(context.Background(), pool, settings.MigrationTimeout) +} diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go new file mode 100644 index 0000000..1539107 --- /dev/null +++ b/internal/bootstrap/pipeline.go @@ -0,0 +1,222 @@ +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func RunDispatcher(logger *slog.Logger) error { + settings, err := config.LoadDispatcher() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-dispatcher", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher", + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0) + dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ + BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, + PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, + RetryDelay: settings.RetryDelay, Retention: settings.Retention, + CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize, + }) + if err != nil { + return err + } + logger.Info("outbox dispatcher started", "stream", settings.Stream) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run) +} + +func RunWorker(logger *slog.Logger) error { + settings, err := config.LoadWorker() + if err != nil { + return err + } + hostname, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-worker", hostname) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil { + return err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + consumer := hostname + "-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + return err + } + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency) + processorWorker, err := worker.New( + logger, postgres.New(pool), broker, objects, processor, metrics, + worker.Config{ + WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, + HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + return err + } + logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run) +} + +func newRedisClient(rawURL string) (*redis.Client, error) { + options, err := redis.ParseURL(rawURL) + if err != nil { + return nil, err + } + client := redis.NewClient(options) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + return client, nil +} + +func runPipelineProcess( + logger *slog.Logger, + metricsAddress string, + metricsHandler http.Handler, + run func(context.Context) error, +) error { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler) + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + listener, err := net.Listen("tcp", metricsAddress) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + runResult := make(chan error, 1) + go func() { runResult <- run(ctx) }() + + var result error + select { + case result = <-runResult: + stop() + case result = <-serveResult: + stop() + case <-ctx.Done(): + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil && result == nil { + result = err + } + if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) { + return nil + } + logger.Info("pipeline process stopped") + return result +} + +func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func workerDatabaseConfig(settings config.WorkerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go new file mode 100644 index 0000000..3f09a3f --- /dev/null +++ b/internal/bootstrap/reconciliation.go @@ -0,0 +1,86 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "flag" + "io" + "os/signal" + "syscall" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" +) + +type reconciliationOutput struct { + Mode string `json:"mode"` + GracePeriod string `json:"grace_period"` + UnreferencedRetention string `json:"unreferenced_retention"` + PageSize int `json:"page_size"` + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func RunReconciliation(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) + flags.SetOutput(io.Discard) + apply := flags.Bool("apply", false, "apply safe repairs") + gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") + retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments") + pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]") + } + if *apply && *gracePeriod <= 0 { + return errors.New("grace-period must be positive in apply mode") + } + settings, err := config.LoadReconciliation() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + OrphanPrefixes: []string{"staging/"}, + UnreferencedRetention: *retention, Apply: *apply, + }) + if err != nil { + return err + } + started := time.Now() + report, err := reconciler.Run(ctx) + if err != nil { + return err + } + mode := "dry-run" + if *apply { + mode = "apply" + } + return json.NewEncoder(output).Encode(reconciliationOutput{ + Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(), + PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report, + }) +} diff --git a/internal/bootstrap/reconciliation_test.go b/internal/bootstrap/reconciliation_test.go new file mode 100644 index 0000000..9a9d3b0 --- /dev/null +++ b/internal/bootstrap/reconciliation_test.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) { + err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard) + require.EqualError(t, err, "grace-period must be positive in apply mode") +} diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go new file mode 100644 index 0000000..70cc3e8 --- /dev/null +++ b/internal/bootstrap/replay.go @@ -0,0 +1,39 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "io" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" +) + +func RunAttachmentReplay(arguments []string, output io.Writer) error { + if len(arguments) != 3 { + return errors.New("usage: messenger replay-attachment ") + } + settings, err := config.LoadJobAdmin() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + job, err := postgres.New(pool).ReplayFailedAttachment( + context.Background(), arguments[0], arguments[1], arguments[2], + ) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt) + return err +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go new file mode 100644 index 0000000..473c0b7 --- /dev/null +++ b/internal/bootstrap/server.go @@ -0,0 +1,123 @@ +package bootstrap + +import ( + "context" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func RunServer(logger *slog.Logger) error { + settings, err := config.LoadServer() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-api", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), databaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil { + return err + } + sessions, err := session.New(settings.CacheURL) + if err != nil { + return err + } + defer func() { _ = sessions.Close() }() + passwords, err := password.New(password.Config{ + MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, + Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, + KeyLength: settings.PasswordKeyLength, + }) + if err != nil { + return err + } + objects, err := objectstore.New(objectStoreConfig(settings)) + if err != nil { + return err + } + repository := postgres.New(pool) + attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, + CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion, + }) + if err != nil { + return err + } + app := httpapi.NewServer(logger, httpapi.Dependencies{ + Users: repository, Chats: repository, Messages: repository, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(), + }, httpapi.Config{ + SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, + RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, + }) + server := &http.Server{ + Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, + ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout, + IdleTimeout: settings.HTTPIdleTimeout, + } + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + logger.Info("server started", "address", server.Addr) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + select { + case err := <-serveResult: + return err + case <-ctx.Done(): + logger.Info("shutdown requested") + app.SetReady(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + _ = server.Close() + return err + } + return nil + } +} + +func databaseConfig(settings config.ServerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func objectStoreConfig(settings config.ServerConfig) objectstore.Config { + return objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + } +} diff --git a/internal/bootstrap/telemetry.go b/internal/bootstrap/telemetry.go new file mode 100644 index 0000000..bb1dcd9 --- /dev/null +++ b/internal/bootstrap/telemetry.go @@ -0,0 +1,23 @@ +package bootstrap + +import ( + "context" + "log/slog" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) { + runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID)) + if err != nil { + return nil, nil, err + } + return runtime, runtime.Logger(), nil +} + +func shutdownTelemetry(runtime *telemetry.Runtime) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = runtime.Shutdown(ctx) +} diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go new file mode 100644 index 0000000..0788997 --- /dev/null +++ b/internal/broker/redisstream/broker.go @@ -0,0 +1,309 @@ +package redisstream + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" +) + +const ( + payloadField = "payload" + failedPayloadMaxBytes = 64 << 10 +) + +var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + +var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer") + +var _ outbox.Publisher = (*Broker)(nil) + +type Config struct { + Stream string + Group string + Consumer string + FailedStream string + FailedMaxLen int64 + Block time.Duration + ClaimMinIdle time.Duration + ReadCount int64 +} + +type Broker struct { + client redis.UniversalClient + config Config +} + +type Delivery struct { + ID string + Envelope Envelope + DecodeError error + + broker *Broker + payload []byte +} + +type FailedDelivery struct { + Code string + FailedAt time.Time +} + +func New(client redis.UniversalClient, config Config) (*Broker, error) { + if client == nil { + return nil, errors.New("redis client is required") + } + if err := config.validate(); err != nil { + return nil, err + } + return &Broker{client: client, config: config}, nil +} + +func (c Config) validate() error { + switch { + case c.Stream == "": + return errors.New("stream is required") + case c.Group == "": + return errors.New("consumer group is required") + case c.Consumer == "": + return errors.New("consumer name is required") + case c.FailedStream == "": + return errors.New("failed stream is required") + case c.FailedStream == c.Stream: + return errors.New("failed stream must differ from job stream") + case c.FailedMaxLen <= 0: + return errors.New("failed stream maximum length must be positive") + case c.Block < 0: + return errors.New("block duration must not be negative") + case c.ClaimMinIdle <= 0: + return errors.New("claim minimum idle time must be positive") + case c.ReadCount <= 0: + return errors.New("read count must be positive") + default: + return nil + } +} + +func (b *Broker) Ensure(ctx context.Context) error { + err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err() + if err != nil && !isBusyGroup(err) { + return fmt.Errorf("create redis stream consumer group: %w", err) + } + return nil +} + +func isBusyGroup(err error) bool { + return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP") +} + +func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() + envelope, err := DecodeEnvelope(payload) + if err != nil { + return "", err + } + payload, err = EncodeEnvelope(envelope) + if err != nil { + return "", err + } + + id, err := b.client.XAdd(ctx, &redis.XAddArgs{ + Stream: b.config.Stream, + Values: map[string]any{payloadField: string(payload)}, + }).Result() + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XADD failed") + return "", fmt.Errorf("publish redis stream job: %w", err) + } + span.SetAttributes(attribute.String("messaging.message.id", id)) + return id, nil +} + +func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() + streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: b.config.Group, + Consumer: b.config.Consumer, + Streams: []string{b.config.Stream, ">"}, + Count: b.config.ReadCount, + Block: b.config.Block, + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XREADGROUP failed") + return nil, fmt.Errorf("read redis stream jobs: %w", err) + } + return b.deliveries(streams) +} + +func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) { + messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: b.config.Stream, + Group: b.config.Group, + Consumer: b.config.Consumer, + MinIdle: b.config.ClaimMinIdle, + Start: start, + Count: b.config.ReadCount, + }).Result() + if err != nil { + return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err) + } + + deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}}) + if err != nil { + return nil, start, err + } + return deliveries, next, nil +} + +func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { + var deliveries []Delivery + for _, stream := range streams { + for _, message := range stream.Messages { + delivery := Delivery{ID: message.ID, broker: b} + payload, err := messagePayload(message) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.payload = boundedPayload(payload) + envelope, err := DecodeEnvelope(payload) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.Envelope = envelope + delivery.payload, err = EncodeEnvelope(envelope) + if err != nil { + return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err) + } + deliveries = append(deliveries, delivery) + } + } + return deliveries, nil +} + +func boundedPayload(payload []byte) []byte { + if len(payload) > failedPayloadMaxBytes { + payload = payload[:failedPayloadMaxBytes] + } + return append([]byte(nil), payload...) +} + +func messagePayload(message redis.XMessage) ([]byte, error) { + value, ok := message.Values[payloadField] + if !ok { + return nil, errors.New("payload field is missing") + } + switch payload := value.(type) { + case string: + return []byte(payload), nil + case []byte: + return payload, nil + default: + return nil, fmt.Errorf("payload has type %T", value) + } +} + +func (d Delivery) Ack(ctx context.Context) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, + ).Int64() + if err != nil { + return fmt.Errorf("ack redis stream job: %w", err) + } + if acked == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const ackOwnedScript = ` +local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[1], ARGV[2]) +end +return acked +` + +func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + if !failureCodePattern.MatchString(failure.Code) { + return errors.New("failure code is invalid") + } + if failure.FailedAt.IsZero() { + return errors.New("failure time is required") + } + + payload := d.payload + if d.DecodeError == nil && len(payload) == 0 { + var err error + payload, err = EncodeEnvelope(d.Envelope) + if err != nil { + return err + } + } + moved, err := d.broker.client.Eval(ctx, moveToFailedScript, + []string{d.broker.config.FailedStream, d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code, + failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt), + strconv.FormatInt(d.broker.config.FailedMaxLen, 10), + ).Int64() + if err != nil { + return fmt.Errorf("move redis stream job to failed stream: %w", err) + } + if moved == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const moveToFailedScript = ` +local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*', + 'payload', ARGV[4], + 'source_message_id', ARGV[2], + 'error_code', ARGV[5], + 'failed_at', ARGV[6], + 'attempt', ARGV[7]) +local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[2], ARGV[2]) +end +return acked +` diff --git a/internal/broker/redisstream/broker_test.go b/internal/broker/redisstream/broker_test.go new file mode 100644 index 0000000..5d96c42 --- /dev/null +++ b/internal/broker/redisstream/broker_test.go @@ -0,0 +1,93 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestBrokerDecodesDeliveries(t *testing.T) { + payload, err := EncodeEnvelope(validEnvelope()) + require.NoError(t, err) + + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: string(payload)}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Equal(t, "1-0", deliveries[0].ID) + require.Equal(t, validEnvelope(), deliveries[0].Envelope) +} + +func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) { + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: `{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"private/object" + }`}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Contains(t, string(deliveries[0].payload), `"storage_key":"private/object"`) +} + +func TestBrokerBoundsMalformedPayloadForFailedStream(t *testing.T) { + payload := make([]byte, failedPayloadMaxBytes+1) + for index := range payload { + payload[index] = 'x' + } + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: payload}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Len(t, deliveries[0].payload, failedPayloadMaxBytes) +} + +func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) { + broker := &Broker{} + _, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`)) + require.Error(t, err) +} + +func TestConfigValidation(t *testing.T) { + config := Config{ + Stream: "attachment-jobs", + Group: "attachment-workers", + Consumer: "worker-1", + FailedStream: "attachment-jobs-failed", + FailedMaxLen: 1000, + Block: time.Second, + ClaimMinIdle: time.Minute, + ReadCount: 10, + } + require.NoError(t, config.validate()) + + config.ClaimMinIdle = 0 + require.Error(t, config.validate()) +} diff --git a/internal/broker/redisstream/envelope.go b/internal/broker/redisstream/envelope.go new file mode 100644 index 0000000..44ffb4a --- /dev/null +++ b/internal/broker/redisstream/envelope.go @@ -0,0 +1,147 @@ +package redisstream + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +const ( + SchemaVersion = 1 + JobType = "attachment.process.v1" +) + +var ( + errInvalidEnvelope = errors.New("invalid job envelope") + traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +) + +type Envelope struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func (e Envelope) Validate() error { + switch { + case e.SchemaVersion != SchemaVersion: + return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope) + case e.Type != JobType: + return fmt.Errorf("%w: unsupported type", errInvalidEnvelope) + case e.JobID == uuid.Nil: + return fmt.Errorf("%w: job_id is required", errInvalidEnvelope) + case e.AttachmentID == uuid.Nil: + return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope) + case e.PipelineVersion < 1: + return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope) + case e.Attempt < 1: + return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope) + case e.CreatedAt.IsZero(): + return fmt.Errorf("%w: created_at is required", errInvalidEnvelope) + case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent): + return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope) + case utf8.RuneCountInString(e.Tracestate) > 512: + return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope) + default: + return nil + } +} + +func EncodeEnvelope(envelope Envelope) ([]byte, error) { + if err := envelope.Validate(); err != nil { + return nil, err + } + + payload, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("encode job envelope: %w", err) + } + + return payload, nil +} + +func DecodeEnvelope(payload []byte) (Envelope, error) { + var wire envelopeJSON + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err) + } + if err := ensureJSONEnd(decoder); err != nil { + return Envelope{}, err + } + + jobID, err := parseUUID(wire.JobID, "job_id") + if err != nil { + return Envelope{}, err + } + attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id") + if err != nil { + return Envelope{}, err + } + createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt) + if err != nil { + return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope) + } + envelope := Envelope{ + SchemaVersion: wire.SchemaVersion, + JobID: jobID, + Type: wire.Type, + AttachmentID: attachmentID, + PipelineVersion: wire.PipelineVersion, + Attempt: wire.Attempt, + CreatedAt: createdAt, + Traceparent: wire.Traceparent, + Tracestate: wire.Tracestate, + } + if err := envelope.Validate(); err != nil { + return Envelope{}, err + } + + return envelope, nil +} + +type envelopeJSON struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func parseUUID(value, field string) (uuid.UUID, error) { + if !uuidPattern.MatchString(value) { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + parsed, err := uuid.Parse(value) + if err != nil { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + return parsed, nil +} + +func ensureJSONEnd(decoder *json.Decoder) error { + var value any + if err := decoder.Decode(&value); !errors.Is(err, io.EOF) { + return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope) + } + return nil +} diff --git a/internal/broker/redisstream/envelope_test.go b/internal/broker/redisstream/envelope_test.go new file mode 100644 index 0000000..f24b68c --- /dev/null +++ b/internal/broker/redisstream/envelope_test.go @@ -0,0 +1,74 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestEnvelopeRoundTrip(t *testing.T) { + envelope := validEnvelope() + payload, err := EncodeEnvelope(envelope) + require.NoError(t, err) + + decoded, err := DecodeEnvelope(payload) + require.NoError(t, err) + require.Equal(t, envelope, decoded) +} + +func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) { + payload := []byte(`{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"must-not-enter-the-broker" + }`) + + _, err := DecodeEnvelope(payload) + require.Error(t, err) +} + +func TestEnvelopeValidation(t *testing.T) { + tests := []struct { + name string + mutate func(*Envelope) + }{ + {name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }}, + {name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }}, + {name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }}, + {name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }}, + {name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }}, + {name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }}, + {name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }}, + {name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }}, + {name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + envelope := validEnvelope() + tt.mutate(&envelope) + require.Error(t, envelope.Validate()) + }) + } +} + +func validEnvelope() Envelope { + return Envelope{ + SchemaVersion: SchemaVersion, + JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"), + Type: JobType, + AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"), + PipelineVersion: 1, + Attempt: 1, + CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC), + Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + Tracestate: "vendor=value", + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..977abc5 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,416 @@ +package config + +import ( + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/spf13/viper" +) + +type ServerConfig struct { + Address string + DatabaseURL string + CacheURL string + SessionTTL time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentBodyMemory int64 + AttachmentDecodeMemory int64 + PipelineVersion int +} + +type MigrationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + MigrationTimeout time.Duration +} + +type ReconciliationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration +} + +type databaseConfig struct { + url string + maxConns int + minConns int + maxConnectionAge time.Duration + maxConnectionIdle time.Duration + healthCheckPeriod time.Duration + dependencyTimeout time.Duration +} + +type objectStorageConfig struct { + endpoint string + region string + bucket string + accessKey string + secretKey string + useTLS bool + timeout time.Duration +} + +func LoadServer() (ServerConfig, error) { + values, err := loadValues() + if err != nil { + return ServerConfig{}, err + } + + host, err := requiredString(values, "server.host") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "server.port") + if err != nil { + return ServerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return ServerConfig{}, err + } + cacheURL, err := requiredString(values, "cache.url") + if err != nil { + return ServerConfig{}, err + } + store, err := requiredString(values, "server.session_store") + if err != nil { + return ServerConfig{}, err + } + if store != "valkey" { + return ServerConfig{}, errors.New("server.session_store must be valkey") + } + + sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour) + if err != nil { + return ServerConfig{}, err + } + idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour) + if err != nil { + return ServerConfig{}, err + } + requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour) + if err != nil { + return ServerConfig{}, err + } + shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024) + if err != nil { + return ServerConfig{}, err + } + passwordIterations, err := integer(values, "server.password.iterations", 1, 20) + if err != nil { + return ServerConfig{}, err + } + passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255) + if err != nil { + return ServerConfig{}, err + } + passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024) + if err != nil { + return ServerConfig{}, err + } + passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024) + if err != nil { + return ServerConfig{}, err + } + swaggerPort, err := requiredPort(values, "server.swagger_port") + if err != nil { + return ServerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) + if err != nil { + return ServerConfig{}, err + } + attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { + return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes) + } + attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + minimumDecodeMemory := int64(attachmentMaxPixels) * 8 + if int64(attachmentDecodeMemory) < minimumDecodeMemory { + return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory) + } + pipelineVersion, err := integer(values, "pipeline.version", 1, 1000) + if err != nil { + return ServerConfig{}, err + } + + return ServerConfig{ + Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, + SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, + DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + RequestTimeout: requestTimeout, + HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout, + HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout, + HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), + PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), + PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), + SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + AttachmentBodyMemory: int64(attachmentBodyMemory), + AttachmentDecodeMemory: int64(attachmentDecodeMemory), + PipelineVersion: pipelineVersion, + }, nil +} + +func LoadMigration() (MigrationConfig, error) { + values, err := loadValues() + if err != nil { + return MigrationConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return MigrationConfig{}, err + } + migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute) + if err != nil { + return MigrationConfig{}, err + } + return MigrationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + MigrationTimeout: migrationTimeout, + }, nil +} + +func LoadReconciliation() (ReconciliationConfig, error) { + values, err := loadValues() + if err != nil { + return ReconciliationConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + return ReconciliationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + }, nil +} + +func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { + endpoint, err := requiredString(values, "object_storage.endpoint") + if err != nil { + return objectStorageConfig{}, err + } + region, err := requiredString(values, "object_storage.region") + if err != nil { + return objectStorageConfig{}, err + } + bucket, err := requiredString(values, "object_storage.bucket") + if err != nil { + return objectStorageConfig{}, err + } + accessKey, err := requiredString(values, "object_storage.access_key") + if err != nil { + return objectStorageConfig{}, err + } + secretKey, err := requiredString(values, "object_storage.secret_key") + if err != nil { + return objectStorageConfig{}, err + } + useTLS, err := boolean(values, "object_storage.use_tls") + if err != nil { + return objectStorageConfig{}, err + } + timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) + if err != nil { + return objectStorageConfig{}, err + } + return objectStorageConfig{ + endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey, + secretKey: secretKey, useTLS: useTLS, timeout: timeout, + }, nil +} + +func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { + databaseURL, err := requiredString(values, "database.url") + if err != nil { + return databaseConfig{}, err + } + maxConns, err := integer(values, "database.max_connections", 1, 100) + if err != nil { + return databaseConfig{}, err + } + minConns, err := integer(values, "database.min_connections", 0, maxConns) + if err != nil { + return databaseConfig{}, err + } + maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour) + if err != nil { + return databaseConfig{}, err + } + maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour) + if err != nil { + return databaseConfig{}, err + } + healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour) + if err != nil { + return databaseConfig{}, err + } + dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute) + if err != nil { + return databaseConfig{}, err + } + return databaseConfig{ + url: databaseURL, maxConns: maxConns, minConns: minConns, + maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle, + healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout, + }, nil +} + +func requiredString(values *viper.Viper, name string) (string, error) { + value := strings.TrimSpace(values.GetString(name)) + if value == "" { + return "", fmt.Errorf("%s is required", name) + } + return value, nil +} + +func requiredPort(values *viper.Viper, name string) (string, error) { + value, err := requiredString(values, name) + if err != nil { + return "", err + } + if _, err := integerValue(value, 1, 65535); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return value, nil +} + +func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { + value, err := requiredString(values, name) + if err != nil { + return 0, err + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum) + } + return parsed, nil +} + +func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { + value, err := requiredString(values, name) + if err != nil { + return 0, err + } + parsed, err := integerValue(value, minimum, maximum) + if err != nil { + return 0, fmt.Errorf("%s: %w", name, err) + } + return parsed, nil +} + +func integerValue(value string, minimum, maximum int) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("must be between %d and %d", minimum, maximum) + } + return parsed, nil +} + +func boolean(values *viper.Viper, name string) (bool, error) { + value, err := requiredString(values, name) + if err != nil { + return false, err + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s: must be true or false", name) + } + return parsed, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..80d2216 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,131 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("APP_HOST", "") + t.Setenv("CACHE_URL", "") + t.Setenv("SWAGGER_PORT", "") + + settings, err := LoadReconciliation() + require.NoError(t, err) + require.Equal(t, "attachments", settings.S3Bucket) +} + +func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("WORKER_CONCURRENCY", "64") + + settings, err := LoadWorker() + require.NoError(t, err) + require.Equal(t, 2, settings.Concurrency) + require.Equal(t, 5*time.Second, settings.HeartbeatInterval) +} + +func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) { + content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1) + setConfigEnvironment(t, content) + + _, err := LoadWorker() + require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease") +} + +func TestLoadValuesRejectsUnknownVersion(t *testing.T) { + setConfigEnvironment(t, "version: 2\n") + + _, err := loadValues() + require.ErrorContains(t, err, "version: must be 1") +} + +func setConfigEnvironment(t *testing.T, content string) { + t.Helper() + path := filepath.Join(t.TempDir(), "messenger.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + t.Setenv("MESSENGER_CONFIG_PATH", path) + for name, value := range map[string]string{ + "APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081", + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + } { + t.Setenv(name, value) + } +} + +const validTestConfig = `version: 1 +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 +database: + min_connections: 0 + max_connections: 4 + max_connection_age: 5m + max_connection_idle: 1m + health_check_period: 30s + dependency_timeout: 5s +migration: + timeout: 30s +object_storage: + use_tls: false + timeout: 5s +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 +pipeline: + version: 1 +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 +monitoring: + metrics_address: 0.0.0.0:9091 +` diff --git a/internal/config/loader.go b/internal/config/loader.go new file mode 100644 index 0000000..6cd115c --- /dev/null +++ b/internal/config/loader.go @@ -0,0 +1,47 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/viper" +) + +const supportedConfigVersion = 1 + +var environmentKeys = map[string]string{ + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +func loadValues() (*viper.Viper, error) { + path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH")) + if path == "" { + return nil, errors.New("MESSENGER_CONFIG_PATH is required") + } + values := viper.New() + values.SetConfigFile(path) + if err := values.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read messenger config: %w", err) + } + if version := values.GetInt("version"); version != supportedConfigVersion { + return nil, fmt.Errorf("version: must be %d", supportedConfigVersion) + } + for key, name := range environmentKeys { + if err := values.BindEnv(key, name); err != nil { + return nil, fmt.Errorf("bind %s: %w", name, err) + } + } + return values, nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go new file mode 100644 index 0000000..76b4f5d --- /dev/null +++ b/internal/config/pipeline.go @@ -0,0 +1,307 @@ +package config + +import ( + "errors" + "fmt" + "net" + "time" + + "github.com/spf13/viper" +) + +type DispatcherConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int + MetricsAddress string +} + +type WorkerConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + Concurrency int + Prefetch int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReadBlock time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + MemoryBytes int64 + MetricsAddress string +} + +type JobAdminConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration +} + +type brokerConfig struct { + url string + stream string + consumerGroup string + failedStream string + failedStreamMaxLength int +} + +func LoadJobAdmin() (JobAdminConfig, error) { + values, err := loadValues() + if err != nil { + return JobAdminConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return JobAdminConfig{}, err + } + return JobAdminConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + }, nil +} + +func LoadDispatcher() (DispatcherConfig, error) { + values, err := loadValues() + if err != nil { + return DispatcherConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + batchSize, err := integer(values, "outbox.batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return DispatcherConfig{}, err + } + return DispatcherConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, + FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, + Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, + MetricsAddress: metricsAddress, + }, nil +} + +func LoadWorker() (WorkerConfig, error) { + values, err := loadValues() + if err != nil { + return WorkerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return WorkerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return WorkerConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return WorkerConfig{}, err + } + concurrency, err := integer(values, "worker.concurrency", 1, 64) + if err != nil { + return WorkerConfig{}, err + } + prefetch, err := integer(values, "worker.prefetch", concurrency, 1024) + if err != nil { + return WorkerConfig{}, err + } + leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if heartbeatInterval >= leaseDuration { + return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease") + } + processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if processingTimeout >= leaseDuration { + return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease") + } + readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute) + if err != nil { + return WorkerConfig{}, err + } + reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + maxAttempts, err := integer(values, "worker.max_attempts", 1, 100) + if err != nil { + return WorkerConfig{}, err + } + retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if retryMax < retryBase { + return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base") + } + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) + if err != nil { + return WorkerConfig{}, err + } + memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return WorkerConfig{}, err + } + return WorkerConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, + S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream, + ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, + FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, + HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, + }, nil +} + +func metricsAddress(values *viper.Viper) (string, error) { + address, err := requiredString(values, "monitoring.metrics_address") + if err != nil { + return "", err + } + _, port, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + return address, nil +} + +func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { + url, err := requiredString(values, "broker.url") + if err != nil { + return brokerConfig{}, err + } + stream, err := requiredString(values, "broker.stream") + if err != nil { + return brokerConfig{}, err + } + consumerGroup, err := requiredString(values, "broker.consumer_group") + if err != nil { + return brokerConfig{}, err + } + failedStream, err := requiredString(values, "broker.failed_stream") + if err != nil { + return brokerConfig{}, err + } + failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000) + if err != nil { + return brokerConfig{}, err + } + return brokerConfig{ + url: url, stream: stream, consumerGroup: consumerGroup, + failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength, + }, nil +} diff --git a/internal/database/database.go b/internal/database/database.go new file mode 100644 index 0000000..c631b06 --- /dev/null +++ b/internal/database/database.go @@ -0,0 +1,94 @@ +package database + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" +) + +var ( + errURLRequired = errors.New("database URL is required") + errMinConnections = errors.New("minimum database connections must not be negative") + errMaxConnections = errors.New("maximum database connections must be positive") + errConnectionRange = errors.New("minimum database connections exceed maximum") + errMaxConnectionAge = errors.New("maximum database connection age must be positive") + errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive") + errHealthCheckPeriod = errors.New("database health check period must be positive") + errConnectionTimeout = errors.New("database connection timeout must be positive") +) + +type Config struct { + URL string + MinConnections int32 + MaxConnections int32 + MaxConnectionAge time.Duration + MaxConnectionIdle time.Duration + HealthCheckPeriod time.Duration + ConnectTimeout time.Duration +} + +func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + poolConfig, err := pgxpool.ParseConfig(config.URL) + if err != nil { + return nil, fmt.Errorf("parse database configuration: %w", err) + } + poolConfig.ConnConfig.Tracer = queryTracer{} + poolConfig.MinConns = config.MinConnections + poolConfig.MaxConns = config.MaxConnections + poolConfig.MaxConnLifetime = config.MaxConnectionAge + poolConfig.MaxConnIdleTime = config.MaxConnectionIdle + poolConfig.HealthCheckPeriod = config.HealthCheckPeriod + + connectCtx, cancel := context.WithTimeout(ctx, config.ConnectTimeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(connectCtx, poolConfig) + if err != nil { + return nil, fmt.Errorf("create database pool: %w", err) + } + if err := pool.Ping(connectCtx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + + return pool, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error { + if timeout <= 0 { + return errors.New("database migration timeout must be positive") + } + migrationCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return applyMigrations(migrationCtx, pool, migrations.FS) +} + +func validateConfig(config Config) error { + switch { + case config.URL == "": + return errURLRequired + case config.MinConnections < 0: + return errMinConnections + case config.MaxConnections < 1: + return errMaxConnections + case config.MinConnections > config.MaxConnections: + return errConnectionRange + case config.MaxConnectionAge <= 0: + return errMaxConnectionAge + case config.MaxConnectionIdle <= 0: + return errMaxConnectionIdle + case config.HealthCheckPeriod <= 0: + return errHealthCheckPeriod + case config.ConnectTimeout <= 0: + return errConnectionTimeout + default: + return nil + } +} diff --git a/internal/database/migrations.go b/internal/database/migrations.go new file mode 100644 index 0000000..095c93d --- /dev/null +++ b/internal/database/migrations.go @@ -0,0 +1,150 @@ +package database + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + migrationLockID int64 = 0x4d54555349324c42 + migrationUnlockTimeout = 5 * time.Second +) + +var migrationNamePattern = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.sql$`) + +type migration struct { + version int64 + name string + checksum string + sql string +} + +func applyMigrations(ctx context.Context, pool *pgxpool.Pool, migrationFS fs.FS) (returnErr error) { + migrations, err := readMigrations(migrationFS) + if err != nil { + return err + } + connection, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection for migrations: %w", err) + } + defer connection.Release() + if _, err := connection.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil { + return fmt.Errorf("lock migrations: %w", err) + } + defer func() { + unlockCtx, cancel := context.WithTimeout(context.Background(), migrationUnlockTimeout) + defer cancel() + if _, err := connection.Exec(unlockCtx, "SELECT pg_advisory_unlock($1)", migrationLockID); err != nil { + _ = connection.Conn().Close(unlockCtx) + if returnErr == nil { + returnErr = fmt.Errorf("unlock migrations: %w", err) + } + } + }() + + if _, err := connection.Exec(ctx, ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +)`); err != nil { + return fmt.Errorf("create migration history: %w", err) + } + for _, migration := range migrations { + if err := applyMigration(ctx, connection, migration); err != nil { + return err + } + } + return nil +} + +func readMigrations(migrationFS fs.FS) ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, ".") + if err != nil { + return nil, fmt.Errorf("read embedded migrations: %w", err) + } + migrations := make([]migration, 0, len(entries)) + versions := make(map[int64]string) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".sql" { + continue + } + matches := migrationNamePattern.FindStringSubmatch(entry.Name()) + if matches == nil { + return nil, fmt.Errorf("invalid migration filename %q", entry.Name()) + } + version, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil || version < 1 { + return nil, fmt.Errorf("invalid migration version in %q", entry.Name()) + } + if previous, exists := versions[version]; exists { + return nil, fmt.Errorf("migration version %d is used by %q and %q", version, previous, entry.Name()) + } + data, err := fs.ReadFile(migrationFS, entry.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", entry.Name(), err) + } + hash := sha256.Sum256(data) + versions[version] = entry.Name() + migrations = append(migrations, migration{ + version: version, + name: entry.Name(), + checksum: hex.EncodeToString(hash[:]), + sql: string(data), + }) + } + if len(migrations) == 0 { + return nil, errors.New("no database migrations found") + } + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].version < migrations[j].version + }) + return migrations, nil +} + +func applyMigration(ctx context.Context, connection *pgxpool.Conn, migration migration) error { + var checksum string + err := connection.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version = $1", migration.version).Scan(&checksum) + switch { + case err == nil && checksum != migration.checksum: + return fmt.Errorf("migration %q checksum does not match applied version", migration.name) + case err == nil: + return nil + case !errors.Is(err, pgx.ErrNoRows): + return fmt.Errorf("read migration %q state: %w", migration.name, err) + } + + transaction, err := connection.Begin(ctx) + if err != nil { + return fmt.Errorf("begin migration %q: %w", migration.name, err) + } + defer func() { + _ = transaction.Rollback(context.WithoutCancel(ctx)) + }() + if _, err := transaction.Exec(ctx, migration.sql, pgx.QueryExecModeSimpleProtocol); err != nil { + return fmt.Errorf("apply migration %q: %w", migration.name, err) + } + if _, err := transaction.Exec(ctx, ` +INSERT INTO schema_migrations (version, name, checksum) +VALUES ($1, $2, $3)`, migration.version, migration.name, migration.checksum); err != nil { + return fmt.Errorf("record migration %q: %w", migration.name, err) + } + if err := transaction.Commit(ctx); err != nil { + return fmt.Errorf("commit migration %q: %w", migration.name, err) + } + return nil +} diff --git a/internal/database/telemetry.go b/internal/database/telemetry.go new file mode 100644 index 0000000..e2c34f8 --- /dev/null +++ b/internal/database/telemetry.go @@ -0,0 +1,28 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type queryTracer struct{} + +func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "postgresql"))) + return ctx +} + +func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + span := trace.SpanFromContext(ctx) + if data.Err != nil { + span.RecordError(data.Err) + span.SetStatus(codes.Error, "query failed") + } + span.End() +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..ab81366 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,10 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New() *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stdout, nil)) +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go new file mode 100644 index 0000000..07a91a4 --- /dev/null +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -0,0 +1,106 @@ +package pipeline + +import ( + "context" + "net/http" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + +type Metrics struct { + meter metric.Meter + workerInFlight metric.Int64UpDownCounter + workerJobs metric.Int64Counter + workerDuration metric.Float64Histogram + workerTimeToReady metric.Float64Histogram + workerRetries metric.Int64Counter + workerReclaimed metric.Int64Counter + outboxPublished metric.Int64Counter + outboxPublishFailure metric.Int64Counter + outboxRecovered metric.Int64Counter + handler http.Handler +} + +func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics { + if meter == nil { + meter = otel.Meter("messenger-pipeline") + } + if handler == nil { + handler = http.NotFoundHandler() + } + workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker.")) + workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result.")) + workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration", + metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), + ) + workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready", + metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), + ) + workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) + workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) + outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) + outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed.")) + outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication.")) + if workerCapacity > 0 { + _, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity", + metric.WithDescription("Maximum concurrent attachment jobs for this worker."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(int64(workerCapacity)) + return nil + }), + ) + } + return &Metrics{ + meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, + workerDuration: workerDuration, workerTimeToReady: workerTimeToReady, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, + outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, + outboxRecovered: outboxRecovered, handler: handler, + } +} + +func (m *Metrics) Handler() http.Handler { + return m.handler +} + +func (m *Metrics) ObserveStarted(ctx context.Context) { + m.workerInFlight.Add(ctx, 1) +} + +func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) { + attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result)) + m.workerInFlight.Add(ctx, -1) + m.workerJobs.Add(ctx, 1, attributes) + m.workerDuration.Record(ctx, duration.Seconds(), attributes) +} + +func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) { + m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process"))) +} + +func (m *Metrics) ObserveRetry(ctx context.Context) { + m.workerRetries.Add(ctx, 1) +} + +func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) { + m.workerReclaimed.Add(ctx, int64(count)) +} + +func (m *Metrics) ObservePublished(ctx context.Context) { + m.outboxPublished.Add(ctx, 1) +} + +func (m *Metrics) ObservePublishFailure(ctx context.Context) { + m.outboxPublishFailure.Add(ctx, 1) +} + +func (m *Metrics) ObserveRecovered(ctx context.Context, count int) { + m.outboxRecovered.Add(ctx, int64(count)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go new file mode 100644 index 0000000..38e5bdc --- /dev/null +++ b/internal/messenger/attachment/service.go @@ -0,0 +1,237 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "mime" + "net/http" + "time" + + "github.com/google/uuid" + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Repository interface { + ChatForUser(context.Context, string, string) (model.Chat, error) + CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) + QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error) + DeletePendingAttachment(context.Context, string, string) error + AttachmentForUser(context.Context, string, string) (model.Attachment, error) + AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) +} + +type ObjectStore interface { + Put(context.Context, string, string, []byte) error + Get(context.Context, string) (io.ReadCloser, error) + Delete(context.Context, string) error +} + +type Config struct { + MaxBytes int64 + MaxPixels int64 + BodyMemoryBytes int64 + DecodeMemoryBytes int64 + CleanupTimeout time.Duration + PipelineVersion int +} + +type Upload struct { + ChatID string + OwnerID string + Filename string + ContentType string + ContentLength int64 + Body io.Reader + TraceContext telemetry.MessageCarrier +} + +type Content struct { + Attachment model.Attachment + Variant model.Variant + Body io.ReadCloser +} + +type Service struct { + repository Repository + objects ObjectStore + config Config + bodyMemory *semaphore.Weighted + decodeMemory *semaphore.Weighted + logger *slog.Logger +} + +func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 { + return nil, errors.New("invalid attachment service configuration") + } + if config.BodyMemoryBytes < config.MaxBytes { + return nil, errors.New("attachment body memory must fit one maximum upload") + } + if config.DecodeMemoryBytes < config.MaxPixels*8 { + return nil, errors.New("attachment decode memory must fit one maximum image") + } + return &Service{ + repository: repository, objects: objects, config: config, + bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes), + decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger, + }, nil +} + +func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) { + if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil { + return model.Attachment{}, mapDependencyError(err) + } + data, contentType, release, err := s.readImage(input) + if err != nil { + return model.Attachment{}, err + } + defer release() + + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/" + uuid.NewString() + digest := sha256.Sum256(data) + pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, + PipelineVersion: s.config.PipelineVersion, + }) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { + s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(ctx, pending.ID, stagingKey) + return model.Attachment{}, model.ErrUnavailable + } + createdAt := time.Now().UTC() + queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, + Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, + }, input.TraceContext.Traceparent, input.TraceContext.Tracestate) + if err != nil { + s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + return model.Attachment{}, mapDependencyError(err) + } + return queued, nil +} + +func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout) + defer cancel() + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { + s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + } +} + +func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + return attachment, nil +} + +func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) { + attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName) + if err != nil { + return Content{}, mapDependencyError(err) + } + body, err := s.objects.Get(ctx, variant.ObjectKey) + if err != nil { + return Content{}, model.ErrUnavailable + } + return Content{Attachment: attachment, Variant: variant, Body: body}, nil +} + +func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { + mediaType, parameters, err := mime.ParseMediaType(input.ContentType) + if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { + return nil, "", nil, model.ErrUnsupportedMedia + } + if input.ContentLength > s.config.MaxBytes { + return nil, "", nil, model.ErrPayloadTooLarge + } + reservation := s.config.MaxBytes + if !s.bodyMemory.TryAcquire(reservation) { + return nil, "", nil, model.ErrCapacity + } + releaseBody := func() { s.bodyMemory.Release(reservation) } + data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1)) + if err != nil { + releaseBody() + return nil, "", nil, model.ErrInvalidInput + } + if int64(len(data)) > s.config.MaxBytes { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + if len(data) == 0 { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + detected := http.DetectContentType(data) + if detected != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || imageContentType(format) != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) + decodeBytes := pixels * 8 + if !s.decodeMemory.TryAcquire(decodeBytes) { + releaseBody() + return nil, "", nil, model.ErrCapacity + } + if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { + s.decodeMemory.Release(decodeBytes) + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + release := func() { + s.decodeMemory.Release(decodeBytes) + releaseBody() + } + return data, mediaType, release, nil +} + +func imageContentType(format string) string { + switch format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + default: + return "" + } +} + +func mapDependencyError(err error) error { + switch { + case errors.Is(err, model.ErrNotFound): + return model.ErrNotFound + case errors.Is(err, model.ErrConflict): + return model.ErrConflict + default: + return model.ErrUnavailable + } +} diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go new file mode 100644 index 0000000..296f283 --- /dev/null +++ b/internal/messenger/attachment/service_test.go @@ -0,0 +1,220 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func TestUploadLifecycle(t *testing.T) { + data := pngFixture(t, 2, 2) + events := make([]string, 0, 3) + repository := &repositoryStub{events: &events} + objects := &objectStub{events: &events} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentQueued, result.Status) + require.Equal(t, []string{"pending", "put", "queued"}, events) +} + +func TestUploadValidation(t *testing.T) { + data := pngFixture(t, 2, 2) + tests := []struct { + name string + contentType string + data []byte + maxBytes int64 + maxPixels int64 + want error + }{ + {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge}, + {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge}, + {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels) + input := upload(test.data, test.contentType) + _, err := service.Upload(context.Background(), input) + require.ErrorIs(t, err, test.want) + }) + } +} + +func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) { + data := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + require.True(t, service.bodyMemory.TryAcquire(1024)) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.bodyMemory.Release(1024) + + require.True(t, service.decodeMemory.TryAcquire(32)) + _, err = service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.decodeMemory.Release(32) +} + +func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) { + oversized := pngFixture(t, 3, 2) + valid := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + + _, err := service.Upload(context.Background(), upload(oversized, "image/png")) + require.ErrorIs(t, err, model.ErrPayloadTooLarge) + _, err = service.Upload(context.Background(), upload(valid, "image/png")) + require.NoError(t, err) +} + +func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { + _, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, + Config{ + MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1, + }, + ) + require.Error(t, err) +} + +func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{} + objects := &objectStub{putErr: errors.New("s3 unavailable")} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.True(t, repository.pendingDeleted) +} + +func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{queueErr: errors.New("connection lost")} + objects := &objectStub{} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.False(t, objects.deleted) + require.False(t, repository.pendingDeleted) +} + +func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { + t.Helper() + service, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, + Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1, + }, + ) + require.NoError(t, err) + return service +} + +func upload(data []byte, contentType string) Upload { + return Upload{ + ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType, + ContentLength: int64(len(data)), Body: bytes.NewReader(data), + } +} + +type repositoryStub struct { + events *[]string + queueErr error + pendingDeleted bool + pending model.Attachment +} + +func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} + +func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) { + attachment.Status = model.AttachmentPending + s.pending = attachment + s.record("pending") + return attachment, nil +} + +func (s *repositoryStub) QueueAttachmentWithJobAndOutbox( + context.Context, + string, + string, + model.AttachmentJob, + string, + string, +) (model.Attachment, error) { + if s.queueErr != nil { + return model.Attachment{}, s.queueErr + } + s.pending.Status = model.AttachmentQueued + s.record("queued") + return s.pending, nil +} + +func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error { + s.pendingDeleted = true + return nil +} + +func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { + return s.pending, nil +} + +func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) { + return s.pending, model.Variant{}, nil +} + +func (s *repositoryStub) record(event string) { + if s.events != nil { + *s.events = append(*s.events, event) + } +} + +type objectStub struct { + events *[]string + putErr error + deleted bool +} + +func (s *objectStub) Put(context.Context, string, string, []byte) error { + if s.events != nil { + *s.events = append(*s.events, "put") + } + return s.putErr +} + +func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (s *objectStub) Delete(context.Context, string) error { + s.deleted = true + return nil +} + +func pngFixture(t *testing.T, width, height int) []byte { + t.Helper() + var data bytes.Buffer + value := image.NewRGBA(image.Rect(0, 0, width, height)) + value.Set(0, 0, color.White) + require.NoError(t, png.Encode(&data, value)) + return data.Bytes() +} diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go new file mode 100644 index 0000000..53e532b --- /dev/null +++ b/internal/messenger/attachment/transform/exif.go @@ -0,0 +1,169 @@ +package transform + +import ( + "context" + "encoding/binary" + "errors" + "image" + "image/color" +) + +var errInvalidEXIF = errors.New("invalid EXIF metadata") + +func jpegOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + return 0, errInvalidEXIF + } + for offset := 2; offset < len(data); { + if err := contextError(ctx); err != nil { + return 0, err + } + if data[offset] != 0xff { + return 0, errInvalidEXIF + } + for offset < len(data) && data[offset] == 0xff { + offset++ + } + if offset >= len(data) { + return 0, errInvalidEXIF + } + marker := data[offset] + offset++ + if marker == 0xd9 || marker == 0xda { + return 1, nil + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + if offset+2 > len(data) { + return 0, errInvalidEXIF + } + length := int(binary.BigEndian.Uint16(data[offset : offset+2])) + if length < 2 || offset+length > len(data) { + return 0, errInvalidEXIF + } + payload := data[offset+2 : offset+length] + offset += length + if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { + continue + } + return tiffOrientation(ctx, payload[6:]) + } + return 1, nil +} + +func tiffOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 8 { + return 0, errInvalidEXIF + } + var order binary.ByteOrder + switch string(data[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 0, errInvalidEXIF + } + if order.Uint16(data[2:4]) != 42 { + return 0, errInvalidEXIF + } + ifdOffset := uint64(order.Uint32(data[4:8])) + if ifdOffset > uint64(len(data)-2) { + return 0, errInvalidEXIF + } + count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2])) + entriesStart := ifdOffset + 2 + if count > (uint64(len(data))-entriesStart)/12 { + return 0, errInvalidEXIF + } + for index := uint64(0); index < count; index++ { + if err := contextError(ctx); err != nil { + return 0, err + } + entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] + if order.Uint16(entry[:2]) != 0x0112 { + continue + } + if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 { + return 0, errInvalidEXIF + } + orientation := int(order.Uint16(entry[8:10])) + if orientation < 1 || orientation > 8 { + return 0, errInvalidEXIF + } + return orientation, nil + } + return 1, nil +} + +func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + outputWidth, outputHeight := width, height + if orientation >= 5 { + outputWidth, outputHeight = height, width + } + output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) + for y := 0; y < outputHeight; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < outputWidth; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) + output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) + } + } + return output, nil +} + +func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { + switch orientation { + case 2: + return width - 1 - x, y + case 3: + return width - 1 - x, height - 1 - y + case 4: + return x, height - 1 - y + case 5: + return y, x + case 6: + return y, height - 1 - x + case 7: + return width - 1 - y, height - 1 - x + case 8: + return width - 1 - y, x + default: + return x, y + } +} + +func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() + output.SetNRGBA(x, y, color.NRGBA{ + R: uint8((r + 0xffff - a) / 0x101), + G: uint8((g + 0xffff - a) / 0x101), + B: uint8((b + 0xffff - a) / 0x101), + A: 0xff, + }) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go new file mode 100644 index 0000000..8a6dd08 --- /dev/null +++ b/internal/messenger/attachment/transform/processor.go @@ -0,0 +1,272 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/jpeg" + _ "image/png" + "io" + "math" + "net/http" + + "github.com/deepteams/webp" +) + +const ( + // DefaultJPEGQuality is used for thumbnail and crop variants. + DefaultJPEGQuality = 85 + // DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size. + DefaultWebPQuality = 82 + // DefaultWebPMethod selects the encoder's balanced quality and CPU mode. + DefaultWebPMethod = 4 +) + +var ( + ErrInvalidConfig = errors.New("invalid image transform configuration") + ErrUnsupportedMedia = errors.New("unsupported image media type") + ErrPayloadTooLarge = errors.New("image exceeds configured limits") + ErrInvalidImage = errors.New("invalid image data") +) + +type Config struct { + MaxBytes int64 + MaxPixels int64 + JPEGQuality int + WebPQuality float32 + WebPMethod int +} + +type Variant struct { + Bytes []byte + ContentType string + Width int + Height int + SHA256 string +} + +type Result struct { + Original Variant + Thumbnail Variant + Crop Variant + Optimized Variant + Orientation int +} + +type Processor struct { + config Config +} + +func DefaultConfig(maxBytes, maxPixels int64) Config { + return Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, + JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod, + } +} + +func NewProcessor(config Config) (*Processor, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 || + config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 { + return nil, ErrInvalidConfig + } + return &Processor{config: config}, nil +} + +func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) { + if err := contextError(ctx); err != nil { + return Result{}, err + } + img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + oriented, err := applyOrientation(ctx, img, orientation) + if err != nil { + return Result{}, err + } + opaque, err := flattenWhite(ctx, oriented) + if err != nil { + return Result{}, err + } + result := Result{ + Original: variant(data, format, width, height), + Orientation: orientation, + } + + thumbnail, err := fitContext(ctx, opaque, 256, 256, false) + if err != nil { + return Result{}, err + } + result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + crop := centeredSquare(opaque) + crop, err = resizeContext(ctx, crop, 128, 128) + if err != nil { + return Result{}, err + } + result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + optimized, err := fitContext(ctx, opaque, 1280, 1280, false) + if err != nil { + return Result{}, err + } + result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + return result, nil +} + +func (p *Processor) decode( + ctx context.Context, + data []byte, + declaredContentType string, +) (image.Image, string, int, int, int, error) { + if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + if len(data) == 0 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(len(data)) > p.config.MaxBytes { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + if http.DetectContentType(data) != declaredContentType { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || decodedFormat != format { + return nil, "", 0, 0, 0, ErrInvalidImage + } + orientation := 1 + if format == "jpeg" { + orientation, err = jpegOrientation(ctx, data) + if err != nil { + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + return nil, "", 0, 0, 0, ErrInvalidImage + } + } + return img, declaredContentType, config.Width, config.Height, orientation, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + if err := contextError(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + +func contextError(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func contentType(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + default: + return "" + } +} + +func encodeJPEG(img image.Image, quality int) (Variant, error) { + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil +} + +func encodeWebP(img image.Image, quality float32, method int) (Variant, error) { + var output bytes.Buffer + options := &webp.EncoderOptions{Quality: quality, Method: method} + if err := webp.Encode(&output, img, options); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil +} + +func variant(data []byte, contentType string, width, height int) Variant { + digest := sha256.Sum256(data) + return Variant{ + Bytes: data, ContentType: contentType, Width: width, Height: height, + SHA256: hex.EncodeToString(digest[:]), + } +} + +// EstimateMemoryWeight returns a conservative reservation for the input, decoded source, +// orientation and alpha copies, variant working buffer, and encoded variant buffers. +func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) { + if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) { + return 0, ErrInvalidImage + } + pixels := int64(width) * int64(height) + optimizedPixels := fittedPixels(width, height, 1280, 1280) + thumbnailPixels := fittedPixels(width, height, 256, 256) + const cropPixels int64 = 128 * 128 + if pixels > (math.MaxInt64-inputBytes)/16 { + return 0, ErrPayloadTooLarge + } + weight := inputBytes + pixels*16 + variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4 + if variants > math.MaxInt64-weight { + return 0, ErrPayloadTooLarge + } + return weight + variants, nil +} + +func fittedPixels(width, height, maxWidth, maxHeight int) int64 { + newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false) + return int64(newWidth) * int64(newHeight) +} diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go new file mode 100644 index 0000000..7b595dd --- /dev/null +++ b/internal/messenger/attachment/transform/processor_test.go @@ -0,0 +1,261 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" + + "github.com/deepteams/webp" +) + +func TestProcessProducesExpectedVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 320*200) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Original, "image/png", 320, 200) + assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) + assertVariant(t, result.Optimized, "image/webp", 320, 200) + if !bytes.Equal(result.Original.Bytes, data) { + t.Fatal("original bytes changed") + } + digest := sha256.Sum256(data) + if result.Original.SHA256 != hex.EncodeToString(digest[:]) { + t.Fatalf("unexpected original digest: %s", result.Original.SHA256) + } + if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil { + t.Fatalf("decode optimized WebP: %v", err) + } +} + +func TestProcessDoesNotUpscaleFitVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 64, 32)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 64*32) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32) + assertVariant(t, result.Optimized, "image/webp", 64, 32) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) +} + +func TestFlattenWhiteCompositesAlpha(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) + source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) + + result, err := flattenWhite(context.Background(), source) + if err != nil { + t.Fatalf("flatten image: %v", err) + } + if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { + t.Fatalf("transparent pixel: got %#v", got) + } + if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) { + t.Fatalf("semi-transparent pixel: got %#v", got) + } +} + +func TestProcessAppliesEXIFOrientation(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + fillGradient(source) + data := injectOrientation(t, encodeJPEGForTest(t, source), 6) + processor := newTestProcessor(t, int64(len(data))+1, 3*2) + + result, err := processor.Process(context.Background(), data, "image/jpeg") + if err != nil { + t.Fatalf("process image: %v", err) + } + + if result.Orientation != 6 { + t.Fatalf("orientation: got %d, want 6", result.Orientation) + } + assertVariant(t, result.Original, "image/jpeg", 3, 2) + assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3) + assertVariant(t, result.Optimized, "image/webp", 2, 3) +} + +func TestProcessRejectsInvalidInputs(t *testing.T) { + valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10))) + + tests := []struct { + name string + data []byte + contentType string + maxPixels int64 + want error + }{ + {name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge}, + {name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels) + _, err := processor.Process(context.Background(), test.data, test.contentType) + if !errors.Is(err, test.want) { + t.Fatalf("error: got %v, want %v", err, test.want) + } + }) + } +} + +func TestProcessIsDeterministic(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 48, 31)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 48*31) + + first, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("first process: %v", err) + } + second, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("second process: %v", err) + } + + for name, variants := range map[string][2]Variant{ + "thumbnail": {first.Thumbnail, second.Thumbnail}, + "crop": {first.Crop, second.Crop}, + "optimized": {first.Optimized, second.Optimized}, + } { + if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) { + t.Fatalf("%s bytes differ", name) + } + } +} + +func TestProcessHonorsCanceledContext(t *testing.T) { + processor := newTestProcessor(t, 1024, 100) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := processor.Process(ctx, []byte("unused"), "image/png") + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +func TestResizeStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := cancelingImage{ + Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)), + cancel: cancel, + } + _, err := resizeContext(ctx, source, 128, 128) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +type cancelingImage struct { + image.Image + cancel context.CancelFunc +} + +func (i cancelingImage) At(x, y int) color.Color { + i.cancel() + return i.Image.At(x, y) +} + +func TestEstimateMemoryWeight(t *testing.T) { + weight, err := EstimateMemoryWeight(1000, 320, 200) + if err != nil { + t.Fatalf("estimate memory: %v", err) + } + want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4) + if weight != want { + t.Fatalf("weight: got %d, want %d", weight, want) + } +} + +func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor { + t.Helper() + processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels)) + if err != nil { + t.Fatalf("new processor: %v", err) + } + return processor +} + +func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) { + t.Helper() + if variant.ContentType != contentType || variant.Width != width || variant.Height != height { + t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d", + variant.ContentType, variant.Width, variant.Height, contentType, width, height) + } + if len(variant.Bytes) == 0 || variant.SHA256 == "" { + t.Fatal("variant has no bytes or digest") + } +} + +func fillGradient(img *image.NRGBA) { + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } +} + +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return output.Bytes() +} + +func encodeJPEGForTest(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil { + t.Fatalf("encode JPEG: %v", err) + } + return output.Bytes() +} + +func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte { + t.Helper() + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + t.Fatal("not a JPEG") + } + tiff := make([]byte, 26) + copy(tiff[:2], "II") + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + payload := append([]byte("Exif\x00\x00"), tiff...) + segment := []byte{0xff, 0xe1, 0, 0} + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + result := make([]byte, 0, len(data)+len(segment)+len(payload)) + result = append(result, data[:2]...) + result = append(result, segment...) + result = append(result, payload...) + return append(result, data[2:]...) +} diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go new file mode 100644 index 0000000..2d86bb8 --- /dev/null +++ b/internal/messenger/attachment/transform/resize.go @@ -0,0 +1,128 @@ +package transform + +import ( + "context" + "image" + "image/color" +) + +func fitContext( + ctx context.Context, + source image.Image, + maxWidth int, + maxHeight int, + upscale bool, +) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) + return resizeContext(ctx, source, width, height) +} + +func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { + if !upscale && width <= maxWidth && height <= maxHeight { + return width, height + } + if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) { + newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width))) + return maxWidth, newHeight + } + newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height))) + return newWidth, maxHeight +} + +func centeredSquare(source image.Image) image.Image { + bounds := source.Bounds() + size := min(bounds.Dx(), bounds.Dy()) + left := bounds.Min.X + (bounds.Dx()-size)/2 + top := bounds.Min.Y + (bounds.Dy()-size)/2 + return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)} +} + +type croppedImage struct { + source image.Image + bounds image.Rectangle + offset image.Point +} + +func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() } +func (i croppedImage) Bounds() image.Rectangle { return i.bounds } +func (i croppedImage) At(x, y int) color.Color { + return i.source.At(i.offset.X+x, i.offset.Y+y) +} + +func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) { + bounds := source.Bounds() + if bounds.Dx() == width && bounds.Dy() == height { + return copyNRGBAContext(ctx, source) + } + output := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) + for x := 0; x < width; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) + c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) + c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) + c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA) + c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA) + output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) + } + } + return output, nil +} + +func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { + coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15 + if coordinate <= 0 { + return 0, min(1, sourceSize-1), 0 + } + maximum := int64(sourceSize-1) << 16 + if coordinate >= maximum { + return sourceSize - 1, sourceSize - 1, 0 + } + first := int(coordinate >> 16) + return first, first + 1, uint32(coordinate & 0xffff) +} + +func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA { + return color.NRGBA{ + R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy), + G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy), + B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy), + A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy), + } +} + +func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { + const one uint64 = 1 << 16 + top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx) + bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx) + value := top*(one-uint64(fy)) + bottom*uint64(fy) + return uint8((value + 1<<31) >> 32) +} + +func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go new file mode 100644 index 0000000..45ff600 --- /dev/null +++ b/internal/messenger/attachment/worker/worker.go @@ -0,0 +1,543 @@ +package worker + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "path" + "strings" + "sync" + "time" + + "golang.org/x/sync/semaphore" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Repository interface { + AttachmentJob(context.Context, string) (model.AttachmentJob, error) + ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) + CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error) + FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) + RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) +} + +type Broker interface { + Read(context.Context) ([]redisstream.Delivery, error) + Reclaim(context.Context, string) ([]redisstream.Delivery, string, error) +} + +type ObjectStore interface { + Get(context.Context, string) (io.ReadCloser, error) + Put(context.Context, string, string, []byte) error + Delete(context.Context, string) error +} + +type ImageProcessor interface { + Process(context.Context, []byte, string) (transform.Result, error) +} + +type Metrics interface { + ObserveStarted(context.Context) + ObserveFinished(context.Context, string, time.Duration) + ObserveTimeToReady(context.Context, time.Duration) + ObserveRetry(context.Context) + ObserveReclaimed(context.Context, int) +} + +type Config struct { + WorkerID string + Concurrency int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + MaxBytes int64 + MaxPixels int64 + MemoryBytes int64 +} + +type FaultPoint string + +const ( + FaultNone FaultPoint = "" + FaultBeforeSideEffect FaultPoint = "before_side_effect" + FaultAfterSideEffect FaultPoint = "after_side_effect" + FaultAfterCommit FaultPoint = "after_commit" +) + +var ErrInjectedFailure = errors.New("injected worker failure") + +type Worker struct { + logger *slog.Logger + repository Repository + broker Broker + objects ObjectStore + processor ImageProcessor + metrics Metrics + config Config + memory *semaphore.Weighted + now func() time.Time +} + +func New( + logger *slog.Logger, + repository Repository, + broker Broker, + objects ObjectStore, + processor ImageProcessor, + metrics Metrics, + config Config, +) (*Worker, error) { + if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil { + return nil, errors.New("worker dependencies are required") + } + if err := validateConfig(config); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Worker{ + logger: logger, repository: repository, broker: broker, objects: objects, + processor: processor, metrics: metrics, config: config, + memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now, + }, nil +} + +func validateConfig(config Config) error { + if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration || + config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || + config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || + config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { + return errors.New("invalid worker configuration") + } + const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128 + if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 || + config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes { + return errors.New("worker memory must fit one maximum image") + } + return nil +} + +func (w *Worker) Run(ctx context.Context) error { + lastReclaim := time.Time{} + reclaimCursor := "0-0" + for ctx.Err() == nil { + deliveries, err := w.broker.Read(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.processBatch(ctx, deliveries) + if w.now().Sub(lastReclaim) < w.config.ReclaimInterval { + continue + } + recovery, err := w.repository.RecoverExpiredAttachmentJobs( + ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase), + ) + if err != nil { + return fmt.Errorf("recover expired attachment jobs: %w", err) + } + if recovery.Requeued > 0 || recovery.Failed > 0 { + w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed) + } + deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.metrics.ObserveReclaimed(ctx, len(deliveries)) + w.processBatch(ctx, deliveries) + reclaimCursor = next + if reclaimCursor == "" { + reclaimCursor = "0-0" + } + lastReclaim = w.now() + } + return nil +} + +func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) { + limit := make(chan struct{}, w.config.Concurrency) + var group sync.WaitGroup + for _, delivery := range deliveries { + if ctx.Err() != nil { + break + } + limit <- struct{}{} + group.Add(1) + go func(delivery redisstream.Delivery) { + defer group.Done() + defer func() { <-limit }() + _ = w.ProcessDelivery(ctx, delivery, FaultNone) + }(delivery) + } + group.Wait() +} + +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) { + started := w.now() + result := "transient_error" + if delivery.DecodeError != nil { + w.metrics.ObserveStarted(ctx) + defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }() + result = "invalid_job" + return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + } + envelope := delivery.Envelope + ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{ + Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate, + }) + ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes( + attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()), + attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt), + attribute.Int("pipeline.version", envelope.PipelineVersion), + )) + w.metrics.ObserveStarted(ctx) + defer func() { + w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) + span.SetAttributes(attribute.String("job.outcome", result)) + if resultErr != nil { + span.RecordError(resultErr) + span.SetStatus(codes.Error, result) + if ctx.Err() == nil { + w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr) + } + } else { + w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result) + } + span.End() + }() + + lease, err := w.repository.ClaimAttachmentJob( + ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, + w.config.WorkerID, w.config.LeaseDuration, + ) + if err != nil { + result, err = w.resolveClaim(ctx, delivery, envelope, err) + return err + } + if fault == FaultBeforeSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + + processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout) + heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx) + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }() + variants, err := w.process(processingCtx, lease) + cancelProcessing() + cancelHeartbeat() + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + return heartbeatErr + } + if err != nil { + result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) + return err + } + if fault == FaultAfterSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants) + if err != nil { + return fmt.Errorf("complete attachment job: %w", err) + } + if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { + if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { + w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + } + } + if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { + return fmt.Errorf("unexpected completion result %q", mutation) + } + if fault == FaultAfterCommit { + result = "injected_failure" + return ErrInjectedFailure + } + if err := delivery.Ack(ctx); err != nil { + return err + } + result = "ready" + w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt)) + return nil +} + +func (w *Worker) maintainLease( + ctx context.Context, + cancelProcessing context.CancelFunc, + lease model.AttachmentJobLease, +) error { + ticker := time.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration) + if err != nil { + if ctx.Err() != nil { + return nil + } + cancelProcessing() + return fmt.Errorf("renew attachment job lease: %w", err) + } + if !renewed { + cancelProcessing() + return model.ErrStaleLease + } + } + } +} + +func (w *Worker) resolveClaim( + ctx context.Context, + delivery redisstream.Delivery, + envelope redisstream.Envelope, + claimErr error, +) (string, error) { + switch { + case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput): + return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobBusy): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobNotDue): + return "not_due", nil + case errors.Is(claimErr, model.ErrConflict): + job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String()) + if err != nil { + return "transient_error", fmt.Errorf("inspect conflicting job: %w", err) + } + switch job.Status { + case model.AttachmentJobReady: + return "duplicate", delivery.Ack(ctx) + case model.AttachmentJobFailed: + code := "processing_failed" + if job.ErrorCode != nil { + code = *job.ErrorCode + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } +} + +func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) { + body, err := w.objects.Get(ctx, lease.Attachment.StagingKey) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > w.config.MaxBytes { + return nil, transform.ErrPayloadTooLarge + } + imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, transform.ErrInvalidImage + } + weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) + if err != nil { + return nil, err + } + if weight > w.config.MemoryBytes { + return nil, errWorkerCapacity + } + if err := w.memory.Acquire(ctx, weight); err != nil { + return nil, err + } + defer w.memory.Release(weight) + + processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType) + if err != nil { + return nil, err + } + variants := []struct { + name string + value transform.Variant + }{ + {name: "original", value: processed.Original}, + {name: "thumbnail", value: processed.Thumbnail}, + {name: "crop", value: processed.Crop}, + {name: "optimized", value: processed.Optimized}, + } + result := make([]model.Variant, 0, len(variants)) + for _, item := range variants { + objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value) + if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil { + return nil, err + } + result = append(result, model.Variant{ + Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)), + SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height, + ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion, + }) + } + return result, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.reader.Read(buffer) + } +} + +var errWorkerCapacity = errors.New("image exceeds worker memory budget") + +func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { + extension := ".bin" + switch value.ContentType { + case "image/png": + extension = ".png" + case "image/jpeg": + extension = ".jpg" + case "image/webp": + extension = ".webp" + } + return path.Join( + "attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion), + name, value.SHA256+extension, + ) +} + +func (w *Worker) resolveProcessingFailure( + ctx context.Context, + delivery redisstream.Delivery, + lease model.AttachmentJobLease, + processingErr error, +) (string, error) { + code, permanent := processingError(processingErr) + if permanent || lease.Job.Attempt >= w.config.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + mutation, err := w.repository.FailAttachmentJob(ctx, lease, code) + if err != nil { + return "transient_error", fmt.Errorf("fail attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected failure result %q", mutation) + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + } + nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) + traceContext := telemetry.ContextCarrier(ctx) + mutation, err := w.repository.RetryAttachmentJob( + ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate, + ) + if err != nil { + return "transient_error", fmt.Errorf("retry attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) + } + w.metrics.ObserveRetry(ctx) + return "retry", delivery.Ack(ctx) +} + +func processingError(err error) (string, bool) { + switch { + case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia): + return "invalid_image", true + case errors.Is(err, transform.ErrPayloadTooLarge): + return "image_too_large", true + case errors.Is(err, objectstore.ErrNotFound): + return "source_missing", true + case errors.Is(err, errWorkerCapacity): + return "worker_capacity", true + default: + return "processing_failed", false + } +} + +func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { + delay := w.config.RetryBase + for step := 1; step < attempt && delay < w.config.RetryMax; step++ { + if delay > w.config.RetryMax/2 { + delay = w.config.RetryMax + break + } + delay *= 2 + } + if delay > w.config.RetryMax { + delay = w.config.RetryMax + } + var hash uint64 + for index := range jobID { + hash = hash*131 + uint64(jobID[index]) + } + jitterRange := min(delay/4, w.config.RetryMax-delay) + if jitterRange <= 0 { + return delay + } + jitter := time.Duration(hash % uint64(jitterRange)) + if delay > time.Duration(math.MaxInt64)-jitter { + return delay + } + return delay + jitter +} + +type nopMetrics struct{} + +func (nopMetrics) ObserveStarted(context.Context) {} +func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} +func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {} +func (nopMetrics) ObserveRetry(context.Context) {} +func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go new file mode 100644 index 0000000..c2ca384 --- /dev/null +++ b/internal/messenger/attachment/worker/worker_test.go @@ -0,0 +1,109 @@ +package worker + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" +) + +func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { + config := validConfig() + require.NoError(t, validateConfig(config)) + + config.MemoryBytes = 384 * 1024 * 1024 + require.ErrorContains(t, validateConfig(config), "memory must fit") +} + +func TestRetryDelayIsBounded(t *testing.T) { + worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}} + for attempt := 1; attempt <= 10; attempt++ { + delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt) + require.GreaterOrEqual(t, delay, time.Second) + require.LessOrEqual(t, delay, 4*time.Second) + } +} + +func TestProcessingErrorClassification(t *testing.T) { + tests := []struct { + err error + code string + permanent bool + }{ + {err: transform.ErrInvalidImage, code: "invalid_image", permanent: true}, + {err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true}, + {err: objectstore.ErrNotFound, code: "source_missing", permanent: true}, + {err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false}, + } + for _, test := range tests { + code, permanent := processingError(test.err) + require.Equal(t, test.code, code) + require.Equal(t, test.permanent, permanent) + } +} + +func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) { + var renewals atomic.Int32 + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + renewals.Add(1) + return true, nil + }} + config := validConfig() + config.HeartbeatInterval = 5 * time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }() + require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) +} + +func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) { + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + return false, nil + }} + config := validConfig() + config.HeartbeatInterval = time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + err := worker.maintainLease(ctx, cancel, testLease()) + require.ErrorIs(t, err, model.ErrStaleLease) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +type repositoryStub struct { + Repository + renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) +} + +func (s *repositoryStub) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + return s.renew(ctx, lease, duration) +} + +func testLease() model.AttachmentJobLease { + return model.AttachmentJobLease{Job: model.AttachmentJob{ + ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1", + }} +} + +func validConfig() Config { + return Config{ + WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, + HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second, + ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, + } +} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go new file mode 100644 index 0000000..2e832d6 --- /dev/null +++ b/internal/messenger/httpapi/attachments.go @@ -0,0 +1,198 @@ +package httpapi + +import ( + "errors" + "io" + "mime" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { + started := time.Now() + result := "client_error" + defer func() { + attributes := metric.WithAttributes(attribute.String("result", result)) + s.metrics.attachmentUploads.Add(r.Context(), 1, attributes) + s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes) + }() + + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return invalidRequest() + } + filename, valid := attachmentFilename(query) + if !valid { + return invalidRequest() + } + uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ + ChatID: chatID, OwnerID: user.ID, Filename: filename, + ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + TraceContext: telemetry.ContextCarrier(r.Context()), + }) + if err != nil { + if errors.Is(err, model.ErrCapacity) { + result = "capacity_rejected" + w.Header().Set("Retry-After", "1") + } else if errors.Is(err, model.ErrUnavailable) { + result = "dependency_error" + } + return attachmentError(err) + } + result = "success" + s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes) + writeJSON(w, http.StatusAccepted, uploaded) + return nil +} + +func attachmentFilename(query url.Values) (string, bool) { + if len(query) != 1 || len(query["filename"]) != 1 { + return "", false + } + filename := query["filename"][0] + length := utf8.RuneCountInString(filename) + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { + return "", false + } + for _, char := range filename { + if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { + return "", false + } + } + return filename, true +} + +func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + writeJSON(w, http.StatusOK, value) + return nil +} + +func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + query, err := url.ParseQuery(r.URL.RawQuery) + variant, valid := attachmentVariant(query) + if err != nil || !valid { + return invalidRequest() + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant) + if err != nil { + return attachmentError(err) + } + defer func() { _ = content.Body.Close() }() + w.Header().Set("Content-Type", content.Variant.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, content.Body); err != nil { + s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err) + } + return nil +} + +func attachmentVariant(query url.Values) (string, bool) { + if len(query) == 0 { + return "original", true + } + if len(query) != 1 || len(query["variant"]) != 1 { + return "", false + } + switch query["variant"][0] { + case "original", "thumbnail", "crop", "optimized": + return query["variant"][0], true + default: + return "", false + } +} + +var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +func traceparent(value string) string { + if traceparentPattern.MatchString(value) { + return value + } + return "" +} + +func tracestate(value string) string { + if len(value) <= 512 { + return value + } + return "" +} + +func attachmentIDFromRequest(r *http.Request) (string, *apiError) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + attachmentID, valid := canonicalID(parts[3]) + if !valid { + return "", invalidRequest() + } + return attachmentID, nil +} + +func attachmentError(err error) *apiError { + switch { + case errors.Is(err, model.ErrNotFound): + return notFound() + case errors.Is(err, model.ErrInvalidInput): + return invalidRequest() + case errors.Is(err, model.ErrUnsupportedMedia): + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} + case errors.Is(err, model.ErrPayloadTooLarge): + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + case errors.Is(err, model.ErrCapacity): + return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + case errors.Is(err, model.ErrConflict): + return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"} + default: + return unavailable() + } +} diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go new file mode 100644 index 0000000..29419a1 --- /dev/null +++ b/internal/messenger/httpapi/auth.go @@ -0,0 +1,111 @@ +package httpapi + +import ( + "errors" + "io" + "net/http" + "strings" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1)) + if err != nil || len(body) != 0 { + return invalidRequest() + } + username, passwordValue, ok := singleBasicAuth(r) + if !ok { + return unauthorized() + } + credentials, err := s.userRepository.UserByUsername(r.Context(), username) + if errors.Is(err, model.ErrNotFound) { + if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { + return internalError() + } + return unauthorized() + } + if err != nil { + return unavailable() + } + verified, err := s.passwords.Verify(passwordValue, credentials.PasswordHash) + if err != nil { + return internalError() + } + if !verified { + return unauthorized() + } + token, expiresAt, err := s.sessions.Create(r.Context(), credentials.ID, s.sessionTTL) + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User User `json:"user"` + }{token, "Bearer", model.FormatTime(expiresAt), credentials.User}) + return nil +} + +func singleBasicAuth(r *http.Request) (string, string, bool) { + if len(r.Header.Values("Authorization")) != 1 { + return "", "", false + } + username, passwordValue, ok := r.BasicAuth() + return username, passwordValue, ok && username != "" +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodDelete { + return methodNotAllowed(w, http.MethodDelete) + } + _, token, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) { + return unauthorized() + } else if err != nil { + return unavailable() + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) authenticate(r *http.Request) (User, string, *apiError) { + values := r.Header.Values("Authorization") + if len(values) != 1 { + return User{}, "", unauthorized() + } + scheme, token, found := strings.Cut(values[0], " ") + if !found || !strings.EqualFold(scheme, "Bearer") || token == "" || strings.ContainsAny(token, " \t\r\n") { + return User{}, "", unauthorized() + } + userID, err := s.sessions.UserID(r.Context(), token) + if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) { + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + user, err := s.userRepository.UserByID(r.Context(), userID) + if errors.Is(err, model.ErrNotFound) { + _ = s.sessions.Delete(r.Context(), token) + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + return user, token, nil +} + +func authenticationChallenge(r *http.Request) string { + if r.URL.Path == "/api/v1/auth/sessions" { + return `Basic realm="messenger", charset="UTF-8"` + } + return `Bearer realm="messenger"` +} diff --git a/internal/messenger/httpapi/errors.go b/internal/messenger/httpapi/errors.go new file mode 100644 index 0000000..95ffd1c --- /dev/null +++ b/internal/messenger/httpapi/errors.go @@ -0,0 +1,61 @@ +package httpapi + +import ( + "encoding/json" + "net/http" +) + +const ( + errorCodeInvalidRequest = "invalid_request" + errorCodeUnauthorized = "unauthorized" + errorCodeNotFound = "not_found" + errorCodeInternal = "internal_error" + errorCodeUnsupportedMediaType = "unsupported_media_type" + errorCodePayloadTooLarge = "payload_too_large" + errorCodeUnavailable = "unavailable" + errorCodeConflict = "conflict" + errorCodeRateLimited = "rate_limited" +) + +type apiError struct { + status int + code string + message string +} + +func methodNotAllowed(w http.ResponseWriter, allowed string) *apiError { + w.Header().Set("Allow", allowed) + return &apiError{http.StatusMethodNotAllowed, errorCodeInvalidRequest, "method not allowed"} +} + +func invalidRequest() *apiError { + return &apiError{http.StatusBadRequest, errorCodeInvalidRequest, "invalid request"} +} + +func unauthorized() *apiError { + return &apiError{http.StatusUnauthorized, errorCodeUnauthorized, "authentication required"} +} + +func notFound() *apiError { + return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"} +} + +func internalError() *apiError { + return &apiError{http.StatusInternalServerError, errorCodeInternal, "internal server error"} +} + +func unavailable() *apiError { + return &apiError{http.StatusServiceUnavailable, errorCodeUnavailable, "service unavailable"} +} + +func writeError(w http.ResponseWriter, requestID string, err *apiError) { + writeJSON(w, err.status, map[string]any{"error": map[string]string{ + "code": err.code, "message": err.message, "request_id": requestID, + }}) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go new file mode 100644 index 0000000..9084d13 --- /dev/null +++ b/internal/messenger/httpapi/handlers.go @@ -0,0 +1,300 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "regexp" + "strings" + "time" + "unicode/utf8" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (s *Server) live(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + if !s.ready.Load() { + return unavailable() + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := s.database.Ping(ctx); err != nil { + return unavailable() + } + if err := s.sessions.Ping(ctx); err != nil { + return unavailable() + } + if err := s.objects.Ping(ctx); err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + var input struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !usernamePattern.MatchString(input.Username) || !validText(input.DisplayName, 100) || !validPassword(input.Password) { + return invalidRequest() + } + passwordHash, err := s.passwords.Hash(input.Password) + if err != nil { + return internalError() + } + user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) + if errors.Is(err, model.ErrConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"} + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, user) + return nil +} + +func validPassword(value string) bool { + length := utf8.RuneCountInString(value) + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && length >= 12 && length <= 128 +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, err := s.authenticate(r) + if err != nil { + return err + } + writeJSON(w, http.StatusOK, user) + return nil +} + +func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if r.Method == http.MethodGet { + return s.listChats(w, r, user) + } + var input struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !validText(input.Title, 100) || len(input.MemberIDs) < 1 || len(input.MemberIDs) > 100 { + return invalidRequest() + } + members := make(map[string]struct{}, len(input.MemberIDs)) + for i, memberID := range input.MemberIDs { + canonical, valid := canonicalID(memberID) + if !valid { + return invalidRequest() + } + if _, duplicate := members[canonical]; duplicate { + return invalidRequest() + } + members[canonical] = struct{}{} + input.MemberIDs[i] = canonical + } + if _, included := members[user.ID]; !included { + return invalidRequest() + } + chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID) + if errors.Is(err, model.ErrInvalidMembers) { + return invalidRequest() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, chat) + return nil +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *apiError { + limit, cursor, err := pageParams(r, "chat", user.ID) + if err != nil { + return err + } + items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Chat{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("chat", user.ID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, chat) + return nil +} + +func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + if r.Method == http.MethodGet { + return s.listMessages(w, r, user, chatID) + } + key, valid := idempotencyKey(r) + if !valid { + return invalidRequest() + } + var input createMessageRequest + if err := readJSON(w, r, &input); err != nil { + return err + } + text, attachmentIDs, valid := input.values() + if !valid || !validMessage(text, attachmentIDs) || !canonicalAttachmentIDs(attachmentIDs) { + return invalidRequest() + } + message, replay, err := s.messageRepository.CreateMessage( + r.Context(), chatID, user.ID, text, attachmentIDs, key, s.idempotencyTTL, + ) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if errors.Is(err, model.ErrIdempotencyConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"} + } + if err != nil { + return unavailable() + } + if replay { + s.metrics.replays.Add(r.Context(), 1) + } + writeJSON(w, http.StatusCreated, message) + return nil +} + +func validMessage(text string, attachmentIDs []string) bool { + if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 { + return false + } + return len(attachmentIDs) > 0 || strings.TrimSpace(text) != "" +} + +func canonicalAttachmentIDs(attachmentIDs []string) bool { + if len(attachmentIDs) > 10 { + return false + } + seen := make(map[string]struct{}, len(attachmentIDs)) + for index, attachmentID := range attachmentIDs { + canonical, valid := canonicalID(attachmentID) + if !valid { + return false + } + if _, duplicate := seen[canonical]; duplicate { + return false + } + seen[canonical] = struct{}{} + attachmentIDs[index] = canonical + } + return true +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError { + limit, cursor, err := pageParams(r, "message", chatID) + if err != nil { + return err + } + items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrNotFound) { + return notFound() + } + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Message{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("message", chatID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func cursorPosition(cursor *pageCursor) *model.Position { + if cursor == nil { + return nil + } + return &model.Position{CreatedAt: cursor.Time, ID: cursor.ID} +} diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go new file mode 100644 index 0000000..9844c25 --- /dev/null +++ b/internal/messenger/httpapi/messenger_interface.go @@ -0,0 +1,49 @@ +package httpapi + +import ( + "context" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type UserRepository interface { + CreateUser(context.Context, string, string, string) (model.User, error) + UserByID(context.Context, string) (model.User, error) + UserByUsername(context.Context, string) (model.UserCredentials, error) +} + +type ChatRepository interface { + CreateChat(context.Context, string, []string, string) (model.Chat, error) + ChatForUser(context.Context, string, string) (model.Chat, error) + ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) +} + +type MessageRepository interface { + CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) + ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) +} + +type AttachmentService interface { + Upload(context.Context, attachment.Upload) (model.Attachment, error) + Metadata(context.Context, string, string) (model.Attachment, error) + Content(context.Context, string, string, string) (attachment.Content, error) +} + +type HealthChecker interface { + Ping(context.Context) error +} + +type SessionStore interface { + HealthChecker + Create(context.Context, string, time.Duration) (string, time.Time, error) + UserID(context.Context, string) (string, error) + Delete(context.Context, string) error +} + +type PasswordService interface { + Hash(string) (string, error) + Verify(string, string) (bool, error) + DummyHash() string +} diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go new file mode 100644 index 0000000..5047ebd --- /dev/null +++ b/internal/messenger/httpapi/metrics.go @@ -0,0 +1,90 @@ +package httpapi + +import ( + "context" + "net/http" + "strconv" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + requestDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30} + uploadDurationBuckets = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} +) + +type metrics struct { + requests metric.Int64Counter + duration metric.Float64Histogram + inFlight metric.Int64UpDownCounter + replays metric.Int64Counter + attachmentUploads metric.Int64Counter + attachmentDuration metric.Float64Histogram + attachmentBytes metric.Int64Counter + handler http.Handler +} + +func newMetrics(meter metric.Meter, handler http.Handler) *metrics { + if meter == nil { + meter = otel.Meter("messenger-api") + } + if handler == nil { + handler = http.NotFoundHandler() + } + requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests.")) + duration, _ := meter.Float64Histogram("messenger_http_request_duration", + metric.WithDescription("HTTP request duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(requestDurationBuckets...), + ) + inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served.")) + replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record.")) + attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result.")) + attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration", + metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(uploadDurationBuckets...), + ) + attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By")) + return &metrics{ + requests: requests, duration: duration, inFlight: inFlight, replays: replays, + attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration, + attachmentBytes: attachmentBytes, handler: handler, + } +} + +func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) { + attributes := metric.WithAttributes( + attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)), + attribute.String("status_class", strconv.Itoa(status/100)+"xx"), + ) + m.requests.Add(ctx, 1, attributes) + m.duration.Record(ctx, time.Since(started).Seconds(), attributes) +} + +func routeTemplate(path string) string { + switch path { + case "/health/live", "/health/ready", "/metrics", "/api/v1/users", "/api/v1/users/me", + "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats": + return path + } + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + return "/api/v1/chats/{chat_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + return "/api/v1/chats/{chat_id}/messages" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + return "/api/v1/chats/{chat_id}/attachments" + } + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + return "/api/v1/attachments/{attachment_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + return "/api/v1/attachments/{attachment_id}/content" + } + return "unknown" +} diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go new file mode 100644 index 0000000..7f3f6dc --- /dev/null +++ b/internal/messenger/httpapi/model.go @@ -0,0 +1,30 @@ +package httpapi + +import ( + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "github.com/google/uuid" +) + +type User = model.User +type Chat = model.Chat +type Message = model.Message +type Attachment = model.Attachment + +func newID() (string, error) { + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + return id.String(), nil +} + +func canonicalID(value string) (string, bool) { + if len(value) != 36 { + return "", false + } + id, err := uuid.Parse(value) + if err != nil { + return "", false + } + return id.String(), true +} diff --git a/internal/messenger/httpapi/request.go b/internal/messenger/httpapi/request.go new file mode 100644 index 0000000..63cb694 --- /dev/null +++ b/internal/messenger/httpapi/request.go @@ -0,0 +1,147 @@ +package httpapi + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +const maxRequestBytes = 1 << 20 + +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type createMessageRequest struct { + Text json.RawMessage `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` +} + +func (r createMessageRequest) values() (string, []string, bool) { + var text *string + if len(r.Text) == 0 || json.Unmarshal(r.Text, &text) != nil || text == nil { + return "", nil, false + } + if len(r.AttachmentIDs) == 0 { + return *text, nil, true + } + var attachmentIDs *[]string + if json.Unmarshal(r.AttachmentIDs, &attachmentIDs) != nil || attachmentIDs == nil { + return "", nil, false + } + return *text, *attachmentIDs, true +} + +func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError { + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "expected application/json"} + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBytes) + data, err := io.ReadAll(r.Body) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "request body too large"} + } + return invalidRequest() + } + if !utf8.Valid(data) { + return invalidRequest() + } + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil || object == nil { + return invalidRequest() + } + normalized, err := json.Marshal(object) + if err != nil { + return invalidRequest() + } + decoder := json.NewDecoder(bytes.NewReader(normalized)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return invalidRequest() + } + return nil +} + +func validText(value string, maxRunes int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes +} + +func idempotencyKey(r *http.Request) (string, bool) { + values := r.Header.Values("Idempotency-Key") + if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) { + return "", false + } + return values[0], true +} + +type pageCursor struct { + Kind string `json:"kind"` + Resource string `json:"resource"` + Time time.Time `json:"time"` + ID string `json:"id"` +} + +func pageParams(r *http.Request, kind, resource string) (int, *pageCursor, *apiError) { + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return 0, nil, invalidRequest() + } + for key := range query { + if key != "limit" && key != "cursor" { + return 0, nil, invalidRequest() + } + if len(query[key]) != 1 { + return 0, nil, invalidRequest() + } + } + limit := 50 + if value, present := query["limit"]; present { + parsed, err := strconv.Atoi(value[0]) + if err != nil || parsed < 1 || parsed > 100 { + return 0, nil, invalidRequest() + } + limit = parsed + } + if value, present := query["cursor"]; present { + cursor, valid := decodeCursor(value[0], kind, resource) + if !valid { + return 0, nil, invalidRequest() + } + return limit, cursor, nil + } + return limit, nil, nil +} + +func encodeCursor(kind, resource string, createdAt time.Time, id string) string { + data, _ := json.Marshal(pageCursor{kind, resource, createdAt, id}) + return base64.RawURLEncoding.EncodeToString(data) +} + +func decodeCursor(value, kind, resource string) (*pageCursor, bool) { + data, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + return nil, false + } + var cursor pageCursor + if err := json.Unmarshal(data, &cursor); err != nil || cursor.Kind != kind || cursor.Resource != resource { + return nil, false + } + if cursor.Time.IsZero() { + return nil, false + } + if _, valid := canonicalID(cursor.ID); !valid { + return nil, false + } + return &cursor, true +} diff --git a/internal/messenger/httpapi/routes.go b/internal/messenger/httpapi/routes.go new file mode 100644 index 0000000..f272767 --- /dev/null +++ b/internal/messenger/httpapi/routes.go @@ -0,0 +1,47 @@ +package httpapi + +import ( + "net/http" + "strings" +) + +type requestHandler func(http.ResponseWriter, *http.Request) *apiError + +func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError { + var handler requestHandler + switch r.URL.Path { + case "/health/live": + handler = s.live + case "/health/ready": + handler = s.readiness + case "/metrics": + s.metrics.handler.ServeHTTP(w, r) + return nil + case "/api/v1/users": + handler = s.users + case "/api/v1/users/me": + handler = s.me + case "/api/v1/auth/sessions": + handler = s.createSession + case "/api/v1/auth/sessions/current": + handler = s.revokeSession + case "/api/v1/chats": + handler = s.chats + default: + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + handler = s.chat + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + handler = s.messages + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + handler = s.uploadAttachment + } else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + handler = s.attachmentMetadata + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + handler = s.attachmentContent + } else { + return notFound() + } + } + return handler(w, r) +} diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go new file mode 100644 index 0000000..fcc18bf --- /dev/null +++ b/internal/messenger/httpapi/server.go @@ -0,0 +1,158 @@ +package httpapi + +import ( + "context" + "log/slog" + "net/http" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Dependencies struct { + Users UserRepository + Chats ChatRepository + Messages MessageRepository + Attachments AttachmentService + Sessions SessionStore + Passwords PasswordService + Database HealthChecker + Objects HealthChecker + Meter metric.Meter + Metrics http.Handler +} + +type Config struct { + SessionTTL time.Duration + IdempotencyTTL time.Duration + RequestTTL time.Duration + InstanceID string + SwaggerPort string +} + +type Server struct { + userRepository UserRepository + chatRepository ChatRepository + messageRepository MessageRepository + attachments AttachmentService + sessions SessionStore + passwords PasswordService + database HealthChecker + objects HealthChecker + sessionTTL time.Duration + idempotencyTTL time.Duration + requestTTL time.Duration + instanceID string + logger *slog.Logger + swaggerPort string + metrics *metrics + ready atomic.Bool +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func NewServer( + logger *slog.Logger, + dependencies Dependencies, + config Config, +) *Server { + s := &Server{ + userRepository: dependencies.Users, chatRepository: dependencies.Chats, + messageRepository: dependencies.Messages, + attachments: dependencies.Attachments, sessions: dependencies.Sessions, + passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, + sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, + metrics: newMetrics(dependencies.Meter, dependencies.Metrics), + } + s.ready.Store(true) + return s +} + +func (s *Server) SetReady(ready bool) { + s.ready.Store(ready) +} + +func (s *Server) Handler() http.Handler { + return http.HandlerFunc(s.serve) +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + tracer := otel.Tracer("messenger-api/http") + route := routeTemplate(r.URL.Path) + requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route))) + defer span.End() + r = r.WithContext(requestContext) + w.Header().Set("X-Instance-Id", s.instanceID) + requestID, err := newID() + if err != nil { + http.Error(w, "server unavailable", http.StatusServiceUnavailable) + return + } + span.SetAttributes(attribute.String("request.id", requestID)) + w.Header().Set("X-Request-Id", requestID) + w.Header().Set("Cache-Control", "no-store") + wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK} + started := time.Now() + s.metrics.inFlight.Add(r.Context(), 1) + defer func() { + s.metrics.inFlight.Add(r.Context(), -1) + if recovered := recover(); recovered != nil { + s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route) + if wrapped.status == http.StatusOK { + writeError(wrapped, requestID, internalError()) + } + } + s.metrics.observe(r.Context(), r, wrapped.status, started) + span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status)) + if wrapped.status >= http.StatusInternalServerError { + span.SetStatus(codes.Error, http.StatusText(wrapped.status)) + } + s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) + }() + if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { + wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key") + wrapped.WriteHeader(http.StatusNoContent) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.requestTTL) + defer cancel() + r = r.WithContext(ctx) + if apiErr := s.route(wrapped, r); apiErr != nil { + if apiErr.status == http.StatusUnauthorized { + wrapped.Header().Set("WWW-Authenticate", authenticationChallenge(r)) + } + writeError(wrapped, requestID, apiErr) + } +} + +func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin != "http://localhost:"+s.swaggerPort && origin != "http://127.0.0.1:"+s.swaggerPort { + return false + } + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id") + w.Header().Add("Vary", "Origin") + return true +} diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go new file mode 100644 index 0000000..c1bf91d --- /dev/null +++ b/internal/messenger/httpapi/server_test.go @@ -0,0 +1,378 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var errDependency = errors.New("dependency unavailable") + +type MessengerSuite struct { + suite.Suite + logger *slog.Logger +} + +func TestMessengerSuite(t *testing.T) { + suite.Run(t, new(MessengerSuite)) +} + +func (s *MessengerSuite) SetupTest() { + s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +type messengerStoreStub struct { + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error +} + +func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } +func (messengerStoreStub) CreateUser(context.Context, string, string, string) (model.User, error) { + return model.User{}, nil +} +func (s messengerStoreStub) UserByID(context.Context, string) (model.User, error) { + return s.userByID, s.userByIDErr +} +func (s messengerStoreStub) UserByUsername(context.Context, string) (model.UserCredentials, error) { + return s.credentials, s.credentialsErr +} +func (messengerStoreStub) CreateChat(context.Context, string, []string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) { + return nil, false, nil +} +func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) { + return s.message, s.messageReplay, s.messageErr +} +func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { + return nil, false, nil +} + +type objectStoreStub struct { + pingErr error +} + +func (s objectStoreStub) Ping(context.Context) error { return s.pingErr } + +type attachmentServiceStub struct { + uploaded model.Attachment + err error +} + +func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) { + return attachment.Content{ + Attachment: s.uploaded, + Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, + Body: io.NopCloser(strings.NewReader("content")), + }, s.err +} + +type sessionStub struct { + pingErr error + userID string + userIDErr error +} + +func (s sessionStub) Ping(context.Context) error { return s.pingErr } +func (sessionStub) Create(context.Context, string, time.Duration) (string, time.Time, error) { + return "token", time.Now().Add(time.Hour), nil +} +func (s sessionStub) UserID(context.Context, string) (string, error) { return s.userID, s.userIDErr } +func (sessionStub) Delete(context.Context, string) error { return nil } + +type passwordsStub struct { + verified bool + err error +} + +func (passwordsStub) Hash(string) (string, error) { return "hash", nil } +func (passwordsStub) DummyHash() string { return "hash" } +func (s passwordsStub) Verify(string, string) (bool, error) { + return s.verified, s.err +} + +func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + objects objectStoreStub + }{ + {name: "postgres", store: messengerStoreStub{pingErr: errDependency}}, + {name: "valkey", sessions: sessionStub{pingErr: errDependency}}, + {name: "s3", objects: objectStoreStub{pingErr: errDependency}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{}) + response := serveRequest(server, http.MethodGet, "/health/ready", "") + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + }{ + {name: "valkey", sessions: sessionStub{userIDErr: errDependency}}, + {name: "postgres", store: messengerStoreStub{userByIDErr: errDependency}, sessions: sessionStub{userID: "user-id"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", "Bearer valid-token") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestAuthenticationChallenges() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "") + if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` { + t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate")) + } + + store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}} + server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("user", "wrong-password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized || response.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatalf("session challenge = %d %q", response.Code, response.Header().Get("WWW-Authenticate")) + } +} + +func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest( + server, + http.MethodGet, + "/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid", + "", + ) + s.Equal(http.StatusUnauthorized, response.Code) + s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate")) +} + +func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + request.ContentLength = -1 + request.SetBasicAuth("user", "password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + for _, response := range []*httptest.ResponseRecorder{ + serveRequest(server, http.MethodGet, "/health/live", ""), + serveRequest(server, http.MethodGet, "/api/v1/users/me", ""), + } { + if value := response.Header().Get("X-Instance-Id"); value != "test-instance" { + t.Fatalf("X-Instance-Id = %q, want test-instance", value) + } + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() { + t := s.T() + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + tests := []struct { + name string + key string + }{ + {name: "missing"}, + {name: "invalid", key: "invalid key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + if tt.key != "" { + request.Header.Set("Idempotency-Key", tt.key) + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } + }) + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesFieldPresenceAndTypes() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + attachmentID := "00000000-0000-0000-0000-000000000002" + invalidBodies := []string{ + `{"attachment_ids":["` + attachmentID + `"]}`, + `{"text":null,"attachment_ids":["` + attachmentID + `"]}`, + `{"text":"x","attachment_ids":null}`, + `{"text":1}`, + `{"text":"x","attachment_ids":{}}`, + } + for _, body := range invalidBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusBadRequest, response.Code, body) + } + + validBodies := []string{ + `{"text":"message"}`, + `{"text":"message","attachment_ids":[]}`, + `{"text":"","attachment_ids":["` + attachmentID + `"]}`, + } + for _, body := range validBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusCreated, response.Code, body) + } +} + +func createMessageRequestResponse(server *Server, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", + strings.NewReader(body), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "message-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} + +func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() { + t := s.T() + store := messengerStoreStub{ + userByID: model.User{ID: "user-id"}, + messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict), + } + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "same-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func (s *MessengerSuite) TestAttachmentFilenameValidation() { + for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} { + _, valid := attachmentFilename(map[string][]string{"filename": {filename}}) + s.False(valid, filename) + } + filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}}) + s.True(valid) + s.Equal("фото.png", filename) +} + +func (s *MessengerSuite) TestUploadRejectsMalformedQuery() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG", + strings.NewReader("image"), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + s.Equal(http.StatusBadRequest, response.Code) +} + +func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server { + return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{}) +} + +func (s *MessengerSuite) server( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + objects objectStoreStub, + attachments AttachmentService, +) *Server { + return NewServer(s.logger, Dependencies{ + Users: store, Chats: store, Messages: store, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: store, Objects: objects, + }, Config{ + SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second, + InstanceID: "test-instance", SwaggerPort: "8081", + }) +} + +func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, strings.NewReader(body)) + if body != "" { + request.Header.Set("Content-Type", "application/json") + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} diff --git a/internal/model/errors.go b/internal/model/errors.go new file mode 100644 index 0000000..83105b2 --- /dev/null +++ b/internal/model/errors.go @@ -0,0 +1,24 @@ +package model + +import "errors" + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrInvalidMembers = errors.New("invalid members") + ErrInvalidCursor = errors.New("invalid cursor") + ErrIdempotencyConflict = errors.New("idempotency conflict") + ErrInvalidToken = errors.New("invalid token") + ErrSessionNotFound = errors.New("session not found") + ErrInvalidInput = errors.New("invalid input") + ErrUnsupportedMedia = errors.New("unsupported media") + ErrPayloadTooLarge = errors.New("payload too large") + ErrCapacity = errors.New("capacity exhausted") + ErrUnavailable = errors.New("dependency unavailable") + ErrStaleLease = errors.New("stale lease") + ErrStaleAttempt = errors.New("stale attempt") + ErrStalePipeline = errors.New("stale pipeline") + ErrJobBusy = errors.New("job is already leased") + ErrJobComplete = errors.New("job is already complete") + ErrJobNotDue = errors.New("job is not due") +) diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..83f7995 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,223 @@ +package model + +import ( + "encoding/json" + "time" +) + +const ( + AttachmentPending = "pending" + AttachmentQueued = "queued" + AttachmentProcessing = "processing" + AttachmentReady = "ready" + AttachmentFailed = "failed" + AttachmentDeleting = "deleting" + + AttachmentJobQueued = "queued" + AttachmentJobProcessing = "processing" + AttachmentJobReady = "ready" + AttachmentJobFailed = "failed" + + JobMutationApplied JobMutationResult = "applied" + JobMutationAlreadyComplete JobMutationResult = "already_complete" + JobMutationStale JobMutationResult = "stale" +) + +const timestampLayout = "2006-01-02T15:04:05.000000Z" + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt time.Time `json:"created_at"` +} + +type UserCredentials struct { + User + PasswordHash string `json:"-"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + StagingKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` + PipelineVersion int `json:"-"` +} + +type StoredObject struct { + Key string + Size int64 + LastModified time.Time +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` +} + +type AttachmentJob struct { + ID string + AttachmentID string + PipelineVersion int + Attempt int + Status string + AvailableAt time.Time + LeaseToken string + LeaseOwner string + LeaseUntil *time.Time + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentJobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type AttachmentJobLease struct { + Job AttachmentJob + Attachment Attachment +} + +type OutboxEvent struct { + ID string + Topic string + PartitionKey string + Payload json.RawMessage + JobID string + Attempt int + AvailableAt time.Time + PublishTries int + LeaseToken string + LeaseUntil *time.Time + LastErrorCode *string + CreatedAt time.Time +} + +type JobMutationResult string + +type RecoveryResult struct { + Requeued int + Failed int +} + +type Position struct { + CreatedAt time.Time + ID string +} + +type userJSON struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type chatJSON struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type messageJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type attachmentJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (u User) MarshalJSON() ([]byte, error) { + return json.Marshal(userJSON{u.ID, u.Username, u.DisplayName, formatTime(u.CreatedAt)}) +} + +func (c Chat) MarshalJSON() ([]byte, error) { + return json.Marshal(chatJSON{c.ID, c.Title, c.MemberIDs, formatTime(c.CreatedAt)}) +} + +func (m Message) MarshalJSON() ([]byte, error) { + attachments := m.AttachmentIDs + if attachments == nil { + attachments = []string{} + } + return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments}) +} + +func (a Attachment) MarshalJSON() ([]byte, error) { + variants := a.Variants + if variants == nil { + variants = []Variant{} + } + return json.Marshal(attachmentJSON{ + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt), + }) +} + +func FormatTime(value time.Time) string { + return formatTime(value) +} + +func formatTime(value time.Time) string { + return value.UTC().Format(timestampLayout) +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..d3a54d9 --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,24 @@ +package model + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type ModelSuite struct { + suite.Suite +} + +func TestModelSuite(t *testing.T) { + suite.Run(t, new(ModelSuite)) +} + +func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() { + createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC) + data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt}) + s.Require().NoError(err) + s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data)) +} diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go new file mode 100644 index 0000000..e2d6ded --- /dev/null +++ b/internal/objectstore/store.go @@ -0,0 +1,225 @@ +package objectstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var ErrNotFound = errors.New("object not found") + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseTLS bool + Timeout time.Duration +} + +type Store struct { + client *minio.Client + bucket string + timeout time.Duration +} + +func New(config Config) (*Store, error) { + endpoint, err := endpointHost(config.Endpoint, config.UseTLS) + if err != nil { + return nil, err + } + if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 { + return nil, errors.New("invalid object store configuration") + } + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), + Secure: config.UseTLS, + Region: config.Region, + }) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil +} + +func endpointHost(value string, useTLS bool) (string, error) { + value = strings.TrimSpace(value) + if !strings.Contains(value, "://") { + if value == "" { + return "", errors.New("S3 endpoint is required") + } + return value, nil + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" { + return "", errors.New("invalid S3 endpoint") + } + if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS") + } + return parsed.Host, nil +} + +func (s *Store) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return fmt.Errorf("check S3 bucket: %w", err) + } + if !exists { + return errors.New("S3 bucket does not exist") + } + return nil +} + +func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, span := s.startSpan(ctx, "PUT") + defer span.End() + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.client.PutObject( + ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, + ) + if err != nil { + recordSpanError(span, err) + return fmt.Errorf("put S3 object: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, span := s.startSpan(ctx, "GET") + ctx, cancel := context.WithTimeout(ctx, s.timeout) + object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + cancel() + recordSpanError(span, err) + span.End() + return nil, objectError("get S3 object", err) + } + if _, err := object.Stat(); err != nil { + cancel() + _ = object.Close() + recordSpanError(span, err) + span.End() + return nil, objectError("stat S3 object", err) + } + return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return model.StoredObject{}, objectError("stat S3 object", err) + } + return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Delete(ctx context.Context, key string) error { + ctx, span := s.startSpan(ctx, "DELETE") + defer span.End() + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + recordSpanError(span, err) + return objectError("delete S3 object", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", errors.New("object list limit must be between 1 and 1000") + } + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + objects := make([]model.StoredObject, 0, limit+1) + options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1} + for info := range s.client.ListObjects(ctx, s.bucket, options) { + if info.Err != nil { + return nil, "", fmt.Errorf("list S3 objects: %w", info.Err) + } + objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) + if len(objects) == limit+1 { + cancel() + break + } + } + var next string + if len(objects) > limit { + objects = objects[:limit] + next = objects[len(objects)-1].Key + } + return objects, next, nil +} + +type timeoutReader struct { + io.ReadCloser + cancel context.CancelFunc +} + +type tracedReader struct { + io.ReadCloser + span trace.Span +} + +func (r *tracedReader) Read(buffer []byte) (int, error) { + read, err := r.ReadCloser.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + recordSpanError(r.span, err) + } + return read, err +} + +func (r *tracedReader) Close() error { + err := r.ReadCloser.Close() + if err != nil { + recordSpanError(r.span, err) + } + r.span.End() + return err +} + +func (r *timeoutReader) Close() error { + err := r.ReadCloser.Close() + r.cancel() + return err +} + +func objectError(action string, err error) error { + response := minio.ToErrorResponse(err) + if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" { + return ErrNotFound + } + return fmt.Errorf("%s: %w", action, err) +} + +func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) { + return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation), + attribute.String("s3.bucket.name", s.bucket))) +} + +func recordSpanError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, "S3 operation failed") +} diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go new file mode 100644 index 0000000..ea74f61 --- /dev/null +++ b/internal/outbox/dispatcher.go @@ -0,0 +1,233 @@ +package outbox + +import ( + "context" + "errors" + "fmt" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +var errLeaseLost = errors.New("outbox lease lost") + +type Store interface { + ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) + MarkOutboxPublished(context.Context, string, string) (bool, error) + ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) + RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) + DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error) +} + +type Publisher interface { + PublishConfirmed(context.Context, []byte) (string, error) +} + +type Metrics interface { + ObservePublished(context.Context) + ObservePublishFailure(context.Context) + ObserveRecovered(context.Context, int) +} + +type Config struct { + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int +} + +type Dispatcher struct { + store Store + publisher Publisher + metrics Metrics + config Config + now func() time.Time +} + +func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) { + if store == nil { + return nil, errors.New("outbox store is required") + } + if publisher == nil { + return nil, errors.New("outbox publisher is required") + } + if err := config.validate(); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil +} + +func (c Config) validate() error { + switch { + case c.BatchSize <= 0: + return errors.New("outbox batch size must be positive") + case c.LeaseDuration <= 0: + return errors.New("outbox lease duration must be positive") + case c.PollInterval <= 0: + return errors.New("outbox poll interval must be positive") + case c.RecoveryInterval <= 0: + return errors.New("outbox recovery interval must be positive") + case c.RetryDelay <= 0: + return errors.New("outbox retry delay must be positive") + case c.Retention <= 0: + return errors.New("outbox retention must be positive") + case c.CleanupInterval <= 0: + return errors.New("outbox cleanup interval must be positive") + case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000: + return errors.New("outbox cleanup batch size must be between 1 and 1000") + default: + return nil + } +} + +func (d *Dispatcher) Run(ctx context.Context) error { + if ctx.Err() != nil { + return nil + } + + pollTicker := time.NewTicker(d.config.PollInterval) + recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + cleanupTicker := time.NewTicker(d.config.CleanupInterval) + defer pollTicker.Stop() + defer recoveryTicker.Stop() + defer cleanupTicker.Stop() + + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + for { + if _, err := d.DispatchOnce(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + + select { + case <-ctx.Done(): + return nil + case <-pollTicker.C: + case <-recoveryTicker.C: + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + case <-cleanupTicker.C: + if _, err := d.CleanupPublished(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + } + } +} + +func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) { + deleted, err := d.store.DeletePublishedOutboxEvents( + ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize, + ) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return deleted, nil +} + +func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { + events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) + if err != nil { + return 0, fmt.Errorf("claim due outbox events: %w", err) + } + + published := 0 + for _, event := range events { + if err := ctx.Err(); err != nil { + return published, nil + } + publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload) + publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish", + trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt), + )) + payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload) + if injectErr != nil { + span.RecordError(injectErr) + span.SetStatus(codes.Error, "inject trace context") + span.End() + return published, fmt.Errorf("inject outbox trace context: %w", injectErr) + } + _, err := d.publisher.PublishConfirmed(publishContext, payload) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "publish failed") + span.End() + if ctx.Err() != nil { + return published, nil + } + d.metrics.ObservePublishFailure(publishContext) + nextAttemptAt := d.now().Add(d.config.RetryDelay) + released, releaseErr := d.store.ReleaseOutboxEvent( + publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + ) + if releaseErr != nil { + return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) + } + if !released { + return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost) + } + continue + } + marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "mark published") + span.End() + return published, fmt.Errorf("mark outbox event published: %w", err) + } + if !marked { + span.SetStatus(codes.Error, "outbox lease lost") + span.End() + return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) + } + span.SetAttributes(attribute.String("messaging.operation.type", "publish")) + span.End() + d.metrics.ObservePublished(publishContext) + published++ + } + return published, nil +} + +func (d *Dispatcher) RecoverExpired(ctx context.Context) error { + recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now()) + if err != nil { + return fmt.Errorf("recover expired outbox leases: %w", err) + } + d.metrics.ObserveRecovered(ctx, recovered) + return nil +} + +type nopMetrics struct{} + +func (nopMetrics) ObservePublished(context.Context) {} +func (nopMetrics) ObservePublishFailure(context.Context) {} +func (nopMetrics) ObserveRecovered(context.Context, int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go new file mode 100644 index 0000000..b9d792c --- /dev/null +++ b/internal/outbox/dispatcher_test.go @@ -0,0 +1,207 @@ +package outbox + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "github.com/stretchr/testify/require" +) + +func TestDispatcherPublishesAndMarksEvent(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, []string{"event-1:lease-1"}, store.marked) +} + +func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) { + event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()} + store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + _, err := dispatcher.DispatchOnce(context.Background()) + require.Error(t, err) + require.Equal(t, 1, publisher.calls) + + store.markErr = nil + store.events[0].LeaseToken = "lease-2" + publisher.messageID = "2-0" + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, 2, publisher.calls) + require.JSONEq(t, string(event.Payload), string(publisher.payloads[0])) + require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1])) + require.Equal(t, []string{"event-1:lease-2"}, store.marked) +} + +func TestDispatcherReleasesPublishFailure(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{err: errors.New("redis unavailable")} + dispatcher := newTestDispatcher(t, store, publisher) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Zero(t, published) + require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released) + require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt) +} + +func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { + store := &storeStub{ + events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}}, + markLost: true, + } + dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"}) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.ErrorIs(t, err, errLeaseLost) + require.Zero(t, published) +} + +func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) { + store := &storeStub{deleted: 2} + dispatcher := newTestDispatcher(t, store, &publisherStub{}) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + deleted, err := dispatcher.CleanupPublished(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, deleted) + require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore) + require.Equal(t, 100, store.cleanupLimit) +} + +func TestDispatcherStopsAfterCancellation(t *testing.T) { + store := &storeStub{} + publisher := &publisherStub{} + dispatcher := newTestDispatcher(t, store, publisher) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { done <- dispatcher.Run(ctx) }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("dispatcher did not stop after cancellation") + } +} + +func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher { + t.Helper() + dispatcher, err := New(store, publisher, nil, Config{ + BatchSize: 10, + LeaseDuration: time.Minute, + PollInterval: time.Millisecond, + RecoveryInterval: time.Second, + RetryDelay: time.Second, + Retention: 24 * time.Hour, + CleanupInterval: time.Hour, + CleanupBatchSize: 100, + }) + require.NoError(t, err) + return dispatcher +} + +func testPayload() json.RawMessage { + return json.RawMessage(`{ + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-10-22T12:00:00Z" + }`) +} + +type storeStub struct { + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int + deleted int + publishedBefore time.Time + cleanupLimit int +} + +func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]model.OutboxEvent(nil), s.events...), nil +} + +func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.markErr != nil { + return false, s.markErr + } + if s.markLost { + return false, nil + } + s.marked = append(s.marked, id+":"+leaseToken) + return true, nil +} + +func (s *storeStub) ReleaseOutboxEvent( + _ context.Context, + id string, + leaseToken string, + code string, + nextAttemptAt time.Time, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.released = id + ":" + leaseToken + ":" + code + s.nextAttemptAt = nextAttemptAt + return true, nil +} + +func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recoveredCalls++ + return 0, nil +} + +func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishedBefore = before + s.cleanupLimit = limit + return s.deleted, nil +} + +type publisherStub struct { + mu sync.Mutex + messageID string + err error + calls int + payloads [][]byte +} + +func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.calls++ + p.payloads = append(p.payloads, append([]byte(nil), payload...)) + return p.messageID, p.err +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 0000000..5dd8b1b --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,133 @@ +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +var ( + ErrInvalidConfig = errors.New("invalid password configuration") + ErrInvalidHash = errors.New("invalid password hash") +) + +type Config struct { + MemoryKiB uint32 + Iterations uint32 + Parallelism uint8 + SaltLength uint32 + KeyLength uint32 +} + +type Hasher struct { + config Config + dummyHash string +} + +func New(config Config) (*Hasher, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + salt := make([]byte, config.SaltLength) + digest := argon2.IDKey([]byte("invalid-user-password"), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return &Hasher{config: config, dummyHash: encode(config, salt, digest)}, nil +} + +func validateConfig(config Config) error { + if config.MemoryKiB < 8*1024 || config.MemoryKiB > 1024*1024 || + config.Iterations < 1 || config.Iterations > 20 || + config.Parallelism < 1 || + config.SaltLength < 8 || config.SaltLength > 1024 || + config.KeyLength < 16 || config.KeyLength > 1024 { + return ErrInvalidConfig + } + return nil +} + +func (h *Hasher) Hash(value string) (string, error) { + salt := make([]byte, h.config.SaltLength) + if _, err := rand.Read(salt); err != nil { + return "", fmt.Errorf("generate password salt: %w", err) + } + digest := argon2.IDKey([]byte(value), salt, h.config.Iterations, h.config.MemoryKiB, h.config.Parallelism, h.config.KeyLength) + return encode(h.config, salt, digest), nil +} + +func (h *Hasher) Verify(value, encodedHash string) (bool, error) { + config, salt, digest, err := decode(encodedHash) + if err != nil { + return false, err + } + actual := argon2.IDKey([]byte(value), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return subtle.ConstantTimeCompare(actual, digest) == 1, nil +} + +func (h *Hasher) DummyHash() string { + return h.dummyHash +} + +func encode(config Config, salt, digest []byte) string { + return fmt.Sprintf( + "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + config.MemoryKiB, + config.Iterations, + config.Parallelism, + base64.RawStdEncoding.EncodeToString(salt), + base64.RawStdEncoding.EncodeToString(digest), + ) +} + +func decode(encodedHash string) (Config, []byte, []byte, error) { + parts := strings.Split(encodedHash, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" || parts[2] != fmt.Sprintf("v=%d", argon2.Version) { + return Config{}, nil, nil, ErrInvalidHash + } + + parameters := strings.Split(parts[3], ",") + if len(parameters) != 3 { + return Config{}, nil, nil, ErrInvalidHash + } + memory, err := parameter(parameters[0], "m=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + iterations, err := parameter(parameters[1], "t=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + parallelism, err := parameter(parameters[2], "p=", 8) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + + salt, err := base64.RawStdEncoding.Strict().DecodeString(parts[4]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + digest, err := base64.RawStdEncoding.Strict().DecodeString(parts[5]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + config := Config{ + MemoryKiB: uint32(memory), Iterations: uint32(iterations), Parallelism: uint8(parallelism), + SaltLength: uint32(len(salt)), KeyLength: uint32(len(digest)), + } + if err := validateConfig(config); err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + return config, salt, digest, nil +} + +func parameter(value, prefix string, bitSize int) (uint64, error) { + if !strings.HasPrefix(value, prefix) { + return 0, ErrInvalidHash + } + return strconv.ParseUint(strings.TrimPrefix(value, prefix), 10, bitSize) +} diff --git a/internal/password/password_test.go b/internal/password/password_test.go new file mode 100644 index 0000000..8e88875 --- /dev/null +++ b/internal/password/password_test.go @@ -0,0 +1,44 @@ +package password + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type PasswordSuite struct { + suite.Suite + hasher *Hasher +} + +func TestPasswordSuite(t *testing.T) { + suite.Run(t, new(PasswordSuite)) +} + +func (s *PasswordSuite) SetupTest() { + hasher, err := New(testConfig()) + s.Require().NoError(err) + s.hasher = hasher +} + +func (s *PasswordSuite) TestHasher() { + hash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + + verified, err := s.hasher.Verify("T3st-password-123", hash) + s.Require().NoError(err) + s.True(verified) + + verified, err = s.hasher.Verify("wrong-password", hash) + s.Require().NoError(err) + s.False(verified) +} + +func (s *PasswordSuite) TestVerifyRejectsMalformedHash() { + _, err := s.hasher.Verify("password", "invalid") + s.Error(err) +} + +func testConfig() Config { + return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32} +} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go new file mode 100644 index 0000000..6c959de --- /dev/null +++ b/internal/postgres/attachment_jobs.go @@ -0,0 +1,563 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const attachmentJobTopic = "attachment.jobs" + +const attachmentJobColumns = ` + id, attachment_id, pipeline_version, attempt, status, available_at, + lease_token, lease_owner, lease_until, error_code, created_at, updated_at` + +func (s *Store) QueueAttachmentWithJobAndOutbox( + ctx context.Context, + attachmentID string, + stagingKey string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 1 { + return model.Attachment{}, model.ErrInvalidInput + } + if job.Attempt == 0 { + job.Attempt = 1 + } + if job.Attempt != 1 { + return model.Attachment{}, model.ErrInvalidInput + } + now := time.Now().UTC() + if job.CreatedAt.IsZero() { + job.CreatedAt = now + } + if job.AvailableAt.IsZero() { + job.AvailableAt = now + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + query := ` + UPDATE attachments + SET status = 'queued', pipeline_version = $3, staging_key = $2, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + var attachment model.Attachment + err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job) + } + if err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment: %w", err) + } + + job.AttachmentID = attachmentID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = job.CreatedAt + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) QueueAttachmentPipelineVersion( + ctx context.Context, + attachmentID string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 2 { + return model.Attachment{}, model.ErrInvalidInput + } + job.Attempt = 1 + now := time.Now().UTC() + job.CreatedAt = now + job.AvailableAt = now + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err) + } + if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 { + return model.Attachment{}, ErrConflict + } + job.AttachmentID = attachment.ID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = now + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', pipeline_version = $2, staging_key = object_key, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 + RETURNING ` + attachmentColumns + if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) + } + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) alreadyQueuedAttachment( + ctx context.Context, + tx pgx.Tx, + attachmentID string, + stagingKey string, + job model.AttachmentJob, +) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready') + AND j.id = $3 AND j.pipeline_version = $4` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrConflict + } + return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error { + const query = ` + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)` + if _, err := tx.Exec( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt, + ); err != nil { + return fmt.Errorf("insert attachment job: %w", err) + } + return nil +} + +func insertAttachmentJobOutbox( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + traceparent string, + tracestate string, +) error { + payload, err := json.Marshal(model.AttachmentJobMessage{ + SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1", + AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, CreatedAt: job.CreatedAt, + Traceparent: traceparent, Tracestate: tracestate, + }) + if err != nil { + return fmt.Errorf("marshal attachment job event: %w", err) + } + const query = ` + INSERT INTO outbox_events ( + id, topic, partition_key, payload, job_id, attempt, available_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload, + job.ID, job.Attempt, job.AvailableAt, + ); err != nil { + return fmt.Errorf("insert attachment job outbox event: %w", err) + } + return nil +} + +func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) { + const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1` + var job model.AttachmentJob + if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrNotFound + } + return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err) + } + return job, nil +} + +func (s *Store) ClaimAttachmentJob( + ctx context.Context, + jobID string, + attachmentID string, + pipelineVersion int, + attempt int, + workerID string, + leaseDuration time.Duration, +) (model.AttachmentJobLease, error) { + if leaseDuration <= 0 || workerID == "" { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJobLease{}, err + } + if attachment.ID != attachmentID || job.AttachmentID != attachmentID { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady { + return model.AttachmentJobLease{}, model.ErrJobComplete + } + if job.Attempt != attempt { + return model.AttachmentJobLease{}, model.ErrStaleAttempt + } + if job.Status == model.AttachmentJobProcessing { + return model.AttachmentJobLease{}, model.ErrJobBusy + } + if job.Status != model.AttachmentJobQueued { + return model.AttachmentJobLease{}, ErrConflict + } + if job.AvailableAt.After(time.Now()) { + return model.AttachmentJobLease{}, model.ErrJobNotDue + } + + job.LeaseToken = uuid.NewString() + job.LeaseOwner = workerID + leaseUntil := time.Now().UTC().Add(leaseDuration) + job.LeaseUntil = &leaseUntil + job.Status = model.AttachmentJobProcessing + const updateJob = ` + UPDATE attachment_jobs + SET status = 'processing', lease_token = $2, lease_owner = $3, + lease_until = $4, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err) + } + const updateAttachment = ` + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'` + result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err) + } + if result.RowsAffected() != 1 { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err) + } + attachment.Status = model.AttachmentProcessing + attachment.ErrorCode = nil + return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil +} + +func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) { + query := ` + SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + ` + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = $1 + FOR UPDATE OF j, a` + var job model.AttachmentJob + var attachment model.Attachment + var leaseToken, leaseOwner *string + var stagingKey *string + err := tx.QueryRow(ctx, query, jobID).Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, model.Attachment{}, ErrNotFound + } + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err) + } + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return job, attachment, nil +} + +func prefixedAttachmentJobColumns(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+ + "%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+ + "%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at", + alias, + ) +} + +func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error { + var leaseToken, leaseOwner *string + err := row.Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + ) + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + return err +} + +func (s *Store) CompleteAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + variants []model.Variant, +) (model.JobMutationResult, error) { + if !validCompletedVariants(variants, lease.Job.PipelineVersion) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin complete attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const deleteVariants = ` + DELETE FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("replace attachment variants: %w", err) + } + + var originalKey string + const insertVariant = ` + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + sha256, width, height, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + for _, variant := range variants { + if variant.Name == "original" { + originalKey = variant.ObjectKey + } + if _, err := tx.Exec( + ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name, + variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width, + variant.Height, variant.ObjectKey, + ); err != nil { + return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err) + } + } + + const completeJob = ` + UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil { + return "", fmt.Errorf("complete attachment job: %w", err) + } + const completeAttachment = ` + UPDATE attachments + SET status = 'ready', object_key = $3, ready_at = clock_timestamp(), + staging_key = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { + return "", fmt.Errorf("complete attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit completed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func validCompletedVariants(variants []model.Variant, pipelineVersion int) bool { + if len(variants) != 4 || pipelineVersion < 1 { + return false + } + required := map[string]struct{}{ + "original": {}, "thumbnail": {}, "crop": {}, "optimized": {}, + } + for _, variant := range variants { + if variant.ObjectKey == "" || variant.PipelineVersion != pipelineVersion { + return false + } + if _, exists := required[variant.Name]; !exists { + return false + } + delete(required, variant.Name) + } + return len(required) == 0 +} + +func (s *Store) RetryAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, + nextAttemptAt time.Time, + traceparent string, + tracestate string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin retry attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = nextAttemptAt.UTC() + job.ErrorCode = &errorCode + const retryJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $3, available_at = $4, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $5, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec( + ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode, + ); err != nil { + return "", fmt.Errorf("retry attachment job: %w", err) + } + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("requeue attachment: %w", err) + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit attachment job retry: %w", err) + } + return model.JobMutationApplied, nil +} + +func (s *Store) FailAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin fail attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $3, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil { + return "", fmt.Errorf("fail attachment job: %w", err) + } + const failAttachment = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return "", fmt.Errorf("fail attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit failed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func lockJobMutation( + ctx context.Context, + tx pgx.Tx, + lease model.AttachmentJobLease, +) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) { + job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID) + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, "", err + } + if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion { + return job, attachment, model.JobMutationStale, nil + } + if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady { + return job, attachment, model.JobMutationAlreadyComplete, nil + } + if job.PipelineVersion != lease.Job.PipelineVersion || + job.Attempt != lease.Job.Attempt || + job.Status != model.AttachmentJobProcessing || + job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken || + job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) { + return job, attachment, model.JobMutationStale, nil + } + return job, attachment, model.JobMutationApplied, nil +} + +func validErrorCode(errorCode string) bool { + return len(errorCode) >= 1 && len(errorCode) <= 64 +} diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go new file mode 100644 index 0000000..9e631a9 --- /dev/null +++ b/internal/postgres/attachment_lease.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE attachment_jobs + SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'), + updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_until > clock_timestamp()` + result, err := s.pool.Exec( + ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(), + ) + if err != nil { + return false, fmt.Errorf("renew attachment job lease: %w", err) + } + return result.RowsAffected() == 1, nil +} diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go new file mode 100644 index 0000000..e305a79 --- /dev/null +++ b/internal/postgres/attachment_recovery.go @@ -0,0 +1,210 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) RecoverExpiredAttachmentJobs( + ctx context.Context, + limit int, + maxAttempts int, + retryAt time.Time, +) (model.RecoveryResult, error) { + if limit < 1 || limit > 1000 || maxAttempts < 1 { + return model.RecoveryResult{}, model.ErrInvalidInput + } + if retryAt.IsZero() { + retryAt = time.Now().UTC() + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectExpired = ` + SELECT id + FROM attachment_jobs + WHERE status = 'processing' AND lease_until <= clock_timestamp() + ORDER BY lease_until, id + LIMIT $1 + FOR UPDATE SKIP LOCKED` + rows, err := tx.Query(ctx, selectExpired, limit) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err) + } + jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err) + } + + var recovered model.RecoveryResult + for _, jobID := range jobIDs { + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.RecoveryResult{}, err + } + if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) { + continue + } + if job.PipelineVersion != attachment.PipelineVersion { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + if job.Attempt >= maxAttempts { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = retryAt.UTC() + const requeueJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = 'lease_expired', updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err) + } + const requeueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.RecoveryResult{}, err + } + recovered.Requeued++ + } + if err := tx.Commit(ctx); err != nil { + return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err) + } + return recovered, nil +} + +func markExpiredJobFailed( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + attachment model.Attachment, + errorCode string, + failAttachment bool, +) error { + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $2, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { + return fmt.Errorf("fail expired attachment job: %w", err) + } + if !failAttachment { + return nil + } + const failAttachmentQuery = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return fmt.Errorf("fail attachment after expired lease: %w", err) + } + return nil +} + +func (s *Store) ReplayFailedAttachment( + ctx context.Context, + attachmentID string, + actor string, + reason string, +) (model.AttachmentJob, error) { + if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 { + return model.AttachmentJob{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectJob = ` + SELECT j.id + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed' + FOR UPDATE OF j, a` + var jobID string + if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrConflict + } + return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err) + } + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJob{}, err + } + previousAttempt := job.Attempt + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = time.Now().UTC() + job.ErrorCode = nil + job.LeaseToken = "" + job.LeaseOwner = "" + job.LeaseUntil = nil + const replayJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err) + } + const replayAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err) + } + const insertAudit = ` + INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID, + previousAttempt, job.Attempt, actor, reason, + ); err != nil { + return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.AttachmentJob{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err) + } + return job, nil +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go new file mode 100644 index 0000000..2765eda --- /dev/null +++ b/internal/postgres/attachments.go @@ -0,0 +1,393 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const attachmentColumns = ` + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key` + +type rowScanner interface { + Scan(...any) error +} + +func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { + query := ` + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING ` + attachmentColumns + + err := scanAttachment(s.pool.QueryRow( + ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename, + attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey, + ), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + query := ` + UPDATE attachments + SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error { + const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'` + if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil { + return fmt.Errorf("delete pending attachment: %w", err) + } + return nil +} + +func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 + WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) + } + if attachment.Status == model.AttachmentReady { + attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion) + if err != nil { + return model.Attachment{}, err + } + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentVariantForUser( + ctx context.Context, + attachmentID string, + userID string, + variantName string, +) (model.Attachment, model.Variant, error) { + attachment, err := s.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, model.Variant{}, err + } + if attachment.Status != model.AttachmentReady { + return model.Attachment{}, model.Variant{}, ErrConflict + } + for _, variant := range attachment.Variants { + if variant.Name == variantName { + return attachment, variant, nil + } + } + if variantName == "original" && attachment.ObjectKey != "" { + return attachment, model.Variant{ + Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey, + PipelineVersion: attachment.PipelineVersion, + }, nil + } + return model.Attachment{}, model.Variant{}, ErrNotFound +} + +func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) { + const query = ` + SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY CASE name + WHEN 'original' THEN 0 + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END` + rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion) + if err != nil { + return nil, fmt.Errorf("list attachment variants: %w", err) + } + defer rows.Close() + + variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant]) + if err != nil { + return nil, fmt.Errorf("scan attachment variants: %w", err) + } + return variants, nil +} + +func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1 + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1 + )` + var exists bool + if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { + return false, fmt.Errorf("check attachment by object key: %w", err) + } + return exists, nil +} + +func (s *Store) AttachmentObjectKeys(ctx context.Context, attachmentID string) ([]string, error) { + const query = ` + SELECT object_key + FROM ( + SELECT object_key + FROM attachments + WHERE id = $1 AND object_key <> '' + UNION + SELECT object_key + FROM attachment_variants + WHERE attachment_id = $1 + ) objects + ORDER BY object_key` + rows, err := s.pool.Query(ctx, query, attachmentID) + if err != nil { + return nil, fmt.Errorf("list attachment object keys: %w", err) + } + defer rows.Close() + + keys, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("scan attachment object keys: %w", err) + } + return keys, nil +} + +func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") + } + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + `, + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id), + objects.object_key + FROM ( + SELECT id AS attachment_id, object_key + FROM attachments + WHERE object_key <> '' + UNION + SELECT attachment_id, object_key + FROM attachment_variants + ) objects + JOIN attachments a ON a.id = objects.attachment_id + WHERE objects.object_key > $1 + ORDER BY objects.object_key + LIMIT $2` + rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) + if err != nil { + return nil, "", fmt.Errorf("list attachments: %w", err) + } + defer rows.Close() + + attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { + var attachment model.Attachment + var objectKey string + err := scanAttachmentWithReferenceAndObjectKey(row, &attachment, &objectKey) + attachment.ObjectKey = objectKey + return publicAttachment(attachment), err + })) + if err != nil { + return nil, "", fmt.Errorf("scan attachments: %w", err) + } + var next string + if len(attachments) > limit { + attachments = attachments[:limit] + next = attachments[len(attachments)-1].ObjectKey + } + return attachments, next, nil +} + +func scanAttachmentWithReferenceAndObjectKey( + row rowScanner, + attachment *model.Attachment, + objectKey *string, +) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, objectKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` + var referenced bool + if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil { + return false, fmt.Errorf("check attachment references: %w", err) + } + return referenced, nil +} + +func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return false, fmt.Errorf("begin attachment deletion claim: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + var status string + var createdAt time.Time + var readyAt *time.Time + err = tx.QueryRow( + ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID, + ).Scan(&status, &createdAt, &readyAt) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("lock attachment deletion candidate: %w", err) + } + if status != candidate.Status { + return false, nil + } + var age time.Time + switch status { + case model.AttachmentPending: + age = createdAt + case model.AttachmentReady: + if readyAt == nil { + return false, nil + } + age = *readyAt + default: + return false, nil + } + if age.After(createdBefore) { + return false, nil + } + var referenced bool + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil { + return false, fmt.Errorf("recheck attachment references: %w", err) + } + if referenced { + return false, nil + } + if _, err := tx.Exec( + ctx, + `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`, + candidate.ID, candidate.Status, + ); err != nil { + return false, fmt.Errorf("mark attachment deleting: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return false, fmt.Errorf("commit attachment deletion claim: %w", err) + } + return true, nil +} + +func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) { + const query = ` + DELETE FROM attachments a + WHERE a.id = $1 + AND a.status = 'deleting' + AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)` + result, err := s.pool.Exec(ctx, query, attachmentID) + if err != nil { + return false, fmt.Errorf("delete claimed attachment: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func scanAttachment(row rowScanner, attachment *model.Attachment) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func attachmentColumnsWithAlias(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ + "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+ + "%[1]s.error_code, %[1]s.staging_key", + alias, + ) +} + +func publicAttachment(attachment model.Attachment) model.Attachment { + variants := make([]model.Variant, 0, len(attachment.Variants)) + for _, variant := range attachment.Variants { + if variant.Name != "original" { + variants = append(variants, variant) + } + } + attachment.Variants = variants + return attachment +} diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go new file mode 100644 index 0000000..39509ad --- /dev/null +++ b/internal/postgres/errors.go @@ -0,0 +1,17 @@ +package postgres + +import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + +var ( + ErrConflict = model.ErrConflict + ErrNotFound = model.ErrNotFound + ErrInvalidMembers = model.ErrInvalidMembers + ErrInvalidCursor = model.ErrInvalidCursor + ErrIdempotencyConflict = model.ErrIdempotencyConflict + ErrStaleLease = model.ErrStaleLease + ErrStaleAttempt = model.ErrStaleAttempt + ErrStalePipeline = model.ErrStalePipeline + ErrJobBusy = model.ErrJobBusy + ErrJobComplete = model.ErrJobComplete + ErrJobNotDue = model.ErrJobNotDue +) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go new file mode 100644 index 0000000..4bbb95b --- /dev/null +++ b/internal/postgres/idempotency.go @@ -0,0 +1,374 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const expiredIdempotencyCleanupBatch = 100 + +func (s *Store) CreateMessage( + ctx context.Context, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + var ( + tx pgx.Tx + err error + visible bool + requestHash [sha256.Size]byte + message model.Message + replay bool + ) + + tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID) + if err != nil { + return model.Message{}, false, err + } + if !visible { + return model.Message{}, false, ErrNotFound + } + validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + if !validAttachments { + return model.Message{}, false, ErrNotFound + } + if failedAttachment { + return model.Message{}, false, ErrConflict + } + if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { + return model.Message{}, false, err + } + + requestHash, err = messageRequestHash(text, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + message, replay, err = createIdempotentMessage( + ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err) + } + return message, replay, nil +} + +func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + payload, err := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, attachmentIDs}) + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err) + } + return sha256.Sum256(payload), nil +} + +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) { + if len(attachmentIDs) == 0 { + return true, false, nil + } + const query = ` + SELECT id, status + FROM attachments + WHERE chat_id = $1 AND id::text = ANY($2) + AND status IN ('queued', 'processing', 'ready', 'failed') + FOR UPDATE` + rows, err := tx.Query(ctx, query, chatID, attachmentIDs) + if err != nil { + return false, false, fmt.Errorf("validate message attachments: %w", err) + } + defer rows.Close() + var count int + var failed bool + for rows.Next() { + var id, status string + if err := rows.Scan(&id, &status); err != nil { + return false, false, fmt.Errorf("scan message attachment: %w", err) + } + count++ + failed = failed || status == model.AttachmentFailed + } + if err := rows.Err(); err != nil { + return false, false, fmt.Errorf("validate message attachments: %w", err) + } + return count == len(attachmentIDs), failed, nil +} + +func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var visible bool + if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + return visible, nil +} + +func createIdempotentMessage( + ctx context.Context, + tx pgx.Tx, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + requestHash []byte, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + for { + var ( + record idempotencyRecord + found bool + err error + messageID string + reserved bool + message model.Message + ) + + record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey) + if err != nil { + return model.Message{}, false, err + } + if found && record.active { + if !bytes.Equal(record.requestHash, requestHash) { + return model.Message{}, false, ErrIdempotencyConflict + } + message, err = messageByID(ctx, tx, record.messageID) + return message, true, err + } + if found { + if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil { + return model.Message{}, false, err + } + } + + messageID = uuid.NewString() + reserved, err = reserveIdempotencyKey( + ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if !reserved { + continue + } + + message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs) + return message, false, err + } +} + +func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error { + const query = ` + WITH expired AS ( + SELECT ctid + FROM message_idempotency + WHERE expires_at <= statement_timestamp() + ORDER BY expires_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM message_idempotency records + USING expired + WHERE records.ctid = expired.ctid` + + if _, err := tx.Exec(ctx, query, limit); err != nil { + return fmt.Errorf("delete expired idempotency records: %w", err) + } + return nil +} + +type idempotencyRecord struct { + requestHash []byte + messageID string + active bool +} + +func lockIdempotencyRecord( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, +) (idempotencyRecord, bool, error) { + const query = ` + SELECT request_hash, message_id, expires_at > statement_timestamp() + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + FOR UPDATE` + + var ( + record idempotencyRecord + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key).Scan( + &record.requestHash, + &record.messageID, + &record.active, + ) + if errors.Is(err, pgx.ErrNoRows) { + return idempotencyRecord{}, false, nil + } + if err != nil { + return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err) + } + return record, true, nil +} + +func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error { + const query = ` + DELETE FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3` + if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil { + return fmt.Errorf("delete expired idempotency key: %w", err) + } + return nil +} + +func reserveIdempotencyKey( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, + requestHash []byte, + messageID string, + ttl time.Duration, +) (bool, error) { + const query = ` + INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at) + VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second') + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING true` + + var ( + inserted bool + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reserve idempotency key: %w", err) + } + return inserted, nil +} + +func insertMessage( + ctx context.Context, + tx pgx.Tx, + messageID string, + chatID string, + senderID string, + text string, + attachmentIDs []string, +) (model.Message, error) { + const query = ` + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + ) + if err != nil { + return model.Message{}, fmt.Errorf("insert message: %w", err) + } + if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil { + return model.Message{}, err + } + message.AttachmentIDs = make([]string, len(attachmentIDs)) + copy(message.AttachmentIDs, attachmentIDs) + return message, nil +} + +func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error { + const query = ` + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES ($1, $2, $3, $4)` + for position, attachmentID := range attachmentIDs { + if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil { + return fmt.Errorf("insert message attachment: %w", err) + } + } + return nil +} + +func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) { + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.id = $1` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, fmt.Errorf("idempotency result message not found") + } + if err != nil { + return model.Message{}, fmt.Errorf("get idempotency result message: %w", err) + } + return message, nil +} diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go new file mode 100644 index 0000000..0817f7c --- /dev/null +++ b/internal/postgres/outbox.go @@ -0,0 +1,136 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) ClaimOutboxEvents( + ctx context.Context, + limit int, + leaseDuration time.Duration, +) ([]model.OutboxEvent, error) { + if limit < 1 || limit > 1000 || leaseDuration <= 0 { + return nil, model.ErrInvalidInput + } + claimToken := uuid.NewString() + query := ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at IS NULL + AND available_at <= clock_timestamp() + AND (claim_until IS NULL OR claim_until <= clock_timestamp()) + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbox_events event + SET claim_token = $2, + claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond', + publish_tries = event.publish_tries + 1 + FROM candidates + WHERE event.id = candidates.id + RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id, + event.attempt, event.available_at, event.publish_tries, event.claim_token, + event.claim_until, event.last_error_code, event.created_at` + rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds()) + if err != nil { + return nil, fmt.Errorf("claim outbox events: %w", err) + } + defer rows.Close() + + events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) { + var event model.OutboxEvent + err := row.Scan( + &event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID, + &event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken, + &event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt, + ) + return event, err + })) + if err != nil { + return nil, fmt.Errorf("scan claimed outbox events: %w", err) + } + return events, nil +} + +func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) { + const query = ` + UPDATE outbox_events + SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL, + last_error_code = NULL + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken) + if err != nil { + return false, fmt.Errorf("mark outbox event published: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) ReleaseOutboxEvent( + ctx context.Context, + eventID string, + leaseToken string, + errorCode string, + nextAttemptAt time.Time, +) (bool, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE outbox_events + SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4 + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode) + if err != nil { + return false, fmt.Errorf("release outbox event: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) { + if now.IsZero() { + now = time.Now().UTC() + } + const query = ` + UPDATE outbox_events + SET claim_token = NULL, claim_until = NULL + WHERE published_at IS NULL AND claim_until <= $1` + result, err := s.pool.Exec(ctx, query, now.UTC()) + if err != nil { + return 0, fmt.Errorf("recover expired outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} + +func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) { + if publishedBefore.IsZero() || limit < 1 || limit > 1000 { + return 0, model.ErrInvalidInput + } + const query = ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at < $1 + ORDER BY published_at, id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM outbox_events event + USING candidates + WHERE event.id = candidates.id` + result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/internal/postgres/store.go b/internal/postgres/store.go new file mode 100644 index 0000000..44980b7 --- /dev/null +++ b/internal/postgres/store.go @@ -0,0 +1,442 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Ping(ctx context.Context) error { + return s.pool.Ping(ctx) +} + +func (s *Store) CreateUser( + ctx context.Context, + username string, + displayName string, + passwordHash string, +) (model.User, error) { + const query = ` + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at` + + var user model.User + err := s.pool.QueryRow(ctx, query, uuid.NewString(), username, displayName, passwordHash).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if isUniqueViolation(err) { + return model.User{}, ErrConflict + } + if err != nil { + return model.User{}, fmt.Errorf("create user: %w", err) + } + + return user, nil +} + +func (s *Store) UserByID(ctx context.Context, id string) (model.User, error) { + const query = ` + SELECT id, username, display_name, created_at + FROM users + WHERE id = $1` + + var user model.User + err := s.pool.QueryRow(ctx, query, id).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.User{}, ErrNotFound + } + if err != nil { + return model.User{}, fmt.Errorf("get user by id: %w", err) + } + + return user, nil +} + +func (s *Store) UserByUsername(ctx context.Context, username string) (model.UserCredentials, error) { + const query = ` + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1` + + var credentials model.UserCredentials + err := s.pool.QueryRow(ctx, query, username).Scan( + &credentials.ID, + &credentials.Username, + &credentials.DisplayName, + &credentials.CreatedAt, + &credentials.PasswordHash, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.UserCredentials{}, ErrNotFound + } + if err != nil { + return model.UserCredentials{}, fmt.Errorf("get user by username: %w", err) + } + + return credentials, nil +} + +func (s *Store) CreateChat( + ctx context.Context, + title string, + memberIDs []string, + callerID string, +) (model.Chat, error) { + if !validMembers(memberIDs, callerID) { + return model.Chat{}, ErrInvalidMembers + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + valid, err := allUsersExist(ctx, tx, memberIDs) + if err != nil { + return model.Chat{}, err + } + if !valid { + return model.Chat{}, ErrInvalidMembers + } + + chat, err := insertChat(ctx, tx, title) + if err != nil { + return model.Chat{}, err + } + if err := insertMembers(ctx, tx, chat.ID, memberIDs); err != nil { + return model.Chat{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Chat{}, fmt.Errorf("commit create chat transaction: %w", err) + } + + chat.MemberIDs = append([]string(nil), memberIDs...) + return chat, nil +} + +func validMembers(memberIDs []string, callerID string) bool { + callerUUID, err := uuid.Parse(callerID) + if err != nil { + return false + } + + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + callerIncluded := false + for _, memberID := range memberIDs { + memberUUID, err := uuid.Parse(memberID) + if err != nil { + return false + } + if _, exists := seen[memberUUID]; exists { + return false + } + + seen[memberUUID] = struct{}{} + callerIncluded = callerIncluded || memberUUID == callerUUID + } + + return callerIncluded +} + +func allUsersExist(ctx context.Context, tx pgx.Tx, memberIDs []string) (bool, error) { + const query = ` + SELECT count(*) + FROM users + WHERE id::text = ANY($1)` + + var count int + if err := tx.QueryRow(ctx, query, memberIDs).Scan(&count); err != nil { + return false, fmt.Errorf("check chat members: %w", err) + } + + return count == len(memberIDs), nil +} + +func insertChat(ctx context.Context, tx pgx.Tx, title string) (model.Chat, error) { + const query = ` + INSERT INTO chats (id, title) + VALUES ($1, $2) + RETURNING id, title, created_at` + + var chat model.Chat + if err := tx.QueryRow(ctx, query, uuid.NewString(), title).Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil { + return model.Chat{}, fmt.Errorf("insert chat: %w", err) + } + + return chat, nil +} + +func insertMembers(ctx context.Context, tx pgx.Tx, chatID string, memberIDs []string) error { + const query = ` + INSERT INTO chat_members (chat_id, user_id, position) + VALUES ($1, $2, $3)` + + batch := &pgx.Batch{} + for position, memberID := range memberIDs { + batch.Queue(query, chatID, memberID, position) + } + + results := tx.SendBatch(ctx, batch) + defer func() { _ = results.Close() }() + for range memberIDs { + if _, err := results.Exec(); err != nil { + return fmt.Errorf("insert chat member: %w", err) + } + } + if err := results.Close(); err != nil { + return fmt.Errorf("close chat members batch: %w", err) + } + + return nil +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (model.Chat, error) { + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id::text ORDER BY cm.position) + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = $1 + AND EXISTS ( + SELECT 1 + FROM chat_members visible + WHERE visible.chat_id = c.id AND visible.user_id = $2 + ) + GROUP BY c.id, c.title, c.created_at` + + var chat model.Chat + err := s.pool.QueryRow(ctx, query, chatID, userID).Scan( + &chat.ID, + &chat.Title, + &chat.CreatedAt, + &chat.MemberIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Chat{}, ErrNotFound + } + if err != nil { + return model.Chat{}, fmt.Errorf("get chat for user: %w", err) + } + + return chat, nil +} + +func (s *Store) ListChats( + ctx context.Context, + userID string, + after *model.Position, + limit int, +) ([]model.Chat, bool, error) { + if after != nil { + valid, err := s.visibleChatPositionExists(ctx, userID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(all_members.user_id::text ORDER BY all_members.position) + FROM chats c + JOIN chat_members visible ON visible.chat_id = c.id AND visible.user_id = $1 + JOIN chat_members all_members ON all_members.chat_id = c.id + WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at, c.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, userID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list chats: %w", err) + } + defer rows.Close() + + chats, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Chat](func(row pgx.CollectableRow) (model.Chat, error) { + var chat model.Chat + err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + return chat, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan chats: %w", err) + } + + return page(chats, limit) +} + +func (s *Store) visibleChatPositionExists( + ctx context.Context, + userID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1 AND c.created_at = $2 AND c.id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, userID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate chat cursor: %w", err) + } + + return exists, nil +} + +func (s *Store) ListMessages( + ctx context.Context, + chatID string, + userID string, + after *model.Position, + limit int, +) ([]model.Message, bool, error) { + visible, err := s.chatVisible(ctx, chatID, userID) + if err != nil { + return nil, false, err + } + if !visible { + return nil, false, ErrNotFound + } + if after != nil { + valid, err := s.messagePositionExists(ctx, chatID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.chat_id = $1 + AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid)) + ORDER BY m.created_at, m.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, chatID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list messages: %w", err) + } + defer rows.Close() + + messages, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Message](func(row pgx.CollectableRow) (model.Message, error) { + var message model.Message + err := row.Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + return message, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan messages: %w", err) + } + + return page(messages, limit) +} + +func (s *Store) chatVisible(ctx context.Context, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, userID).Scan(&exists); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + + return exists, nil +} + +func (s *Store) messagePositionExists( + ctx context.Context, + chatID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM messages + WHERE chat_id = $1 AND created_at = $2 AND id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate message cursor: %w", err) + } + + return exists, nil +} + +func page[T any](items []T, limit int) ([]T, bool, error) { + if limit < 1 { + return nil, false, fmt.Errorf("limit must be positive") + } + if len(items) <= limit { + return items, false, nil + } + + return items[:limit], true, nil +} + +func isUniqueViolation(err error) bool { + var postgresError *pgconn.PgError + return errors.As(err, &postgresError) && postgresError.Code == "23505" +} diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go new file mode 100644 index 0000000..e4184db --- /dev/null +++ b/internal/postgres/store_integration_test.go @@ -0,0 +1,711 @@ +package postgres_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" +) + +type PostgresSuite struct { + suite.Suite + pool *pgxpool.Pool + store *postgres.Store +} + +func TestPostgresSuite(t *testing.T) { + suite.Run(t, new(PostgresSuite)) +} + +func (s *PostgresSuite) SetupSuite() { + s.pool = openTestDatabase(s.T()) + s.store = postgres.New(s.pool) +} + +func (s *PostgresSuite) TearDownSuite() { + if s.pool != nil { + s.pool.Close() + } +} + +func (s *PostgresSuite) TestConcurrentUsernameCreation() { + t := s.T() + store := s.store + username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] + + const workers = 8 + results := make(chan error, workers) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + _, err := store.CreateUser(context.Background(), username, "Race user", "password-hash") + results <- err + }() + } + group.Wait() + close(results) + + var created, conflicts int + for err := range results { + switch { + case err == nil: + created++ + case errors.Is(err, postgres.ErrConflict): + conflicts++ + default: + t.Fatalf("CreateUser() error = %v", err) + } + } + if created != 1 || conflicts != workers-1 { + t.Fatalf("created = %d, conflicts = %d", created, conflicts) + } +} + +func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() { + t := s.T() + pool := s.pool + store := s.store + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + + before := chatCount(t, pool) + _, err = store.CreateChat(context.Background(), "Invalid chat", []string{owner.ID, uuid.NewString()}, owner.ID) + if !errors.Is(err, postgres.ErrInvalidMembers) { + t.Fatalf("CreateChat() error = %v, want ErrInvalidMembers", err) + } + if after := chatCount(t, pool); after != before { + t.Fatalf("chat count changed from %d to %d", before, after) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + key := uuid.NewString() + + first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour) + if err != nil || replay { + t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err) + } + second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour) + if err != nil || !replay { + t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err) + } + s.Equal(first, second) + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { + t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash") + s.Require().NoError(err) + attachment := model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + } + pending, err := s.store.CreatePendingAttachment(context.Background(), attachment) + s.Require().NoError(err) + created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + s.Equal("ready", created.Status) + s.Empty(created.Variants) + s.Nil(created.ErrorCode) + + visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID) + s.Require().NoError(err) + s.Equal(created, visible) + _, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID) + s.ErrorIs(err, postgres.ErrNotFound) + + message, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + s.Equal([]string{created.ID}, message.AttachmentIDs) + + messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10) + s.Require().NoError(err) + s.Len(messages, 1) + s.Equal([]string{created.ID}, messages[0].AttachmentIDs) + referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) + s.Require().NoError(err) + s.True(referenced) + deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(deleted) +} + +func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + s.Equal(model.AttachmentQueued, attachment.Status) + + events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute) + s.Require().NoError(err) + s.Require().NotEmpty(events) + var event model.OutboxEvent + for _, candidate := range events { + if candidate.JobID == job.ID { + event = candidate + break + } + } + s.Require().NotEmpty(event.ID) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(event.Payload, &message)) + s.Equal(job.ID, message.JobID) + s.Equal(attachment.ID, message.AttachmentID) + s.Equal(1, message.Attempt) + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + + _, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "queued attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute, + ) + s.ErrorIs(err, model.ErrInvalidInput) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + s.Equal(model.AttachmentProcessing, lease.Attachment.Status) + s.NotEmpty(lease.Attachment.StagingKey) + + variants := completedVariants(attachment.ID, 1) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationAlreadyComplete, result) + + metadata, variant, err := s.store.AttachmentVariantForUser( + context.Background(), attachment.ID, owner.ID, "thumbnail", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, metadata.Status) + s.Len(metadata.Variants, 3) + s.Equal(variants[1].ObjectKey, variant.ObjectKey) + listedKeys := make(map[string]struct{}) + var after string + for { + listed, next, err := s.store.ListAttachments(context.Background(), after, 1000) + s.Require().NoError(err) + for _, candidate := range listed { + if candidate.ID == attachment.ID { + listedKeys[candidate.ObjectKey] = struct{}{} + } + } + if next == "" { + break + } + after = next + } + for _, candidate := range variants { + s.Contains(listedKeys, candidate.ObjectKey) + } + + v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2} + queuedV2, err := s.store.QueueAttachmentPipelineVersion( + context.Background(), attachment.ID, v2, "", "", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentQueued, queuedV2.Status) + s.Equal(2, queuedV2.PipelineVersion) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + _, secondJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + events, err := s.store.ClaimOutboxEvents(context.Background(), 1000, time.Minute) + s.Require().NoError(err) + + eventIDs := make([]string, 0, 2) + for _, event := range events { + if event.JobID != firstJob.ID && event.JobID != secondJob.ID { + continue + } + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + eventIDs = append(eventIDs, event.ID) + } + s.Require().Len(eventIDs, 2) + _, err = s.pool.Exec( + context.Background(), + `UPDATE outbox_events SET published_at = clock_timestamp() - INTERVAL '2 days' WHERE id IN ($1, $2)`, + eventIDs[0], eventIDs[1], + ) + s.Require().NoError(err) + + deleted, err := s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + deleted, err = s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + + var remaining int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM outbox_events WHERE id IN ($1, $2)`, eventIDs[0], eventIDs[1], + ).Scan(&remaining) + s.Require().NoError(err) + s.Zero(remaining) +} + +func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + + _, _, err = s.store.CreateMessage( + context.Background(), chatID, owner.ID, "failed attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrConflict) + metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentFailed, metadata.Status) + s.Require().NotNil(metadata.ErrorCode) + s.Equal("invalid_image", *metadata.ErrorCode) + + replayed, err := s.store.ReplayFailedAttachment( + context.Background(), attachment.ID, "operator@example", "input repaired", + ) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + s.Equal(model.AttachmentJobQueued, replayed.Status) + + var auditCount int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID, + ).Scan(&auditCount) + s.Require().NoError(err) + s.Equal(1, auditCount) +} + +func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now()) + s.Require().NoError(err) + s.Equal(1, recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(2, current.Attempt) + s.Equal(model.AttachmentJobQueued, current.Status) + + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + + complete := completedVariants(attachment.ID, job.PipelineVersion) + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, complete[:3]) + s.ErrorIs(err, model.ErrInvalidInput) + + duplicate := append([]model.Variant(nil), complete...) + duplicate[3].Name = duplicate[2].Name + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, duplicate) + s.ErrorIs(err, model.ErrInvalidInput) + + mismatched := append([]model.Variant(nil), complete...) + mismatched[3].PipelineVersion++ + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, mismatched) + s.ErrorIs(err, model.ErrInvalidInput) +} + +func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID) + s.Require().NoError(err) + s.True(deleted) + + pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + + claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(claimed) + visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, visible.Status) +} + +func (s *PostgresSuite) TestReconciliationDryRunPreservesDeletingAttachment() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "deleting.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("e", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + + reconciler, err := reconcile.New(s.store, emptyObjectStore{}, reconcile.Config{ + Prefix: "attachments/", PageSize: 1000, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, + }) + s.Require().NoError(err) + report, err := reconciler.Run(context.Background()) + s.Require().NoError(err) + s.Zero(report.DeletedUnreferencedRows) + + var status string + err = s.pool.QueryRow(context.Background(), "SELECT status FROM attachments WHERE id = $1", pending.ID).Scan(&status) + s.Require().NoError(err) + s.Equal(model.AttachmentDeleting, status) +} + +type emptyObjectStore struct{} + +func (emptyObjectStore) List(context.Context, string, string, int) ([]model.StoredObject, string, error) { + return nil, "", nil +} + +func (emptyObjectStore) Delete(context.Context, string) error { + return errors.New("unexpected object deletion") +} + +func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { + owner, firstChatID := createOwnerAndChat(s.T(), s.store) + secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) + s.Require().NoError(err) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + _, _, err = s.store.CreateMessage( + context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrNotFound) +} + +func (s *PostgresSuite) TestConcurrentMessageIdempotency() { + t := s.T() + pool := s.pool + stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)} + owner, chatID := createOwnerAndChat(t, stores[0]) + key := uuid.NewString() + + const workers = 8 + type result struct { + message model.Message + replay bool + err error + } + results := make(chan result, workers) + var group sync.WaitGroup + for worker := range workers { + group.Add(1) + go func() { + defer group.Done() + message, replay, err := stores[worker%len(stores)].CreateMessage( + context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour, + ) + results <- result{message: message, replay: replay, err: err} + }() + } + group.Wait() + close(results) + + var messageID string + var created int + for result := range results { + if result.err != nil { + t.Fatalf("CreateMessage() error = %v", result.err) + } + if messageID == "" { + messageID = result.message.ID + } + if result.message.ID != messageID { + t.Fatalf("message ID = %s, want %s", result.message.ID, messageID) + } + if !result.replay { + created++ + } + } + if created != 1 { + t.Fatalf("new message results = %d, want 1", created) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + key := uuid.NewString() + + if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil { + t.Fatalf("owner CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil { + t.Fatalf("second user CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil { + t.Fatalf("other chat CreateMessage() error = %v", err) + } + if _, err := pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour) + if err != nil || replay || message.Text != "after expiry" { + t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err) + } +} + +func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { + t := s.T() + owner, chatID := createOwnerAndChat(t, s.store) + expiredKey := uuid.NewString() + if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil { + t.Fatalf("create expired message: %v", err) + } + if _, err := s.pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + if _, _, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour, + ); err != nil { + t.Fatalf("create new message: %v", err) + } + + var remaining int + if err := s.pool.QueryRow(context.Background(), ` + SELECT count(*) + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil { + t.Fatalf("count expired idempotency records: %v", err) + } + if remaining != 0 { + t.Fatalf("expired idempotency records = %d, want 0", remaining) + } +} + +func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) { + t.Helper() + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + return owner, chat.ID +} + +func createQueuedAttachment( + t *testing.T, + store *postgres.Store, + ownerID string, + chatID string, +) (model.Attachment, model.AttachmentJob) { + t.Helper() + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: stagingKey, + }) + if err != nil { + t.Fatalf("CreatePendingAttachment() error = %v", err) + } + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1, + } + queued, err := store.QueueAttachmentWithJobAndOutbox( + context.Background(), pending.ID, stagingKey, job, "", "", + ) + if err != nil { + t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err) + } + return queued, job +} + +func completedVariants(attachmentID string, pipelineVersion int) []model.Variant { + types := []struct { + name string + contentType string + extension string + }{ + {name: "original", contentType: "image/png", extension: "png"}, + {name: "thumbnail", contentType: "image/jpeg", extension: "jpg"}, + {name: "crop", contentType: "image/jpeg", extension: "jpg"}, + {name: "optimized", contentType: "image/webp", extension: "webp"}, + } + variants := make([]model.Variant, 0, len(types)) + for index, variant := range types { + variants = append(variants, model.Variant{ + Name: variant.name, ContentType: variant.contentType, SizeBytes: int64(128 - index*16), + SHA256: strings.Repeat(string(rune('a'+index)), 64), Width: 320 - index*32, Height: 200 - index*20, + ObjectKey: fmt.Sprintf("attachments/%s/v%d/%s.%s", attachmentID, pipelineVersion, variant.name, variant.extension), + PipelineVersion: pipelineVersion, + }) + } + return variants +} + +func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + return count +} + +func openTestDatabase(t *testing.T) *pgxpool.Pool { + t.Helper() + url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE") + pool, err := database.Open(context.Background(), database.Config{ + URL: url, MinConnections: 0, MaxConnections: 10, + MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute, + HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("database.Open() error = %v", err) + } + if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil { + pool.Close() + t.Fatalf("database.Migrate() error = %v", err) + } + return pool +} + +func testSecret(t *testing.T, valueName, fileName string) string { + t.Helper() + if value := strings.TrimSpace(os.Getenv(valueName)); value != "" { + return value + } + path := strings.TrimSpace(os.Getenv(fileName)) + if path == "" { + t.Skipf("set %s or %s to run integration tests", valueName, fileName) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", fileName, err) + } + return strings.TrimSpace(string(data)) +} + +func uniqueUsername() string { + return "u_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] +} + +func chatCount(t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM chats").Scan(&count); err != nil { + t.Fatalf("count chats: %v", err) + } + return count +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..15607f8 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,410 @@ +package reconcile + +import ( + "context" + "errors" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type MetadataStore interface { + ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentObjectKeys(context.Context, string) ([]string, error) + AttachmentExistsByObjectKey(context.Context, string) (bool, error) + ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) + DeleteAttachmentIfDeleting(context.Context, string) (bool, error) +} + +type ObjectStore interface { + List(context.Context, string, string, int) ([]model.StoredObject, string, error) + Delete(context.Context, string) error +} + +type Config struct { + Prefix string + OrphanPrefixes []string + PageSize int + GraceTime time.Duration + UnreferencedRetention time.Duration + Apply bool + Now func() time.Time +} + +type Report struct { + MetadataScanned int `json:"metadata_scanned"` + ObjectsScanned int `json:"objects_scanned"` + Consistent int `json:"consistent"` + FreshPendingAttachments int `json:"fresh_pending_attachments"` + OldPendingAttachments int `json:"old_pending_attachments"` + DeletingAttachments int `json:"deleting_attachments"` + FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"` + OldUnreferencedAttachments int `json:"old_unreferenced_attachments"` + DeletedAttachmentObjects int `json:"deleted_attachment_objects"` + FreshOrphanObjects int `json:"fresh_orphan_objects"` + OldOrphanObjects int `json:"old_orphan_objects"` + DeletedOrphanObjects int `json:"deleted_orphan_objects"` + FreshMissingObjects int `json:"fresh_missing_objects"` + ReferencedMissingObjects int `json:"referenced_missing_objects"` + OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` + DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` +} + +type Reconciler struct { + metadata MetadataStore + objects ObjectStore + config Config +} + +func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) { + if config.PageSize < 1 || config.PageSize > 1000 { + return nil, errors.New("page size must be between 1 and 1000") + } + if config.GraceTime < 0 { + return nil, errors.New("grace time must not be negative") + } + if config.UnreferencedRetention <= 0 { + return nil, errors.New("unreferenced retention must be positive") + } + for _, prefix := range config.OrphanPrefixes { + if prefix == "" || prefix == config.Prefix { + return nil, errors.New("orphan prefix must be non-empty and distinct") + } + } + if config.Now == nil { + config.Now = time.Now + } + return &Reconciler{metadata: metadata, objects: objects, config: config}, nil +} + +func (r *Reconciler) Run(ctx context.Context) (Report, error) { + metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize) + if err != nil { + return Report{}, err + } + objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize) + if err != nil { + return Report{}, err + } + + now := r.config.Now() + anomalyCutoff := now.Add(-r.config.GraceTime) + retentionCutoff := now.Add(-r.config.UnreferencedRetention) + var report Report + for metadata.current != nil || objects.current != nil { + switch { + case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey: + if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + case objects.current == nil || metadata.current.ObjectKey < objects.current.Key: + if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + default: + if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + } + } + for _, prefix := range r.config.OrphanPrefixes { + if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil { + return report, err + } + } + return report, nil +} + +func (r *Reconciler) scanOrphanPrefix( + ctx context.Context, + prefix string, + cutoff time.Time, + report *Report, +) error { + objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize) + if err != nil { + return err + } + for objects.current != nil { + if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil { + return err + } + if err := objects.advance(ctx); err != nil { + return err + } + } + return nil +} + +func (r *Reconciler) handlePair( + ctx context.Context, + metadata model.Attachment, + retentionCutoff time.Time, + anomalyCutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + report.ObjectsScanned++ + switch metadata.Status { + case model.AttachmentQueued, model.AttachmentProcessing, model.AttachmentFailed: + report.Consistent++ + return nil + case model.AttachmentPending: + if metadata.CreatedAt.After(anomalyCutoff) { + report.FreshPendingAttachments++ + report.Consistent++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, anomalyCutoff, report) + case model.AttachmentDeleting: + report.DeletingAttachments++ + return r.deleteClaimedPair(ctx, metadata, report) + case model.AttachmentReady: + if metadata.Referenced { + report.Consistent++ + return nil + } + age := metadata.CreatedAt + if metadata.ReadyAt != nil { + age = *metadata.ReadyAt + } + if age.After(retentionCutoff) { + report.FreshUnreferencedAttachments++ + report.Consistent++ + return nil + } + report.OldUnreferencedAttachments++ + return r.deletePair(ctx, metadata, retentionCutoff, report) + default: + return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status) + } +} + +func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error { + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff) + if err != nil { + return fmt.Errorf("claim attachment %q: %w", attachment.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedPair(ctx, attachment, report) +} + +func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + objectKeys, err := r.metadata.AttachmentObjectKeys(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("list attachment %q object keys: %w", attachment.ID, err) + } + for _, objectKey := range objectKeys { + if err := r.objects.Delete(ctx, objectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", objectKey, err) + } + report.DeletedAttachmentObjects++ + } + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +func (r *Reconciler) handleOrphanObject( + ctx context.Context, + object model.StoredObject, + cutoff time.Time, + report *Report, +) error { + report.ObjectsScanned++ + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + report.Consistent++ + return nil + } + if object.LastModified.After(cutoff) { + report.FreshOrphanObjects++ + return nil + } + report.OldOrphanObjects++ + if !r.config.Apply { + return nil + } + if err := r.objects.Delete(ctx, object.Key); err != nil { + return fmt.Errorf("delete orphan object %q: %w", object.Key, err) + } + report.DeletedOrphanObjects++ + return nil +} + +func (r *Reconciler) handleMissingObject( + ctx context.Context, + metadata model.Attachment, + cutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + if metadata.Status == model.AttachmentQueued || + metadata.Status == model.AttachmentProcessing || + metadata.Status == model.AttachmentFailed { + return nil + } + if metadata.Status == model.AttachmentPending { + if metadata.CreatedAt.After(cutoff) { + report.FreshPendingAttachments++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, cutoff, report) + } + if metadata.Status == model.AttachmentDeleting { + report.DeletingAttachments++ + return r.deleteClaimedMetadata(ctx, metadata, report) + } + if metadata.CreatedAt.After(cutoff) { + report.FreshMissingObjects++ + return nil + } + if metadata.Referenced { + report.ReferencedMissingObjects++ + return nil + } + report.OldUnreferencedMetadata++ + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff) + if err != nil { + return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedMetadata(ctx, metadata, report) +} + +func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) + if err != nil { + return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +type metadataIterator struct { + store MetadataStore + pageSize int + after string + page []model.Attachment + index int + current *model.Attachment + done bool +} + +func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) { + iterator := &metadataIterator{store: store, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *metadataIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list attachment metadata: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("metadata page is empty but has continuation") + } + } +} + +type objectIterator struct { + store ObjectStore + prefix string + pageSize int + after string + page []model.StoredObject + index int + current *model.StoredObject + done bool +} + +func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) { + iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *objectIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("object page is empty but has continuation") + } + } +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 0000000..36a2955 --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,340 @@ +package reconcile + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func TestReconcilerLifecycleAndRepeat(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + fresh := now.Add(-10 * time.Minute) + metadata := &metadataStub{items: map[string]model.Attachment{ + "attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false), + "attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true), + "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), + "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), + "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), + "staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, + "attachments/b/original": {Key: "attachments/b/original", LastModified: old}, + "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, + "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, + "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + "staging/orphan/original": {Key: "staging/orphan/original", LastModified: old}, + "staging/pending/original": {Key: "staging/pending/original", LastModified: old}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + OrphanPrefixes: []string{"staging/"}, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, report.OldPendingAttachments) + require.Equal(t, 1, report.OldUnreferencedAttachments) + require.Equal(t, 1, report.FreshUnreferencedAttachments) + require.Equal(t, 2, report.DeletedOrphanObjects) + require.Equal(t, 3, report.DeletedAttachmentObjects) + require.Equal(t, 4, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, "attachments/b/original") + require.Contains(t, metadata.items, "attachments/d/original") + require.NotContains(t, metadata.items, "attachments/a/original") + require.NotContains(t, metadata.items, "attachments/c/original") + require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, metadata.items, "staging/pending/original") + require.NotContains(t, objects.items, "attachments/orphan/original") + require.NotContains(t, objects.items, "staging/orphan/original") + require.NotContains(t, objects.items, "staging/pending/original") + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedOrphanObjects) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { + now := time.Now().UTC() + metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedOrphanObjects) + require.Contains(t, objects.items, "attachments/a/original") +} + +func TestReconcilerDeletingAttachmentRespectsApplyMode(t *testing.T) { + now := time.Now().UTC() + key := "attachments/a/original" + tests := []struct { + name string + apply bool + objectPresent bool + }{ + {name: "dry run with object", objectPresent: true}, + {name: "dry run without object"}, + {name: "apply with object", apply: true, objectPresent: true}, + {name: "apply without object", apply: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentDeleting, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: make(map[string]model.StoredObject)} + if test.objectPresent { + objects.items[key] = model.StoredObject{Key: key, LastModified: now.Add(-time.Hour)} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: test.apply, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, report.DeletingAttachments) + var expectedDeletedRows, expectedDeletedObjects int + if test.apply { + expectedDeletedRows = 1 + } + if test.apply && test.objectPresent { + expectedDeletedObjects = 1 + } + require.Equal(t, expectedDeletedRows, report.DeletedUnreferencedRows) + require.Equal(t, expectedDeletedObjects, report.DeletedAttachmentObjects) + _, metadataExists := metadata.items[key] + require.Equal(t, !test.apply, metadataExists) + _, objectExists := objects.items[key] + require.Equal(t, test.objectPresent && !test.apply, objectExists) + }) + } +} + +func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) { + now := time.Now().UTC() + key := "staging/a/original" + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentQueued, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + key: {Key: key, LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", OrphanPrefixes: []string{"staging/"}, PageSize: 1, + GraceTime: time.Minute, UnreferencedRetention: time.Hour, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.OldOrphanObjects) + require.Zero(t, report.FreshOrphanObjects) + require.Contains(t, objects.items, key) +} + +func TestReconcilerPreservesAttachmentDuringReprocessing(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, status := range []string{ + model.AttachmentQueued, + model.AttachmentProcessing, + model.AttachmentFailed, + } { + t.Run(status, func(t *testing.T) { + original := attachment("a", "attachments/a/v1/original.png", status, now.Add(-48*time.Hour), false) + original.PipelineVersion = 2 + missing := attachment("a", "attachments/a/v1/optimized.webp", status, now.Add(-48*time.Hour), false) + missing.PipelineVersion = 2 + metadata := &metadataStub{items: map[string]model.Attachment{ + original.ObjectKey: original, + missing.ObjectKey: missing, + }} + objects := &objectStub{items: map[string]model.StoredObject{ + original.ObjectKey: {Key: original.ObjectKey, LastModified: now.Add(-48 * time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedAttachmentObjects) + require.Zero(t, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, original.ObjectKey) + require.Contains(t, metadata.items, missing.ObjectKey) + require.Contains(t, objects.items, original.ObjectKey) + }) + } +} + +func TestReconcilerDeletesAllAttachmentVariantsOnce(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + keys := []string{ + "attachments/a/v1/crop.jpg", + "attachments/a/v1/optimized.webp", + "attachments/a/v1/original.png", + "attachments/a/v1/thumbnail.jpg", + } + metadata := &metadataStub{items: make(map[string]model.Attachment, len(keys))} + objects := &objectStub{items: make(map[string]model.StoredObject, len(keys))} + for _, key := range keys { + metadata.items[key] = attachment("a", key, model.AttachmentReady, old, false) + objects.items[key] = model.StoredObject{Key: key, LastModified: old} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, len(keys), report.DeletedAttachmentObjects) + require.Equal(t, 1, report.DeletedUnreferencedRows) + require.Empty(t, metadata.items) + require.Empty(t, objects.items) + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestNewRejectsInvalidRetention(t *testing.T) { + _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) + require.Error(t, err) +} + +func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment { + readyAt := createdAt + return model.Attachment{ + ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt, + ReadyAt: &readyAt, Referenced: referenced, + } +} + +type metadataStub struct { + items map[string]model.Attachment + appeared map[string]bool +} + +func (s *metadataStub) AttachmentObjectKeys(_ context.Context, attachmentID string) ([]string, error) { + var keys []string + for key, item := range s.items { + if item.ID == attachmentID { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys, nil +} + +func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.Attachment, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) { + if s.appeared[key] { + return true, nil + } + _, exists := s.items[key] + return exists, nil +} + +func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { + claim := false + for key, item := range s.items { + if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { + continue + } + age := item.CreatedAt + if item.Status == model.AttachmentReady && item.ReadyAt != nil { + age = *item.ReadyAt + } + if age.After(cutoff) { + return false, nil + } + item.Status = model.AttachmentDeleting + s.items[key] = item + claim = true + } + return claim, nil +} + +func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + deleted := false + for key, item := range s.items { + if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { + delete(s.items, key) + deleted = true + } + } + return deleted, nil +} + +type objectStub struct { + items map[string]model.StoredObject +} + +func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.StoredObject, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *objectStub) Delete(_ context.Context, key string) error { + delete(s.items, key) + return nil +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 0000000..a7df4b9 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,163 @@ +package session + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/valkey-io/valkey-go" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const ( + tokenLength = 32 + keyPrefix = "session:" +) + +var ( + ErrNotFound = model.ErrSessionNotFound + ErrInvalidToken = model.ErrInvalidToken + ErrInvalidRecord = errors.New("invalid session record") + ErrInvalidTTL = errors.New("invalid session TTL") + ErrTokenGeneration = errors.New("session token generation failed") + ErrStore = model.ErrUnavailable +) + +type record struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Store struct { + client valkey.Client +} + +func New(rawURL string) (*Store, error) { + option, err := valkey.ParseURL(rawURL) + if err != nil { + return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err) + } + + client, err := valkey.NewClient(option) + if err != nil { + return nil, fmt.Errorf("%w: create Valkey client: %w", ErrStore, err) + } + + return &Store{client: client}, nil +} + +func (s *Store) Create(ctx context.Context, userID string, ttl time.Duration) (string, time.Time, error) { + if strings.TrimSpace(userID) == "" { + return "", time.Time{}, ErrInvalidRecord + } + if ttl < time.Second { + return "", time.Time{}, ErrInvalidTTL + } + + token, err := newToken() + if err != nil { + return "", time.Time{}, err + } + + record := record{ + UserID: userID, + ExpiresAt: time.Now().UTC().Add(ttl).Truncate(time.Microsecond), + } + payload, err := json.Marshal(record) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: encode session: %w", ErrStore, err) + } + + command := s.client.B().Set().Key(s.key(token)).Value(valkey.BinaryString(payload)).Ex(ttl).Build() + if err := s.client.Do(ctx, command).Error(); err != nil { + return "", time.Time{}, fmt.Errorf("%w: create session: %w", ErrStore, err) + } + + return token, record.ExpiresAt, nil +} + +func newToken() (string, error) { + random := make([]byte, tokenLength) + if _, err := rand.Read(random); err != nil { + return "", fmt.Errorf("%w: %v", ErrTokenGeneration, err) + } + return base64.RawURLEncoding.EncodeToString(random), nil +} + +func (s *Store) UserID(ctx context.Context, token string) (string, error) { + key, err := s.sessionKey(token) + if err != nil { + return "", err + } + + payload, err := s.client.Do(ctx, s.client.B().Get().Key(key).Build()).AsBytes() + if valkey.IsValkeyNil(err) { + return "", ErrNotFound + } + if err != nil { + return "", fmt.Errorf("%w: get session: %w", ErrStore, err) + } + + var record record + if err := json.Unmarshal(payload, &record); err != nil { + return "", fmt.Errorf("%w: decode session: %w", ErrStore, err) + } + if strings.TrimSpace(record.UserID) == "" || record.ExpiresAt.IsZero() { + return "", fmt.Errorf("%w: %w", ErrStore, ErrInvalidRecord) + } + if !record.ExpiresAt.After(time.Now()) { + return "", ErrNotFound + } + + return record.UserID, nil +} + +func (s *Store) Delete(ctx context.Context, token string) error { + key, err := s.sessionKey(token) + if err != nil { + return err + } + + deleted, err := s.client.Do(ctx, s.client.B().Del().Key(key).Build()).AsInt64() + if err != nil { + return fmt.Errorf("%w: delete session: %w", ErrStore, err) + } + if deleted == 0 { + return ErrNotFound + } + + return nil +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.client.Do(ctx, s.client.B().Ping().Build()).Error(); err != nil { + return fmt.Errorf("%w: ping Valkey: %w", ErrStore, err) + } + return nil +} + +func (s *Store) Close() error { + s.client.Close() + return nil +} + +func (s *Store) sessionKey(token string) (string, error) { + decoded, err := base64.RawURLEncoding.Strict().DecodeString(token) + if err != nil || len(decoded) != tokenLength { + return "", ErrInvalidToken + } + return s.key(token), nil +} + +func (s *Store) key(token string) string { + digest := sha256.Sum256([]byte(token)) + return keyPrefix + hex.EncodeToString(digest[:]) +} diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go new file mode 100644 index 0000000..a82e5eb --- /dev/null +++ b/internal/session/session_integration_test.go @@ -0,0 +1,72 @@ +package session_test + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" +) + +type SessionSuite struct { + suite.Suite + store *session.Store +} + +func TestSessionSuite(t *testing.T) { + suite.Run(t, new(SessionSuite)) +} + +func (s *SessionSuite) SetupSuite() { + rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) + if rawURL == "" { + path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE")) + if path == "" { + s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") + } + data, err := os.ReadFile(path) + s.Require().NoError(err) + rawURL = strings.TrimSpace(string(data)) + } + store, err := session.New(rawURL) + s.Require().NoError(err) + s.store = store +} + +func (s *SessionSuite) TearDownSuite() { + if s.store != nil { + s.Require().NoError(s.store.Close()) + } +} + +func (s *SessionSuite) TestSessionLifecycle() { + userID := uuid.NewString() + token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute) + s.Require().NoError(err) + s.NotEmpty(token) + s.True(expiresAt.After(time.Now())) + + actualUserID, err := s.store.UserID(context.Background(), token) + s.Require().NoError(err) + s.Equal(userID, actualUserID) + s.Require().NoError(s.store.Delete(context.Background(), token)) + + _, err = s.store.UserID(context.Background(), token) + s.ErrorIs(err, session.ErrNotFound) +} + +func (s *SessionSuite) TestSessionExpires() { + token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second) + s.Require().NoError(err) + + s.Eventually(func() bool { + _, err = s.store.UserID(context.Background(), token) + return errors.Is(err, session.ErrNotFound) + }, 3*time.Second, 50*time.Millisecond) +} diff --git a/internal/telemetry/database.go b/internal/telemetry/database.go new file mode 100644 index 0000000..6390cd3 --- /dev/null +++ b/internal/telemetry/database.go @@ -0,0 +1,39 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + connectionStateUsed = attribute.String("state", "used") + connectionStateIdle = attribute.String("state", "idle") + connectionStateConnecting = attribute.String("state", "connecting") +) + +func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error { + _, err := meter.Int64ObservableGauge("messenger_db_pool_connections", + metric.WithDescription("PostgreSQL pool connections by state."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + stats := pool.Stat() + observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed)) + observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle)) + observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting)) + return nil + }), + ) + if err != nil { + return err + } + _, err = meter.Int64ObservableCounter("messenger_db_pool_waited", + metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(pool.Stat().EmptyAcquireCount()) + return nil + }), + ) + return err +} diff --git a/internal/telemetry/logging.go b/internal/telemetry/logging.go new file mode 100644 index 0000000..880d349 --- /dev/null +++ b/internal/telemetry/logging.go @@ -0,0 +1,74 @@ +package telemetry + +import ( + "context" + "log/slog" + "strings" + + "go.opentelemetry.io/otel/trace" +) + +type correlationHandler struct { + stdout slog.Handler + otel slog.Handler +} + +func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler { + return correlationHandler{stdout: stdout, otel: otel} +} + +func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level) +} + +func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error { + span := trace.SpanContextFromContext(ctx) + if span.IsValid() { + record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String())) + } + record = sanitizedRecord(record) + if err := h.stdout.Handle(ctx, record); err != nil { + return err + } + return h.otel.Handle(ctx, record) +} + +func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + attributes = sanitizedAttributes(attributes) + return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)} +} + +func (h correlationHandler) WithGroup(name string) slog.Handler { + return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)} +} + +func sanitizedRecord(record slog.Record) slog.Record { + copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC) + record.Attrs(func(attribute slog.Attr) bool { + if !sensitiveKey(attribute.Key) { + copy.AddAttrs(attribute) + } + return true + }) + return copy +} + +func sanitizedAttributes(attributes []slog.Attr) []slog.Attr { + result := make([]slog.Attr, 0, len(attributes)) + for _, attribute := range attributes { + if !sensitiveKey(attribute.Key) { + result = append(result, attribute) + } + } + return result +} + +func sensitiveKey(key string) bool { + key = strings.ToLower(key) + for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} { + if strings.Contains(key, marker) { + return true + } + } + return false +} diff --git a/internal/telemetry/propagation.go b/internal/telemetry/propagation.go new file mode 100644 index 0000000..92c3dac --- /dev/null +++ b/internal/telemetry/propagation.go @@ -0,0 +1,70 @@ +package telemetry + +import ( + "context" + "encoding/json" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +type MessageCarrier struct { + Traceparent string + Tracestate string +} + +func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +func Inject(ctx context.Context, carrier propagation.TextMapCarrier) { + otel.GetTextMapPropagator().Inject(ctx, carrier) +} + +func ContextCarrier(ctx context.Context) MessageCarrier { + carrier := propagation.MapCarrier{} + Inject(ctx, carrier) + return MessageCarrier{ + Traceparent: carrier.Get("traceparent"), + Tracestate: carrier.Get("tracestate"), + } +} + +func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context { + headers := propagation.MapCarrier{ + "traceparent": carrier.Traceparent, + "tracestate": carrier.Tracestate, + } + return Extract(ctx, headers) +} + +func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context { + var carrier struct { + Traceparent string `json:"traceparent"` + Tracestate string `json:"tracestate"` + } + if json.Unmarshal(payload, &carrier) != nil { + return ctx + } + return ExtractMessage(ctx, MessageCarrier(carrier)) +} + +func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) { + var value map[string]json.RawMessage + if err := json.Unmarshal(payload, &value); err != nil { + return nil, err + } + carrier := ContextCarrier(ctx) + setJSONField(value, "traceparent", carrier.Traceparent) + setJSONField(value, "tracestate", carrier.Tracestate) + return json.Marshal(value) +} + +func setJSONField(value map[string]json.RawMessage, key, field string) { + if field == "" { + delete(value, key) + return + } + encoded, _ := json.Marshal(field) + value[key] = encoded +} diff --git a/internal/telemetry/propagation_test.go b/internal/telemetry/propagation_test.go new file mode 100644 index 0000000..31895a5 --- /dev/null +++ b/internal/telemetry/propagation_test.go @@ -0,0 +1,33 @@ +package telemetry + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + + payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`)) + require.NoError(t, err) + var value map[string]any + require.NoError(t, json.Unmarshal(payload, &value)) + require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"]) + require.NotContains(t, value, "baggage") + + extracted := ExtractJSONMessage(context.Background(), payload) + require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID()) +} diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go new file mode 100644 index 0000000..90267e6 --- /dev/null +++ b/internal/telemetry/runtime.go @@ -0,0 +1,252 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/bridges/otelslog" + otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" +) + +const ( + defaultEnvironment = "lab" + defaultNamespace = "mtusi-highload" +) + +type Config struct { + ServiceName string + Version string + InstanceID string + Endpoint string + Environment string + Namespace string + Insecure bool +} + +type Runtime struct { + traces *trace.TracerProvider + metrics *metric.MeterProvider + logs *log.LoggerProvider + logger *slog.Logger + handler http.Handler +} + +func ConfigFromEnvironment(serviceName, instanceID string) Config { + resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES")) + return Config{ + ServiceName: serviceName, + Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), + InstanceID: instanceID, + Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), + Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), + Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), + Insecure: true, + } +} + +func parseResourceAttributes(raw string) map[string]string { + result := make(map[string]string) + for _, pair := range strings.Split(raw, ",") { + key, value, found := strings.Cut(pair, "=") + if found && strings.TrimSpace(key) != "" { + result[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + } + return result +} + +func firstValue(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func New(ctx context.Context, config Config) (*Runtime, error) { + if err := config.validate(); err != nil { + return nil, err + } + res, err := newResource(ctx, config) + if err != nil { + return nil, err + } + registry := prometheus.NewRegistry() + prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry)) + if err != nil { + return nil, fmt.Errorf("create Prometheus reader: %w", err) + } + + traceOptions := []trace.TracerProviderOption{trace.WithResource(res)} + metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)} + logOptions := []log.LoggerProviderOption{log.WithResource(res)} + if config.Endpoint != "" { + traceExporter, err := newTraceExporter(ctx, config) + if err != nil { + return nil, err + } + metricExporter, err := newMetricExporter(ctx, config) + if err != nil { + return nil, err + } + logExporter, err := newLogExporter(ctx, config) + if err != nil { + return nil, err + } + traceOptions = append(traceOptions, trace.WithBatcher(traceExporter, + trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512), + trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second), + )) + metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader( + metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second), + ))) + logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter, + log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512), + log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second), + ))) + } + + traces := trace.NewTracerProvider(traceOptions...) + metrics := metric.NewMeterProvider(metricOptions...) + logs := log.NewLoggerProvider(logOptions...) + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(traces) + otel.SetMeterProvider(metrics) + otel.SetTextMapPropagator(propagator) + if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil { + return nil, fmt.Errorf("start Go runtime metrics: %w", err) + } + + stdout := slog.NewJSONHandler(os.Stdout, nil) + otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs)) + logger := slog.New(newCorrelationHandler(stdout, otelHandler)) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + return &Runtime{ + traces: traces, metrics: metrics, logs: logs, logger: logger, + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + }, nil +} + +func (c Config) validate() error { + switch { + case strings.TrimSpace(c.ServiceName) == "": + return errors.New("telemetry service name is required") + case strings.TrimSpace(c.InstanceID) == "": + return errors.New("telemetry instance ID is required") + case strings.TrimSpace(c.Environment) == "": + return errors.New("telemetry environment is required") + case strings.TrimSpace(c.Namespace) == "": + return errors.New("telemetry service namespace is required") + default: + return nil + } +} + +func newResource(ctx context.Context, config Config) (*resource.Resource, error) { + res, err := resource.New(ctx, + resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(), + resource.WithAttributes( + semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version), + semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace), + semconv.DeploymentEnvironmentName(config.Environment), + ), + ) + if err != nil { + return nil, fmt.Errorf("create telemetry resource: %w", err) + } + return res, nil +} + +func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) { + var options []otlptracegrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlptracegrpc.WithInsecure()) + } + exporter, err := otlptracegrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + return exporter, nil +} + +func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) { + var options []otlpmetricgrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlpmetricgrpc.WithInsecure()) + } + exporter, err := otlpmetricgrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP metric exporter: %w", err) + } + return exporter, nil +} + +func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) { + var options []otlploggrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlploggrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlploggrpc.WithInsecure()) + } + exporter, err := otlploggrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP log exporter: %w", err) + } + return exporter, nil +} + +func (r *Runtime) Logger() *slog.Logger { + return r.logger +} + +func (r *Runtime) MeterProvider() *metric.MeterProvider { + return r.metrics +} + +func (r *Runtime) MetricsHandler() http.Handler { + return r.handler +} + +func (r *Runtime) Shutdown(ctx context.Context) error { + var result error + for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} { + if err := shutdown(ctx); err != nil { + result = errors.Join(result, err) + } + } + return result +} diff --git a/internal/telemetry/runtime_test.go b/internal/telemetry/runtime_test.go new file mode 100644 index 0000000..e8b41b1 --- /dev/null +++ b/internal/telemetry/runtime_test.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func TestConfigValidation(t *testing.T) { + valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"} + require.NoError(t, valid.validate()) + + invalid := valid + invalid.ServiceName = "" + require.EqualError(t, invalid.validate(), "telemetry service name is required") +} + +func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) { + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + first := &recordHandler{} + second := &recordHandler{} + handler := newCorrelationHandler(first, second) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0) + record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret")) + + require.NoError(t, handler.Handle(ctx, record)) + for _, captured := range []*recordHandler{first, second} { + require.Equal(t, traceID.String(), captured.attributes["trace_id"]) + require.Equal(t, spanID.String(), captured.attributes["span_id"]) + require.Equal(t, "http_request_completed", captured.attributes["event"]) + require.NotContains(t, captured.attributes, "authorization") + } +} + +type recordHandler struct { + attributes map[string]string +} + +func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recordHandler) Handle(_ context.Context, record slog.Record) error { + h.attributes = make(map[string]string) + record.Attrs(func(attribute slog.Attr) bool { + h.attributes[attribute.Key] = attribute.Value.String() + return true + }) + return nil +} +func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordHandler) WithGroup(string) slog.Handler { return h } diff --git a/lab6.env.example b/lab6.env.example new file mode 100644 index 0000000..ca34a40 --- /dev/null +++ b/lab6.env.example @@ -0,0 +1,93 @@ +# Изолированное имя Compose-проекта и его volumes/networks. +COMPOSE_PROJECT_NAME=mtusi2026lab6 + +# Адрес и внутренний порт HTTP-сервера. +APP_HOST=0.0.0.0 +APP_PORT=8080 +MESSENGER_CONFIG_PATH=config/messenger.yaml + +# Адреса публикации сервисов на локальной машине. +API_BIND_ADDRESS=127.0.0.1:8080 +SWAGGER_BIND_ADDRESS=127.0.0.1:8081 +PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 +GRAFANA_BIND_ADDRESS=127.0.0.1:3000 +LOKI_BIND_ADDRESS=127.0.0.1:3100 +TEMPO_BIND_ADDRESS=127.0.0.1:3200 + +# Ресурсные атрибуты и внутренний OTLP endpoint приложений. +DEPLOYMENT_ENVIRONMENT=lab +SERVICE_VERSION=lab6 +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +# Число API-реплик за Nginx. +API_REPLICAS=2 + +# Учётные данные локального PostgreSQL и строка подключения API. +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD=local-postgres-password +DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable + +# Пароль локального Valkey и строка подключения API. +VALKEY_PASSWORD=local-valkey-password +CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 + +# Redis Streams используется только как устойчивый брокер заданий. +REDIS_BROKER_PASSWORD=local-redis-broker-password +BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 + +# Административные данные MinIO используются только при bootstrap. +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=local-minio-admin-password + +# API использует отдельного пользователя с доступом только к bucket вложений. +S3_ENDPOINT=s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_ACCESS_KEY=messenger-app +S3_SECRET_KEY=local-messenger-s3-password + +# Порт Swagger внутри конфигурации CORS API. +SWAGGER_PORT=8081 + +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=512m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=1 +GRAFANA_MEMORY=512m +OTEL_COLLECTOR_CPUS=0.5 +OTEL_COLLECTOR_MEMORY=256m +LOKI_CPUS=1 +LOKI_MEMORY=512m +TEMPO_CPUS=1 +TEMPO_MEMORY=512m +POSTGRES_CPUS=2 +POSTGRES_MEMORY=1g +VALKEY_CPUS=1 +VALKEY_MEMORY=256m +REDIS_BROKER_CPUS=1 +REDIS_BROKER_MEMORY=256m +DISPATCHER_CPUS=0.5 +DISPATCHER_MEMORY=128m +WORKER_CPUS=2 +WORKER_MEMORY=1g +S3_CPUS=1 +S3_MEMORY=512m +S3_BOOTSTRAP_CPUS=0.25 +S3_BOOTSTRAP_MEMORY=256m + +# Пароль локального администратора Grafana. +GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/migrations/0001_initial.sql b/migrations/0001_initial.sql new file mode 100644 index 0000000..2f9b90b --- /dev/null +++ b/migrations/0001_initial.sql @@ -0,0 +1,50 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_display_name_not_blank CHECK (display_name ~ '[^[:space:]]'), + CONSTRAINT users_password_hash_length CHECK (char_length(password_hash) BETWEEN 1 AND 1024), + CONSTRAINT users_password_hash_not_blank CHECK (password_hash ~ '[^[:space:]]') +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100), + CONSTRAINT chats_title_not_blank CHECK (title ~ '[^[:space:]]') +); + +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_position_unique UNIQUE (chat_id, position), + CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE +); + +CREATE INDEX chat_members_user_id_chat_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_text_not_blank CHECK (text ~ '[^[:space:]]'), + CONSTRAINT messages_sender_membership_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); + +CREATE INDEX messages_chat_id_created_at_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0002_message_idempotency.sql b/migrations/0002_message_idempotency.sql new file mode 100644 index 0000000..91443a4 --- /dev/null +++ b/migrations/0002_message_idempotency.sql @@ -0,0 +1,19 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL, + chat_id UUID NOT NULL, + key TEXT NOT NULL, + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32), + CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at) +); + +CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..91d1cc9 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,47 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_length, + DROP CONSTRAINT messages_text_not_blank; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id); diff --git a/migrations/0004_attachment_lifecycle.sql b/migrations/0004_attachment_lifecycle.sql new file mode 100644 index 0000000..1596c99 --- /dev/null +++ b/migrations/0004_attachment_lifecycle.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachments + ADD COLUMN status TEXT NOT NULL DEFAULT 'ready', + ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp(); + +UPDATE attachments +SET ready_at = created_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status IN ('ready', 'deleting') AND ready_at IS NOT NULL) + ); + +CREATE INDEX attachments_status_created_at_object_key_idx + ON attachments (status, created_at, object_key); diff --git a/migrations/0005_allow_pending_deletion.sql b/migrations/0005_allow_pending_deletion.sql new file mode 100644 index 0000000..0b20acb --- /dev/null +++ b/migrations/0005_allow_pending_deletion.sql @@ -0,0 +1,9 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status = 'ready' AND ready_at IS NOT NULL) OR + status = 'deleting' + ); diff --git a/migrations/0006_async_attachment_pipeline.sql b/migrations/0006_async_attachment_pipeline.sql new file mode 100644 index 0000000..2b061b6 --- /dev/null +++ b/migrations/0006_async_attachment_pipeline.sql @@ -0,0 +1,135 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status, + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code TEXT, + ADD COLUMN staging_key TEXT; + +UPDATE attachments +SET staging_key = object_key +WHERE status = 'pending'; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK ( + status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting') + ), + ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1), + ADD CONSTRAINT attachments_error_code CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR + (status <> 'failed' AND error_code IS NULL) + ), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'ready' AND ready_at IS NOT NULL) OR + (status <> 'ready') + ), + ADD CONSTRAINT attachments_staging_key_length CHECK ( + staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024 + ); + +CREATE INDEX attachments_status_updated_at_idx + ON attachments (status, updated_at, id); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued', + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + lease_token UUID, + lease_owner TEXT, + lease_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_lease CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR + (status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL) + ), + CONSTRAINT attachment_jobs_error_code CHECK ( + error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (available_at, id) + WHERE status = 'queued'; + +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until, id) + WHERE status = 'processing'; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key), + CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE TABLE outbox_events ( + id UUID PRIMARY KEY, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload JSONB NOT NULL, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + publish_tries INTEGER NOT NULL DEFAULT 0, + claim_token UUID, + claim_until TIMESTAMPTZ, + last_error_code TEXT, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt), + CONSTRAINT outbox_events_attempt CHECK (attempt >= 1), + CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0), + CONSTRAINT outbox_events_claim CHECK ( + (claim_token IS NULL AND claim_until IS NULL) OR + (claim_token IS NOT NULL AND claim_until IS NOT NULL) + ), + CONSTRAINT outbox_events_last_error_code CHECK ( + last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX outbox_events_due_idx + ON outbox_events (available_at, id) + WHERE published_at IS NULL; + +CREATE TABLE attachment_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + previous_attempt INTEGER NOT NULL, + new_attempt INTEGER NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1), + CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128), + CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512) +); + +CREATE INDEX attachment_replay_audit_job_created_at_idx + ON attachment_replay_audit (job_id, created_at); diff --git a/migrations/0007_outbox_retention.sql b/migrations/0007_outbox_retention.sql new file mode 100644 index 0000000..cc936f6 --- /dev/null +++ b/migrations/0007_outbox_retention.sql @@ -0,0 +1,3 @@ +CREATE INDEX outbox_events_published_at_idx + ON outbox_events (published_at, id) + WHERE published_at IS NOT NULL; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 0000000..08bbf60 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import "embed" + +// FS contains database migrations applied by the application at startup. +// +//go:embed *.sql +var FS embed.FS diff --git a/monitoring/grafana/dashboards/messenger-lab6.json b/monitoring/grafana/dashboards/messenger-lab6.json new file mode 100644 index 0000000..f20cabe --- /dev/null +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -0,0 +1,1512 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "type": "row", + "title": "API and edge", + "id": 100, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [ + { + "type": "timeseries", + "title": "API requests by route", + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API responses by status class", + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API latency p50 / p95 / p99 by route", + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50 {{method}} {{route}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{method}} {{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99 {{method}} {{route}}", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API replicas and requests in flight", + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "expr": "sum(up{job=\"messenger\"})", + "legendFormat": "healthy replicas", + "refId": "A" + }, + { + "expr": "sum(messenger_http_requests_in_flight)", + "legendFormat": "in flight", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API CPU usage", + "id": 5, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{job=\"messenger\"}[1m]))", + "legendFormat": "API", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Nginx requests and connections", + "id": 6, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(nginx_http_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "A" + }, + { + "expr": "sum(nginx_connections_active)", + "legendFormat": "active connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "PostgreSQL", + "id": 101, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [ + { + "type": "timeseries", + "title": "PostgreSQL availability and connections", + "id": 7, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "expr": "up{job=\"postgres\"}", + "legendFormat": "up", + "refId": "A" + }, + { + "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", + "legendFormat": "connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL transactions", + "id": 8, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "targets": [ + { + "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m]))", + "legendFormat": "commit/s", + "refId": "A" + }, + { + "expr": "sum(rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m]))", + "legendFormat": "rollback/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL database size", + "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname=\"messenger\"}", + "legendFormat": "messenger", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment jobs by status", + "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_count", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Oldest attachment job", + "id": 11, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_oldest_age_seconds", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Pending outbox", + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_outbox_pending", + "legendFormat": "events", + "refId": "A" + }, + { + "expr": "messenger_outbox_oldest_pending_age_seconds", + "legendFormat": "oldest age, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } + ] + }, + { + "type": "row", + "title": "Valkey and Redis", + "id": 102, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [ + { + "type": "timeseries", + "title": "Valkey and Redis availability", + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "targets": [ + { + "expr": "up{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "up{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis memory", + "id": 14, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey cache hits and misses", + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_keyspace_hits_total{job=\"valkey\"}[1m]))", + "legendFormat": "hits/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_keyspace_misses_total{job=\"valkey\"}[1m]))", + "legendFormat": "misses/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis commands", + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_commands_processed_total{job=\"valkey\"}[1m]))", + "legendFormat": "Valkey commands/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_commands_processed_total{job=\"redis-broker\"}[1m]))", + "legendFormat": "Redis commands/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "Object storage", + "id": 103, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "panels": [ + { + "type": "timeseries", + "title": "MinIO availability", + "id": 17, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 68 + }, + "targets": [ + { + "expr": "up{job=\"minio\"}", + "legendFormat": "up", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO objects and requests", + "id": 18, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_object_total)", + "legendFormat": "objects", + "refId": "A" + }, + { + "expr": "sum(rate(minio_s3_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "B" + }, + { + "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", + "legendFormat": "errors/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO storage", + "id": 19, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_total_bytes)", + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(minio_cluster_capacity_usable_free_bytes)", + "legendFormat": "free", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "Attachment pipeline", + "id": 104, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 76 + }, + "panels": [ + { + "type": "timeseries", + "title": "Pipeline availability and consumers", + "id": 20, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "targets": [ + { + "expr": "sum(up{job=\"attachment-worker\"})", + "legendFormat": "workers up", + "refId": "A" + }, + { + "expr": "sum(up{job=\"outbox-dispatcher\"})", + "legendFormat": "dispatcher up", + "refId": "B" + }, + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "active consumers", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment upload results", + "id": 21, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", + "legendFormat": "bytes/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + } + }, + { + "type": "timeseries", + "title": "Worker throughput by result", + "id": 22, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker processing latency p95", + "id": 23, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", + "legendFormat": "upload p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker jobs in flight", + "id": 24, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 93 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Retries and reclaimed deliveries", + "id": 25, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", + "legendFormat": "retries/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", + "legendFormat": "reclaimed/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Outbox dispatcher", + "id": 26, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_outbox_published_total[1m]))", + "legendFormat": "published/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", + "legendFormat": "failures/s", + "refId": "B" + }, + { + "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", + "legendFormat": "recovered/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream lag, PEL and DLQ", + "id": 27, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_lag{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "lag", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_messages_pending{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "PEL", + "refId": "B" + }, + { + "expr": "max(redis_stream_length{job=\"redis-broker\",stream=\"attachment-jobs-failed\"})", + "legendFormat": "DLQ", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream consumers", + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "consumers", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_consumer_idle_seconds{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "max idle, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } + ] + }, + { + "type": "row", + "title": "SLO and alerts", + "id": 105, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "panels": [ + { + "type": "timeseries", + "title": "Business API availability", + "id": 31, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 110 + }, + "targets": [ + { + "expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "availability", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "API error-budget burn rate (99.5% SLO)", + "id": 32, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 110 + }, + "targets": [ + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachments ready within 30 seconds", + "id": 33, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 118 + }, + "targets": [ + { + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "ready in 30s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachment ready-time error-budget burn", + "id": 34, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "targets": [ + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "SLO alert state", + "id": 35, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 126 + }, + "targets": [ + { + "expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})", + "legendFormat": "{{alertname}} {{alertstate}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Runtime resources", + "id": 107, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 134 + }, + "panels": [ + { + "type": "timeseries", + "title": "Process CPU by service", + "id": 38, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Go heap by service", + "id": 39, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Database pool used and wait rate", + "id": 40, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 143 + }, + "targets": [ + { + "expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{service_name}} used", + "refId": "A" + }, + { + "expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))", + "legendFormat": "{{service_name}} waits/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Worker saturation", + "id": 41, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 143 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "sum(messenger_attachment_worker_capacity)", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Logs and traces", + "id": 106, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 151 + }, + "panels": [ + { + "type": "logs", + "title": "Correlated application logs", + "id": 36, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 152 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"messenger-.+\"} | json", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + } + }, + { + "type": "traces", + "title": "Recent application traces (TraceQL)", + "id": 37, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 152 + }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "query": "{ resource.service.name =~ \"messenger-.*\" }", + "queryType": "traceql", + "refId": "A", + "limit": 50, + "tableType": "traces" + } + ], + "options": { + "showSpanFilters": true + } + }, + { + "type": "timeseries", + "title": "Trace-derived service call rate", + "id": 42, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))", + "legendFormat": "{{service}} {{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "type": "timeseries", + "title": "Collector delivery failures and queue pressure", + "id": 43, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))", + "legendFormat": "{{exporter}} spans failed/s", + "refId": "A" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))", + "legendFormat": "{{exporter}} logs failed/s", + "refId": "B" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}} queue", + "refId": "C" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_capacity)", + "legendFormat": "{{exporter}} capacity", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + } + ], + "refresh": "5s", + "schemaVersion": 41, + "tags": [ + "highload", + "lab6", + "observability", + "slo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "Messenger lab6 observability", + "uid": "messenger-lab6", + "version": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Explore logs and traces", + "type": "link", + "url": "/explore" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "API runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Pipeline runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md" + } + ] +} diff --git a/monitoring/grafana/provisioning/alerting/.gitkeep b/monitoring/grafana/provisioning/alerting/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/grafana/provisioning/dashboards/messenger.yml b/monitoring/grafana/provisioning/dashboards/messenger.yml new file mode 100644 index 0000000..fa7934c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/messenger.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Messenger + folder: Highload + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml new file mode 100644 index 0000000..99af68b --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -0,0 +1,74 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: TraceID + matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + nodeGraph: + enabled: true + search: + hide: false + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"' + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service + queries: + - name: Request rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))' + - name: Error rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))' diff --git a/monitoring/grafana/provisioning/plugins/.gitkeep b/monitoring/grafana/provisioning/plugins/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/plugins/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml new file mode 100644 index 0000000..7666281 --- /dev/null +++ b/monitoring/postgres-queries.yml @@ -0,0 +1,39 @@ +messenger_attachment_jobs: + query: | + WITH statuses(status) AS ( + VALUES ('queued'), ('processing'), ('failed') + ) + SELECT + statuses.status, + COUNT(j.id)::bigint AS count, + COALESCE(EXTRACT(EPOCH FROM (clock_timestamp() - MIN(j.created_at))), 0)::double precision AS oldest_age_seconds + FROM statuses + LEFT JOIN attachment_jobs j ON j.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: Attachment job status. + - count: + usage: GAUGE + description: Attachment jobs in this status. + - oldest_age_seconds: + usage: GAUGE + description: Age of the oldest attachment job in this status. + +messenger_outbox: + query: | + SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::bigint AS pending, + COALESCE( + EXTRACT(EPOCH FROM (clock_timestamp() - MIN(created_at) FILTER (WHERE published_at IS NULL))), + 0 + )::double precision AS oldest_pending_age_seconds + FROM outbox_events + metrics: + - pending: + usage: GAUGE + description: Unpublished outbox events. + - oldest_pending_age_seconds: + usage: GAUGE + description: Age of the oldest unpublished outbox event. diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 0000000..70b5dfd --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,85 @@ +global: + scrape_interval: 2s + evaluation_interval: 5s + external_labels: + deployment_environment: lab + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: messenger + dns_sd_configs: + - names: ["api"] + type: A + port: 8080 + refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-api + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: nginx + static_configs: + - targets: ["nginx-exporter:9113"] + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: valkey + static_configs: + - targets: ["valkey-exporter:9121"] + + - job_name: redis-broker + static_configs: + - targets: ["redis-broker-exporter:9121"] + + - job_name: attachment-worker + dns_sd_configs: + - names: ["worker"] + type: A + port: 9091 + refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-worker + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: outbox-dispatcher + static_configs: + - targets: ["dispatcher:9091"] + relabel_configs: + - target_label: service_name + replacement: messenger-dispatcher + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: ["s3:9000"] + + - job_name: otel-applications + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml new file mode 100644 index 0000000..4999ddb --- /dev/null +++ b/monitoring/rules/messenger.yml @@ -0,0 +1,178 @@ +groups: + - name: messenger-api-slo + interval: 5s + rules: + - alert: MessengerAPIBusiness5xxHigh + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0)) + / + clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001) + ) > 0.05 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: Business API 5xx ratio is above 5% + description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnFast + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001) + ) > 0.072 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001) + ) > 0.072 + for: 30s + labels: + severity: critical + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning fast + description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnSlow + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001) + ) > 0.03 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001) + ) > 0.03 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning steadily + description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - name: messenger-attachment-slo + interval: 5s + rules: + - alert: AttachmentReadyHighErrorBudgetBurnFast + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)), + 0.000001 + ) > 0.144 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)), + 0.000001 + ) > 0.144 + for: 30s + labels: + severity: critical + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning fast + description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentReadyHighErrorBudgetBurnSlow + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)), + 0.000001 + ) > 0.06 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)), + 0.000001 + ) > 0.06 + for: 1m + labels: + severity: warning + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning steadily + description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentWorkerUnavailable + expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: No attachment worker is available + description: Prometheus has not observed a healthy worker target for 30 seconds. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentQueueBacklogGrowing + expr: | + max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30 + and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: Attachment queue backlog is growing + description: The oldest queued attachment is older than 30 seconds and its age continues to increase. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentRedisStreamLagHigh + expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20 + for: 1m + labels: + severity: warning + service: redis-broker + annotations: + summary: Redis Stream consumer lag is high + description: The attachment worker group has more than 20 messages waiting for delivery. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md diff --git a/monitoring/runbooks/api-availability.md b/monitoring/runbooks/api-availability.md new file mode 100644 index 0000000..2ff5453 --- /dev/null +++ b/monitoring/runbooks/api-availability.md @@ -0,0 +1,72 @@ +# API availability SLO + +## Цель и сигналы + +Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%. + +Основные сигналы: + +- `messenger_http_requests_total{method,route,status_class}` — число запросов; +- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency; +- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets; +- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения; +- trace `messenger-api` — путь одного запроса по зависимостям; +- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`. + +`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента. +2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью. +3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler. +4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL: + + ```text + { resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error } + ``` + +5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют. +6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki: + + ```text + {service_name="messenger-api"} | json | trace_id="" + ``` + +7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют. + +## Проверка alert + +Состояние правил видно в Grafana Alerting и в Prometheus: + +```bash +curl -s http://127.0.0.1:9090/api/v1/alerts +``` + +Контролируемый отказ PostgreSQL: + +```bash +ENV_FILE=lab6.env.example scripts/incident-database.sh +``` + +Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются. + +Ожидаемая последовательность: + +```text +нормальный error ratio +→ PostgreSQL остановлен +→ 5xx растёт +→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing +→ PostgreSQL восстановлен +→ успешные ответы возвращаются +→ оперативный alert resolved; burn-rate хранит историю длинного окна +``` + +## Действия + +1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи. +2. Если API targets недоступны, восстановить реплики по одной и проверить readiness. +3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины. +4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline. +5. Не удалять volumes и не сбрасывать очередь ради закрытия alert. diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md new file mode 100644 index 0000000..c2ae007 --- /dev/null +++ b/monitoring/runbooks/attachment-pipeline.md @@ -0,0 +1,85 @@ +# Attachment pipeline SLO + +## Цель и сигналы + +Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget. + +Основные сигналы: + +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker; +- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; +- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; +- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL; +- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job; +- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker; +- `messenger_outbox_pending` — задержка до публикации; +- Redis Stream lag и PEL — доставка и неподтверждённые сообщения. + +Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента. +2. Определить место задержки: + - растёт `messenger_outbox_pending` — проблема до публикации в broker; + - растут stream lag и `queued` — worker не успевает или недоступен; + - растут PEL и `processing` — delivery получена, но не завершена; + - растёт `failed` — постоянные ошибки или исчерпаны retry; + - busy равен capacity — worker насыщен. +3. В Explore → Tempo найти upload по route или worker consumer span: + + ```text + { resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" } + ``` + + ```text + { resource.service.name = "messenger-worker" && name = "attachment.process" } + ``` + +4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label. +5. Перейти из выбранного span в Loki или выполнить: + + ```text + {service_name="messenger-worker"} | json | trace_id="" + ``` + +6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются. + +## Проверка alert + +Короткая деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh short +``` + +Длительная деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh long +``` + +Ожидаемая последовательность: + +```text +worker up, backlog около нуля +→ worker остановлен +→ queued и oldest age растут +→ short: AttachmentWorkerUnavailable firing +→ long: AttachmentQueueBacklogGrowing firing +→ worker запущен +→ backlog дренируется +→ все принятые attachments достигают ready +→ alert resolved +``` + +Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload. + +## Действия + +1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования. +2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency. +3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина. +4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim. +5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины. +6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment. diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..d62dcd6 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,57 @@ +user nginx; +worker_processes auto; +error_log /dev/stderr notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + + log_format structured escape=json + '{"time":"$time_iso8601","request_id":"$http_x_request_id",' + '"method":"$request_method","uri":"$request_uri","status":$status,' + '"request_time":$request_time,"upstream":"$upstream_addr",' + '"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}'; + + access_log /dev/stdout structured; + + upstream messenger { + zone messenger 64k; + include /etc/nginx/conf.d/upstream.conf; + keepalive 32; + } + + server { + listen 8080; + client_max_body_size 11m; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering off; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } +} diff --git a/nginx/upstream.conf b/nginx/upstream.conf new file mode 100644 index 0000000..f6131cb --- /dev/null +++ b/nginx/upstream.conf @@ -0,0 +1 @@ +server api:8080 resolve max_fails=1 fail_timeout=3s; diff --git a/scripts/incident-database.sh b/scripts/incident-database.sh new file mode 100755 index 0000000..bc3c6e0 --- /dev/null +++ b/scripts/incident-database.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${DB_OUTAGE_SECONDS:-75} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh} +burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/database-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d postgres migrate api nginx >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode messages \ + --duration "$((outage_seconds + 20))" \ + --concurrency 4 \ + --rate "${DB_LOAD_RATE:-20}" \ + --started-file "$output_dir/load-started" \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +remaining=60 +while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do + sleep 0.5 + remaining=$((remaining - 1)) +done +test -f "$output_dir/load-started" +sleep 2 +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop postgres >/dev/null +sleep "$outage_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null +fi +python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null +wait "$load_pid" || true +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "postgres_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-telemetry.sh b/scripts/incident-telemetry.sh new file mode 100755 index 0000000..1f0cc50 --- /dev/null +++ b/scripts/incident-telemetry.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/telemetry-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d tempo loki >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop tempo loki >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "$outage_seconds" \ + --concurrency 4 \ + --rate "${TELEMETRY_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$outage_seconds" +wait "$load_pid" +load_pid= +health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready") +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 tempo loki >/dev/null +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" "$health_code" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration, health_code = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "telemetry_backends_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "injected_duration_seconds": int(duration), + "api_ready_http_status_during_outage": int(health_code), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-worker.sh b/scripts/incident-worker.sh new file mode 100755 index 0000000..016df05 --- /dev/null +++ b/scripts/incident-worker.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +set -eu + +mode=${1:-short} +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +case "$mode" in + short) + incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75} + load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60} + requests=${SHORT_WORKER_REQUESTS:-300} + default_alert=AttachmentWorkerUnavailable + ;; + long) + incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180} + load_seconds=${LONG_WORKER_LOAD_SECONDS:-120} + requests=${LONG_WORKER_REQUESTS:-1000} + default_alert=AttachmentQueueBacklogGrowing + ;; + *) + echo "usage: $0 [short|long]" >&2 + exit 2 + ;; +esac +worker_alert_name=${WORKER_ALERT_NAME:-$default_alert} + +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/worker-$mode-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d worker >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop worker >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode attachments \ + --duration "$load_seconds" \ + --requests "$requests" \ + --concurrency 4 \ + --rate "${WORKER_LOAD_RATE:-5}" \ + --wait-ready \ + --ready-timeout 480 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$incident_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null +fi +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 worker >/dev/null +wait "$load_pid" +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null + if [ "$mode" = "long" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null + fi +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, mode, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "worker_unavailable", + "profile": mode, + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"], +} +if mode == "long": + report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"]) +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/lab6-alert.py b/scripts/lab6-alert.py new file mode 100755 index 0000000..1e75ce9 --- /dev/null +++ b/scripts/lab6-alert.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.request + + +def fetch(prometheus_url): + url = prometheus_url.rstrip("/") + "/api/v1/alerts" + with urllib.request.urlopen(url, timeout=10) as response: + payload = json.loads(response.read(4 << 20).decode("utf-8")) + return payload.get("data", {}).get("alerts", []) + + +def matching(alerts, name): + return [alert for alert in alerts if alert.get("labels", {}).get("alertname") == name] + + +def main(): + parser = argparse.ArgumentParser(description="Wait for a Prometheus alert transition") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--name", required=True) + parser.add_argument("--state", choices=("firing", "resolved"), required=True) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + deadline = time.monotonic() + args.timeout + last = [] + error = None + observed = False + while time.monotonic() < deadline: + try: + last = matching(fetch(args.prometheus_url), args.name) + error = None + if args.state == "firing" and any(item.get("state") == "firing" for item in last): + observed = True + break + if args.state == "resolved" and not any(item.get("state") in ("pending", "firing") for item in last): + observed = True + break + except (urllib.error.URLError, json.JSONDecodeError) as current_error: + error = str(current_error) + time.sleep(2) + + report = { + "schema_version": 1, + "alert": args.name, + "expected_state": args.state, + "observed": observed, + "observed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "last_states": sorted({item.get("state", "") for item in last}), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-correlation.py b/scripts/lab6-correlation.py new file mode 100755 index 0000000..9b981a7 --- /dev/null +++ b/scripts/lab6-correlation.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import re +import time +import urllib.error +import urllib.parse +import urllib.request + +TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b') +SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher") + + +def get_json(url): + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(8 << 20).decode("utf-8")) + + +def tempo_query(base_url, query): + url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100}) + payload = get_json(url) + return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")} + + +def extract_trace_ids(payload): + result = set() + for stream in payload.get("data", {}).get("result", []): + for value in stream.get("values", []): + candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""] + if len(value) > 2: + candidates.append(value[2]) + for candidate in candidates: + if isinstance(candidate, dict): + trace_id = candidate.get("trace_id") or candidate.get("traceid") + if trace_id: + result.add(str(trace_id)) + candidate = json.dumps(candidate) + result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate))) + return result + + +def loki_logs(base_url, service, start_ns): + query = '{service_name="' + service + '"}' + url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({ + "query": query, + "start": str(start_ns), + "limit": 5000, + "direction": "backward", + }) + payload = get_json(url) + count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", [])) + return count, extract_trace_ids(payload) + + +def capture(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES} + logs = {} + log_trace_ids = {} + for service in SERVICES: + count, ids = loki_logs(loki_url, service, start_ns) + logs[service] = count + log_trace_ids[service] = ids + api_worker = traces["messenger-api"] & traces["messenger-worker"] + all_trace_ids = set().union(*traces.values()) + all_log_trace_ids = set().union(*log_trace_ids.values()) + return { + "traces_by_service": {service: len(ids) for service, ids in traces.items()}, + "logs_by_service": logs, + "api_worker_shared_traces": len(api_worker), + "traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids), + } + + +def capture_database(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = tempo_query( + tempo_url, + '{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }', + ) + log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns) + return { + "database_error_traces": len(traces), + "api_logs": log_count, + "database_error_traces_with_logs": len(traces & log_trace_ids), + } + + +def complete(report): + return ( + report["traces_by_service"]["messenger-api"] > 0 + and report["traces_by_service"]["messenger-worker"] > 0 + and report["logs_by_service"]["messenger-api"] > 0 + and report["logs_by_service"]["messenger-worker"] > 0 + and report["api_worker_shared_traces"] > 0 + and report["traces_with_correlated_logs"] > 0 + ) + + +def database_complete(report): + return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0 + + +def main(): + parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki") + parser.add_argument("--tempo-url", default="http://127.0.0.1:3200") + parser.add_argument("--loki-url", default="http://127.0.0.1:3100") + parser.add_argument("--lookback", type=float, default=900) + parser.add_argument("--wait", type=float, default=45) + parser.add_argument("--output", required=True) + parser.add_argument("--require", action="store_true") + parser.add_argument("--mode", choices=("baseline", "database"), default="baseline") + args = parser.parse_args() + + deadline = time.monotonic() + args.wait + error = None + report = None + while time.monotonic() < deadline: + try: + report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback) + error = None + if database_complete(report) if args.mode == "database" else complete(report): + break + except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error: + error = str(current_error) + time.sleep(2) + + result = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "lookback_seconds": args.lookback, + "signals": report, + "mode": args.mode, + "complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + if args.require and not result["complete"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-observe.py b/scripts/lab6-observe.py new file mode 100644 index 0000000..4614176 --- /dev/null +++ b/scripts/lab6-observe.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.parse +import urllib.request + +QUERIES = { + "api_error_ratio_1m": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)', + "api_request_rate_1m": 'sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))', + "queued_jobs": 'sum(messenger_attachment_jobs_count{status="queued"})', + "processing_jobs": 'sum(messenger_attachment_jobs_count{status="processing"})', + "oldest_queued_job_seconds": 'max(messenger_attachment_jobs_oldest_age_seconds{status="queued"})', + "worker_jobs_in_flight": 'sum(messenger_attachment_worker_jobs_in_flight)', + "worker_capacity": 'sum(messenger_attachment_worker_capacity)', + "pending_outbox": 'sum(messenger_outbox_pending)', + "worker_up": 'min(up{job="attachment-worker"})', + "collector_export_failures_5m": 'sum(increase(otelcol_exporter_send_failed_spans_total[5m])) + sum(increase(otelcol_exporter_send_failed_log_records_total[5m]))', +} + + +def get_json(url): + try: + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(4 << 20).decode("utf-8")), None + except (urllib.error.URLError, json.JSONDecodeError) as error: + return None, str(error) + + +def query(prometheus_url, expression): + url = prometheus_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + payload, error = get_json(url) + if error: + return {"error": error} + if payload.get("status") != "success": + return {"error": payload.get("error", "query failed")} + rows = payload.get("data", {}).get("result", []) + return { + "series": [ + {"labels": row.get("metric", {}), "value": row.get("value", [None, None])[1]} + for row in rows + ] + } + + +def alerts(prometheus_url): + payload, error = get_json(prometheus_url.rstrip("/") + "/api/v1/alerts") + if error: + return {"error": error, "items": []} + items = [] + for alert in payload.get("data", {}).get("alerts", []): + labels = alert.get("labels", {}) + items.append({ + "name": labels.get("alertname", ""), + "state": alert.get("state", ""), + "severity": labels.get("severity", ""), + "active_at": alert.get("activeAt", ""), + }) + return {"items": sorted(items, key=lambda item: (item["name"], item["state"]))} + + +def main(): + parser = argparse.ArgumentParser(description="Capture a bounded Prometheus snapshot for Lab 6 evidence") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--phase", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + report = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "phase": args.phase, + "queries": {name: query(args.prometheus_url, expression) for name, expression in QUERIES.items()}, + "alerts": alerts(args.prometheus_url), + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-traffic.py b/scripts/lab6-traffic.py new file mode 100755 index 0000000..244d53b --- /dev/null +++ b/scripts/lab6-traffic.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import json +import math +from pathlib import Path +import secrets +import struct +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import zlib + +MAX_RESPONSE_BYTES = 1 << 20 + + +def png_fixture(width=96, height=64): + rows = [] + for y in range(height): + pixels = bytearray() + for x in range(width): + pixels.extend(((x * 3) % 256, (y * 5) % 256, (x + y) % 256)) + rows.append(b"\x00" + bytes(pixels)) + raw = b"".join(rows) + + def chunk(name, payload): + return struct.pack(">I", len(payload)) + name + payload + struct.pack(">I", zlib.crc32(name + payload) & 0xFFFFFFFF) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"") + + +def request(method, url, body=None, headers=None, timeout=10): + req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + payload = response.read(MAX_RESPONSE_BYTES) + return response.status, payload + except urllib.error.HTTPError as error: + return error.code, error.read(MAX_RESPONSE_BYTES) + except urllib.error.URLError: + return 0, b"" + + +def decode_json(payload): + return json.loads(payload.decode("utf-8")) + + +def json_request(method, url, value=None, token=None, headers=None): + request_headers = {"Accept": "application/json"} + body = None + if value is not None: + body = json.dumps(value, separators=(",", ":")).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if token: + request_headers["Authorization"] = "Bearer " + token + if headers: + request_headers.update(headers) + status, payload = request(method, url, body, request_headers) + return status, decode_json(payload) if payload else {} + + +def bootstrap(base_url): + suffix = secrets.token_hex(6) + password = "Lab6-" + secrets.token_urlsafe(18) + users = [] + for prefix in ("telemetry_a_", "telemetry_b_"): + username = prefix + suffix + status, user = json_request("POST", base_url + "/api/v1/users", { + "username": username, + "display_name": "Synthetic telemetry load", + "password": password, + }) + if status != 201: + raise RuntimeError("register user returned HTTP %d" % status) + users.append((username, user["id"])) + + credentials = base64.b64encode((users[0][0] + ":" + password).encode("utf-8")).decode("ascii") + status, payload = request("POST", base_url + "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + credentials, + "Accept": "application/json", + }) + if status != 201: + raise RuntimeError("create session returned HTTP %d" % status) + token = decode_json(payload)["token"] + + status, chat = json_request("POST", base_url + "/api/v1/chats", { + "title": "Lab 6 synthetic telemetry", + "member_ids": [users[0][1], users[1][1]], + }, token) + if status != 201: + raise RuntimeError("create chat returned HTTP %d" % status) + return token, chat["id"] + + +class Results: + def __init__(self): + self.lock = threading.Lock() + self.latencies = [] + self.statuses = {} + self.attachment_ids = [] + + def record(self, status, elapsed, attachment_id=None): + with self.lock: + self.latencies.append(elapsed) + key = str(status) + self.statuses[key] = self.statuses.get(key, 0) + 1 + if attachment_id: + self.attachment_ids.append(attachment_id) + + +def percentile(values, quantile): + if not values: + return None + ordered = sorted(values) + index = max(0, math.ceil(quantile * len(ordered)) - 1) + return round(ordered[index] * 1000, 3) + + +def run_request(base_url, token, chat_id, mode, sequence, fixture): + if mode == "mixed": + mode = "attachments" if sequence % 3 == 0 else "messages" + started = time.monotonic() + if mode == "messages": + status, _ = json_request("POST", base_url + "/api/v1/chats/" + chat_id + "/messages", { + "text": "synthetic telemetry event", + "attachment_ids": [], + }, token, {"Idempotency-Key": secrets.token_hex(16)}) + return status, time.monotonic() - started, None + + upload_url = base_url + "/api/v1/chats/" + chat_id + "/attachments?" + urllib.parse.urlencode({ + "filename": "synthetic.png", + }) + status, payload = request("POST", upload_url, fixture, { + "Authorization": "Bearer " + token, + "Content-Type": "image/png", + "Accept": "application/json", + }) + attachment_id = None + if status == 202: + attachment_id = decode_json(payload).get("id") + return status, time.monotonic() - started, attachment_id + + +def wait_for_attachments(base_url, token, attachment_ids, timeout): + remaining = set(attachment_ids) + states = {"ready": 0, "failed": 0, "timeout": 0} + deadline = time.monotonic() + timeout + while remaining and time.monotonic() < deadline: + for attachment_id in list(remaining): + status, metadata = json_request("GET", base_url + "/api/v1/attachments/" + attachment_id, token=token) + if status != 200: + continue + state = metadata.get("status") + if state in ("ready", "failed"): + states[state] += 1 + remaining.remove(attachment_id) + if remaining: + time.sleep(0.25) + states["timeout"] = len(remaining) + return states + + +def main(): + parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents") + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed") + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--rate", type=float, default=0, help="Approximate aggregate request rate; zero is unthrottled") + parser.add_argument("--wait-ready", action="store_true") + parser.add_argument("--ready-timeout", type=float, default=120) + parser.add_argument("--output") + parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready") + args = parser.parse_args() + + if args.duration <= 0 or args.duration > 1800: + parser.error("duration must be in (0, 1800]") + if args.requests < 0: + parser.error("requests must be non-negative") + if args.concurrency < 1 or args.concurrency > 64: + parser.error("concurrency must be in [1, 64]") + if args.rate < 0 or args.rate > 1000: + parser.error("rate must be in [0, 1000]") + + base_url = args.base_url.rstrip("/") + token, chat_id = bootstrap(base_url) + if args.started_file: + started_file = Path(args.started_file) + started_file.parent.mkdir(parents=True, exist_ok=True) + started_file.touch() + fixture = png_fixture() + results = Results() + next_sequence = 0 + sequence_lock = threading.Lock() + deadline = time.monotonic() + args.duration + + def worker(): + nonlocal next_sequence + while time.monotonic() < deadline: + with sequence_lock: + if args.requests and next_sequence >= args.requests: + return + sequence = next_sequence + next_sequence += 1 + status, elapsed, attachment_id = run_request(base_url, token, chat_id, args.mode, sequence, fixture) + results.record(status, elapsed, attachment_id) + if args.rate: + delay = args.concurrency / args.rate - elapsed + if delay > 0: + time.sleep(delay) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [pool.submit(worker) for _ in range(args.concurrency)] + for future in futures: + future.result() + elapsed = time.monotonic() - started + + attachment_states = None + if args.wait_ready and results.attachment_ids: + attachment_states = wait_for_attachments(base_url, token, results.attachment_ids, args.ready_timeout) + + total = len(results.latencies) + successes = sum(count for status, count in results.statuses.items() if status.startswith("2")) + report = { + "schema_version": 1, + "workload": args.mode, + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - elapsed)), + "duration_seconds": round(elapsed, 3), + "concurrency": args.concurrency, + "requested_rate": args.rate or None, + "requests": total, + "successful_requests": successes, + "request_rate": round(total / elapsed, 3) if elapsed else None, + "status_counts": dict(sorted(results.statuses.items())), + "latency_ms": { + "p50": percentile(results.latencies, 0.50), + "p95": percentile(results.latencies, 0.95), + "p99": percentile(results.latencies, 0.99), + }, + "attachments_accepted": len(results.attachment_ids), + "attachment_terminal_states": attachment_states, + } + encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(encoded, encoding="utf-8") + print(encoded, end="") + if not total: + raise SystemExit("no requests were sent") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d587251 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + +docker compose stop nginx api worker dispatcher redis-broker valkey postgres s3 +docker compose start postgres valkey redis-broker s3 +docker compose run --rm s3-bootstrap +docker compose run --rm migrate +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..695e3f8 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,223 @@ +#!/bin/sh + +set -eu + +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +manifest=$1 +deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"} +current_manifest=$deploy_dir/current-images.env +active_containers=$deploy_dir/active-api-containers +previous_containers=$deploy_dir/previous-api-containers +upstream_file=$deploy_dir/upstream.conf +upstream_backup=$deploy_dir/upstream.previous.conf +candidate_file=$deploy_dir/candidate-api-containers +replicas=${API_REPLICAS:-2} +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5} +switched=false +nginx_created=false +upstream_changed=false +old_pool="" + +mkdir -p "$deploy_dir" +: > "$candidate_file" +export NGINX_UPSTREAM_FILE=$upstream_file + +load_manifest() { + set -a + # shellcheck disable=SC1090 + . "$1" + set +a +} + +compose() { + docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@" +} + +container_name() { + name=$(docker inspect --format '{{.Name}}' "$1") + printf '%s\n' "${name#/}" +} + +running_container() { + [ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ] +} + +write_upstream() { + output=$1 + containers=$2 + : > "$output" + for container in $containers; do + name=$(container_name "$container") + printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + done +} + +wait_for_containers() { + containers=$1 + deadline=$(($(date +%s) + 120)) + + while [ "$(date +%s)" -lt "$deadline" ]; do + count=0 + healthy=0 + for container in $containers; do + count=$((count + 1)) + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container") + if [ "$status" = healthy ]; then + healthy=$((healthy + 1)) + fi + done + if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then + return 0 + fi + sleep 1 + done + + printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2 + return 1 +} + +smoke_check() { + compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready +} + +remove_containers() { + containers=$1 + for container in $containers; do + docker rm -f "$container" >/dev/null 2>&1 || true + done +} + +restore_upstream() { + if [ -s "$upstream_backup" ]; then + cp "$upstream_backup" "$upstream_file" + compose exec -T nginx nginx -t + compose exec -T nginx nginx -s reload + fi +} + +rollback() { + status=$? + trap - EXIT INT TERM + candidates=$(cat "$candidate_file" 2>/dev/null || true) + + if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then + printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2 + restore_upstream || true + if [ "$switched" = true ]; then + smoke_check || true + fi + elif [ "$nginx_created" = true ]; then + compose stop nginx >/dev/null 2>&1 || true + fi + remove_containers "$candidates" + exit "$status" +} + +trap rollback EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +load_manifest "$manifest" +new_api_image=$API_IMAGE + +if [ -f "$previous_containers" ]; then + remove_containers "$(cat "$previous_containers")" + rm -f "$previous_containers" +fi + +if [ -s "$active_containers" ]; then + for container in $(cat "$active_containers"); do + if running_container "$container"; then + old_pool="$old_pool $container" + fi + done +fi +if [ -z "$old_pool" ]; then + old_pool=$(compose ps -q api) +fi + +if [ -s "$upstream_file" ]; then + : +elif [ -n "$old_pool" ]; then + write_upstream "$upstream_file" "$old_pool" +else + cp nginx/upstream.conf "$upstream_file" +fi +cp "$upstream_file" "$upstream_backup" + +compose pull api nginx swagger prometheus grafana s3 s3-bootstrap +compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 +compose run --rm --no-deps s3-bootstrap +compose run --rm --no-deps migrate + +nginx_container=$(compose ps -q nginx) +if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then + compose up -d --no-build --no-deps --force-recreate nginx + nginx_container=$(compose ps -q nginx) +fi +if [ -n "$nginx_container" ]; then + running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container") + if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then + printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2 + exit 1 + fi +fi + +API_IMAGE=$new_api_image +export API_IMAGE +short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12) +if [ -z "$short_digest" ]; then + short_digest=$(date +%s) +fi +generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"} + +candidate_pool="" +index=1 +while [ "$index" -le "$replicas" ]; do + name="${project}-api-${short_digest}-${generation}-${index}" + container=$(compose run -d --no-deps --name "$name" api) + candidate_pool="$candidate_pool $container" + printf '%s\n' "$container" >> "$candidate_file" + index=$((index + 1)) +done +wait_for_containers "$candidate_pool" + +write_upstream "$upstream_file.next" "$candidate_pool" +cp "$upstream_file.next" "$upstream_file" +upstream_changed=true + +if [ -n "$(compose ps -q nginx)" ]; then + compose exec -T nginx nginx -t + switched=true + compose exec -T nginx nginx -s reload +else + nginx_created=true + compose up -d --no-build --no-deps nginx + switched=true +fi +smoke_check + +application_network="${project}_application" +for container in $candidate_pool; do + docker network disconnect "$application_network" "$container" + docker network connect --alias api "$application_network" "$container" +done +smoke_check + +compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana + +printf '%s\n' "$candidate_pool" > "$active_containers" +cp "$manifest" "$current_manifest" +rm -f "$candidate_file" "$upstream_file.next" +trap - EXIT INT TERM + +if [ -n "$old_pool" ]; then + for container in $old_pool; do + docker stop --time 6 "$container" >/dev/null || true + done + printf '%s\n' "$old_pool" > "$previous_containers" +fi diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..5327251 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/scripts/test-lab5-acceptance.sh b/scripts/test-lab5-acceptance.sh new file mode 100755 index 0000000..57f27ca --- /dev/null +++ b/scripts/test-lab5-acceptance.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -eu + +state_file=$(mktemp) + +restore_pipeline() { + rm -f "$state_file" + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM + +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file" +docker compose restart api redis-broker >/dev/null +docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null +docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file" diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 0000000..dd6433c --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -eu + +restore_pipeline() { + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment diff --git a/scripts/test-lab6.sh b/scripts/test-lab6.sh new file mode 100755 index 0000000..c2ffb45 --- /dev/null +++ b/scripts/test-lab6.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +profile=${1:-full} +case "$profile" in + smoke|full) ;; + *) + echo "usage: $0 [smoke|full]" >&2 + exit 2 + ;; +esac + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir=${EVIDENCE_DIR:-evidence/runtime/lab6-$run_id} +mkdir -p "$output_dir" + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + compose up -d postgres redis-broker s3 valkey api nginx dispatcher worker otel-collector prometheus loki tempo grafana >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +test -f "$env_file" +compose config --quiet +compose up -d --build --wait --wait-timeout 180 +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "${BASELINE_LOAD_SECONDS:-15}" \ + --requests "${BASELINE_REQUESTS:-120}" \ + --concurrency 4 \ + --rate "${BASELINE_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/baseline-traffic.json" >/dev/null +python3 scripts/lab6-observe.py \ + --prometheus-url "$prometheus_url" \ + --phase baseline \ + --output "$output_dir/baseline-observability.json" >/dev/null +python3 scripts/lab6-correlation.py \ + --tempo-url "$tempo_url" \ + --loki-url "$loki_url" \ + --output "$output_dir/baseline-correlation.json" \ + --require >/dev/null + +if [ "$profile" = "full" ]; then + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh +fi + +arguments= +if [ "$profile" = "full" ]; then + arguments="--require-incidents" + if [ "${REQUIRE_ALERT_TRANSITIONS:-1}" = "1" ]; then + arguments="$arguments --require-alerts" + fi +fi +# shellcheck disable=SC2086 +python3 scripts/validate-lab6-evidence.py "$output_dir" $arguments +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/validate-lab6-evidence.py b/scripts/validate-lab6-evidence.py new file mode 100755 index 0000000..733ad02 --- /dev/null +++ b/scripts/validate-lab6-evidence.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path + + +def load(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def check(condition, message, errors): + if not condition: + errors.append(message) + + +def main(): + parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence") + parser.add_argument("directory") + parser.add_argument("--require-incidents", action="store_true") + parser.add_argument("--require-alerts", action="store_true") + args = parser.parse_args() + + root = Path(args.directory) + errors = [] + baseline = root / "baseline-traffic.json" + check(baseline.exists(), "baseline-traffic.json is missing", errors) + if baseline.exists(): + report = load(baseline) + check(report.get("requests", 0) > 0, "baseline sent no requests", errors) + check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors) + + correlation = root / "baseline-correlation.json" + check(correlation.exists(), "baseline-correlation.json is missing", errors) + if correlation.exists(): + check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors) + + observability = root / "baseline-observability.json" + check(observability.exists(), "baseline-observability.json is missing", errors) + if observability.exists(): + firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"] + check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors) + + summaries = list(root.rglob("summary.json")) + if args.require_incidents: + names = {load(path).get("incident") for path in summaries} + for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"): + check(required in names, "incident evidence is missing: " + required, errors) + + for summary_path in summaries: + summary = load(summary_path) + incident = summary.get("incident") + traffic_path = summary_path.parent / "traffic.json" + check(traffic_path.exists(), str(traffic_path) + " is missing", errors) + if not traffic_path.exists(): + continue + traffic = load(traffic_path) + check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors) + if incident == "worker_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors) + check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors) + check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors) + elif incident == "postgres_unavailable": + statuses = traffic.get("status_counts", {}) + failures = sum(count for status, count in statuses.items() if not status.startswith("2")) + check(failures > 0, incident + " produced no observable failed requests", errors) + correlation_path = summary_path.parent / "database-correlation.json" + check(correlation_path.exists(), str(correlation_path) + " is missing", errors) + if correlation_path.exists(): + check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors) + elif incident == "telemetry_backends_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors) + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors) + check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors) + check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors) + + if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"): + for name in ("alert-firing.json", "alert-resolved.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors) + if incident == "worker_unavailable" and summary.get("profile") == "long": + for name in ("burn-fast-firing.json", "burn-slow-firing.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors) + + result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)} + print(json.dumps(result, ensure_ascii=False, indent=2)) + if errors: + raise SystemExit(1) + + +if __name__ == "__main__": + main() -- 2.54.0 From 48918e8b19e17e4a98cdf9a8228d68d48c95765c Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 6 Nov 2026 21:14:49 +0300 Subject: [PATCH 2/5] feat: secure service connections and login --- cmd/dependencycheck/main.go | 33 ++- cmd/lab5experiment/api_scenario.go | 38 ++- cmd/lab5experiment/main.go | 30 ++- cmd/messenger/main.go | 4 +- cmd/mtlsproxy/main.go | 43 ++++ config/messenger.yaml | 33 ++- go.mod | 48 ++-- go.sum | 102 ++++---- internal/bootstrap/migration.go | 4 +- internal/bootstrap/pipeline.go | 105 ++++++--- internal/bootstrap/reconciliation.go | 12 +- internal/bootstrap/replay.go | 6 +- internal/bootstrap/server.go | 139 ++++++++--- internal/bootstrap/telemetry.go | 2 +- internal/broker/redisstream/broker.go | 2 +- internal/config/config.go | 219 ++++++++++++++---- internal/config/config_test.go | 90 +++++++ internal/config/loader.go | 74 +++++- internal/config/pipeline.go | 62 ++++- internal/database/database.go | 2 +- internal/loginlimit/limiter.go | 111 +++++++++ .../loginlimit/limiter_integration_test.go | 111 +++++++++ internal/messenger/attachment/service.go | 4 +- internal/messenger/attachment/service_test.go | 2 +- .../messenger/attachment/worker/worker.go | 10 +- .../attachment/worker/worker_test.go | 6 +- internal/messenger/httpapi/attachments.go | 6 +- internal/messenger/httpapi/auth.go | 69 +++++- internal/messenger/httpapi/errors.go | 4 + internal/messenger/httpapi/handlers.go | 5 +- .../messenger/httpapi/messenger_interface.go | 10 +- internal/messenger/httpapi/model.go | 2 +- internal/messenger/httpapi/routes.go | 3 - internal/messenger/httpapi/server.go | 50 +++- internal/messenger/httpapi/server_test.go | 157 +++++++++++-- internal/objectstore/store.go | 34 ++- internal/outbox/dispatcher.go | 4 +- internal/outbox/dispatcher_test.go | 2 +- internal/password/password.go | 8 + internal/password/password_test.go | 55 +++++ internal/postgres/attachment_jobs.go | 2 +- internal/postgres/attachment_lease.go | 2 +- internal/postgres/attachment_recovery.go | 2 +- internal/postgres/attachments.go | 2 +- internal/postgres/errors.go | 2 +- internal/postgres/idempotency.go | 2 +- internal/postgres/outbox.go | 2 +- internal/postgres/store.go | 15 +- internal/postgres/store_integration_test.go | 29 ++- internal/reconcile/reconcile.go | 2 +- internal/reconcile/reconcile_test.go | 2 +- internal/security/mtlsproxy/proxy.go | 95 ++++++++ internal/security/tlsconfig/client.go | 50 ++++ internal/security/tlsconfig/client_test.go | 60 +++++ internal/security/tlsconfig/server.go | 54 +++++ internal/security/tlsconfig/server_test.go | 185 +++++++++++++++ internal/session/session.go | 9 +- internal/session/session_integration_test.go | 12 +- internal/telemetry/runtime.go | 68 ++++-- 59 files changed, 1980 insertions(+), 316 deletions(-) create mode 100644 cmd/mtlsproxy/main.go create mode 100644 internal/loginlimit/limiter.go create mode 100644 internal/loginlimit/limiter_integration_test.go create mode 100644 internal/security/mtlsproxy/proxy.go create mode 100644 internal/security/tlsconfig/client.go create mode 100644 internal/security/tlsconfig/client_test.go create mode 100644 internal/security/tlsconfig/server.go create mode 100644 internal/security/tlsconfig/server_test.go diff --git a/cmd/dependencycheck/main.go b/cmd/dependencycheck/main.go index eaa3d64..57fe0ef 100644 --- a/cmd/dependencycheck/main.go +++ b/cmd/dependencycheck/main.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "flag" "fmt" @@ -17,10 +19,11 @@ import ( ) type apiClient struct { - baseURL string - client *http.Client - token string - userID string + baseURL string + client *http.Client + transport *http.Transport + token string + userID string } func main() { @@ -32,9 +35,23 @@ func main() { } func run() error { - baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + baseURL := flag.String("base-url", "https://localhost:8443", "API base URL") + caFile := flag.String("ca-file", ".runtime/security/public-ca/ca.crt", "trusted CA certificate") flag.Parse() - client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}} + caPEM, err := os.ReadFile(*caFile) + if err != nil { + return fmt.Errorf("read trusted CA: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return fmt.Errorf("trusted CA does not contain a certificate") + } + transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}} + defer transport.CloseIdleConnections() + client := &apiClient{ + baseURL: *baseURL, client: &http.Client{Transport: transport, Timeout: 10 * time.Second}, + transport: transport, + } if err := client.registerAndLogin(); err != nil { return err } @@ -130,7 +147,9 @@ func (c *apiClient) verifyConcurrentIdempotency() error { instances = make(map[string]struct{}) messageID string ) - transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers} + transport := c.transport.Clone() + transport.DisableKeepAlives = true + transport.MaxConnsPerHost = workers client := &http.Client{Transport: transport, Timeout: 10 * time.Second} defer transport.CloseIdleConnections() diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go index 048a121..e238e09 100644 --- a/cmd/lab5experiment/api_scenario.go +++ b/cmd/lab5experiment/api_scenario.go @@ -3,6 +3,8 @@ package main import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/json" "errors" @@ -12,16 +14,18 @@ import ( _ "image/jpeg" "io" "net/http" + "os" "strings" "time" "github.com/google/uuid" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) type apiState struct { BaseURL string `json:"base_url"` + CAFile string `json:"ca_file"` Token string `json:"token"` AttachmentID string `json:"attachment_id"` } @@ -41,11 +45,15 @@ type apiChat struct { func runAPIProduce(arguments []string, output io.Writer) error { flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + baseURL := flags.String("base-url", "https://nginx:8443", "messenger base URL") + caFile := flags.String("ca-file", "/run/security/public-ca/ca.crt", "trusted CA certificate") if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("usage: lab5experiment api-produce [-base-url URL]") + return errors.New("usage: lab5experiment api-produce [-base-url URL] [-ca-file PATH]") + } + client, err := newAPIClient(*caFile) + if err != nil { + return err } - client := &http.Client{Timeout: 10 * time.Second} suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] password := "Lab5-" + uuid.NewString() owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) @@ -102,7 +110,9 @@ func runAPIProduce(arguments []string, output io.Writer) error { if response.StatusCode != http.StatusConflict { return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) } - return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) + return json.NewEncoder(output).Encode(apiState{ + BaseURL: *baseURL, CAFile: *caFile, Token: token, AttachmentID: attachment.ID, + }) } func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { @@ -116,7 +126,10 @@ func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { if err := json.NewDecoder(input).Decode(&state); err != nil { return fmt.Errorf("decode API scenario state: %w", err) } - client := &http.Client{Timeout: 10 * time.Second} + client, err := newAPIClient(state.CAFile) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) defer cancel() var attachment model.Attachment @@ -176,6 +189,19 @@ func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { }) } +func newAPIClient(caFile string) (*http.Client, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read trusted CA: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return nil, errors.New("trusted CA does not contain a certificate") + } + transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}} + return &http.Client{Transport: transport, Timeout: 10 * time.Second}, nil +} + func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { body, err := json.Marshal(map[string]string{ "username": username, "display_name": username, "password": password, diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go index 35c5241..8a0fc72 100644 --- a/cmd/lab5experiment/main.go +++ b/cmd/lab5experiment/main.go @@ -21,15 +21,16 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" ) const ( @@ -132,7 +133,7 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout, }) if err != nil { pool.Close() @@ -143,6 +144,15 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { pool.Close() return nil, nil, err } + brokerTLS, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: settings.BrokerTLS.CAPath, CertificatePath: settings.BrokerTLS.CertificatePath, + PrivateKeyPath: settings.BrokerTLS.PrivateKeyPath, ServerName: settings.BrokerTLS.ServerName, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions.TLSConfig = brokerTLS client := redis.NewClient(redisOptions) if err := client.Ping(ctx).Err(); err != nil { pool.Close() diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index c2c6c96..39bd0bd 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -5,8 +5,8 @@ import ( "log/slog" "os" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/logger" ) func main() { diff --git a/cmd/mtlsproxy/main.go b/cmd/mtlsproxy/main.go new file mode 100644 index 0000000..30be9b4 --- /dev/null +++ b/cmd/mtlsproxy/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/mtlsproxy" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + config := mtlsproxy.Config{ + ListenAddress: required("PROXY_LISTEN_ADDRESS"), TargetAddress: required("PROXY_TARGET_ADDRESS"), + CAPath: required("PROXY_TLS_CA_PATH"), CertificatePath: required("PROXY_TLS_CERTIFICATE_PATH"), + PrivateKeyPath: required("PROXY_TLS_PRIVATE_KEY_PATH"), AllowedClientURI: required("PROXY_ALLOWED_CLIENT_URI"), + ALPNProtocols: strings.Split(required("PROXY_ALPN_PROTOCOLS"), ","), + DialTimeout: 5 * time.Second, + } + if config.ListenAddress == "" || config.TargetAddress == "" || config.CAPath == "" || + config.CertificatePath == "" || config.PrivateKeyPath == "" || config.AllowedClientURI == "" || + len(config.ALPNProtocols) == 0 || config.ALPNProtocols[0] == "" { + return errors.New("mTLS proxy environment is incomplete") + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + return mtlsproxy.Run(ctx, config) +} + +func required(name string) string { + return os.Getenv(name) +} diff --git a/config/messenger.yaml b/config/messenger.yaml index 031e152..1bc0f3a 100644 --- a/config/messenger.yaml +++ b/config/messenger.yaml @@ -1,9 +1,21 @@ version: 1 server: + health_address: 127.0.0.1:8082 + metrics_address: 0.0.0.0:9091 + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/api/tls.crt + private_key_path: /run/security/api/tls.key + allowed_client_uri: spiffe://mtusi/lab7/load-balancer + allowed_metrics_client_uri: spiffe://mtusi/lab7/prometheus session_store: valkey session_ttl: 1h idempotency_ttl: 24h + login_rate_limit: + account_attempts: 5 + source_attempts: 20 + window: 1m request_timeout: 5s http: read_header_timeout: 5s @@ -26,11 +38,19 @@ database: health_check_period: 30s dependency_timeout: 5s +cache: + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/client/tls.crt + private_key_path: /run/security/client/tls.key + server_name: redis-gateway + migration: timeout: 30s object_storage: - use_tls: false + use_tls: true + ca_path: /run/security/internal-ca/ca.crt timeout: 5s attachments: @@ -43,6 +63,11 @@ pipeline: version: 1 broker: + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/client/tls.crt + private_key_path: /run/security/client/tls.key + server_name: redis-gateway stream: attachment-jobs consumer_group: attachment-workers failed_stream: attachment-jobs-failed @@ -74,3 +99,9 @@ worker: monitoring: metrics_address: 0.0.0.0:9091 + health_address: 127.0.0.1:9092 + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/metrics/tls.crt + private_key_path: /run/security/metrics/tls.key + allowed_client_uri: spiffe://mtusi/lab7/prometheus diff --git a/go.mod b/go.mod index 37c0a6e..e046575 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ -module gitea.privcloud.tech/dizi/mtusi2026lab6 +module gitea.privcloud.tech/dizi/mtusi2026lab7 go 1.26.0 +toolchain go1.26.10 + require ( github.com/deepteams/webp v1.2.8 github.com/google/uuid v1.6.0 @@ -12,20 +14,21 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 - go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 + go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 - go.opentelemetry.io/otel v1.38.0 - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 go.opentelemetry.io/otel/exporters/prometheus v0.60.0 - go.opentelemetry.io/otel/metric v1.38.0 - go.opentelemetry.io/otel/sdk v1.38.0 - go.opentelemetry.io/otel/sdk/log v0.14.0 - go.opentelemetry.io/otel/sdk/metric v1.38.0 - go.opentelemetry.io/otel/trace v1.38.0 + go.opentelemetry.io/otel/metric v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/sdk/log v0.21.0 + go.opentelemetry.io/otel/sdk/metric v1.45.0 + go.opentelemetry.io/otel/trace v1.45.0 golang.org/x/crypto v0.57.0 golang.org/x/sync v0.23.0 + google.golang.org/grpc v1.83.2 ) require ( @@ -37,15 +40,15 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect @@ -66,18 +69,17 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tinylib/msgp v1.6.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect - go.opentelemetry.io/otel/log v0.14.0 // indirect - go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect + go.opentelemetry.io/otel/log v0.21.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.58.0 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect - google.golang.org/grpc v1.75.0 // indirect - google.golang.org/protobuf v1.36.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 63eb5b2..08c0288 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= @@ -38,8 +38,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -48,8 +48,8 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -89,8 +89,8 @@ github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7D github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= @@ -116,40 +116,42 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s= -go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g= +go.opentelemetry.io/contrib/bridges/otelslog v0.20.0/go.mod h1:yMSQaiiq5dpfrSJCYLBcqFeJkFFI67seT4ngvx6jfVo= go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE= go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 h1:klTViGcsvLCd1xN3rZzfZ12NslC/OimbmR+k+A006RI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0/go.mod h1:jRsK04CWmXuY8A0O+wMpSf+t90RHZ53o5Qmxn2PQPfk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI= go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= -go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA= +go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc= +go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo= +go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ= +go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= @@ -158,24 +160,24 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go index 0e7dd49..3ce60ec 100644 --- a/internal/bootstrap/migration.go +++ b/internal/bootstrap/migration.go @@ -3,8 +3,8 @@ package bootstrap import ( "context" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" ) func RunMigrations() error { diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index 1539107..e25b114 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "crypto/tls" "errors" "log/slog" "net" @@ -14,16 +15,17 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) func RunDispatcher(logger *slog.Logger) error { @@ -48,7 +50,11 @@ func RunDispatcher(logger *slog.Logger) error { if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil { return err } - client, err := newRedisClient(settings.BrokerURL) + brokerTLSConfiguration, err := loadClientTLS(settings.BrokerTLS) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL, brokerTLSConfiguration) if err != nil { return err } @@ -61,9 +67,6 @@ func RunDispatcher(logger *slog.Logger) error { if err != nil { return err } - if err := broker.Ensure(context.Background()); err != nil { - return err - } metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0) dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, @@ -74,8 +77,14 @@ func RunDispatcher(logger *slog.Logger) error { if err != nil { return err } + metricsTLSConfiguration, err := loadServerTLS(settings.MetricsTLS) + if err != nil { + return err + } logger.Info("outbox dispatcher started", "stream", settings.Stream) - return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run) + return runPipelineProcess( + logger, settings.MetricsAddress, settings.HealthAddress, metricsTLSConfiguration, metrics.Handler(), dispatcher.Run, + ) } func RunWorker(logger *slog.Logger) error { @@ -103,12 +112,16 @@ func RunWorker(logger *slog.Logger) error { objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout, }) if err != nil { return err } - client, err := newRedisClient(settings.BrokerURL) + brokerTLSConfiguration, err := loadClientTLS(settings.BrokerTLS) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL, brokerTLSConfiguration) if err != nil { return err } @@ -144,15 +157,36 @@ func RunWorker(logger *slog.Logger) error { if err != nil { return err } + metricsTLSConfiguration, err := loadServerTLS(settings.MetricsTLS) + if err != nil { + return err + } logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency) - return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run) + return runPipelineProcess( + logger, settings.MetricsAddress, settings.HealthAddress, metricsTLSConfiguration, metrics.Handler(), processorWorker.Run, + ) } -func newRedisClient(rawURL string) (*redis.Client, error) { +func loadClientTLS(config config.ClientTLSConfig) (*tls.Config, error) { + return tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: config.CAPath, CertificatePath: config.CertificatePath, + PrivateKeyPath: config.PrivateKeyPath, ServerName: config.ServerName, + }) +} + +func loadServerTLS(config config.ServerTLSConfig) (*tls.Config, error) { + return tlsconfig.LoadServer(tlsconfig.ServerConfig{ + CAPath: config.CAPath, CertificatePath: config.CertificatePath, + PrivateKeyPath: config.PrivateKeyPath, AllowedClientURI: config.AllowedClientURI, + }) +} + +func newRedisClient(rawURL string, tlsConfiguration *tls.Config) (*redis.Client, error) { options, err := redis.ParseURL(rawURL) if err != nil { return nil, err } + options.TLSConfig = tlsConfiguration.Clone() client := redis.NewClient(options) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -166,19 +200,32 @@ func newRedisClient(rawURL string) (*redis.Client, error) { func runPipelineProcess( logger *slog.Logger, metricsAddress string, + healthAddress string, + metricsTLSConfiguration *tls.Config, metricsHandler http.Handler, run func(context.Context) error, ) error { - mux := http.NewServeMux() - mux.Handle("/metrics", metricsHandler) - mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) - server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second} - listener, err := net.Listen("tcp", metricsAddress) + metricsMux := http.NewServeMux() + metricsMux.Handle("/metrics", metricsHandler) + metricsServer := &http.Server{ + Addr: metricsAddress, Handler: metricsMux, ReadHeaderTimeout: 5 * time.Second, + TLSConfig: metricsTLSConfiguration, + } + healthMux := http.NewServeMux() + healthMux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + healthServer := &http.Server{Addr: healthAddress, Handler: healthMux, ReadHeaderTimeout: 5 * time.Second} + metricsListener, err := net.Listen("tcp", metricsAddress) if err != nil { return err } - serveResult := make(chan error, 1) - go func() { serveResult <- server.Serve(listener) }() + healthListener, err := net.Listen("tcp", healthAddress) + if err != nil { + _ = metricsListener.Close() + return err + } + serveResult := make(chan error, 2) + go func() { serveResult <- metricsServer.Serve(tls.NewListener(metricsListener, metricsTLSConfiguration)) }() + go func() { serveResult <- healthServer.Serve(healthListener) }() ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -195,8 +242,10 @@ func runPipelineProcess( } shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil && result == nil { - result = err + for _, server := range []*http.Server{metricsServer, healthServer} { + if err := server.Shutdown(shutdownCtx); err != nil && result == nil { + result = err + } } if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) { return nil diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go index 3f09a3f..e018498 100644 --- a/internal/bootstrap/reconciliation.go +++ b/internal/bootstrap/reconciliation.go @@ -10,11 +10,11 @@ import ( "syscall" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/reconcile" ) type reconciliationOutput struct { @@ -57,7 +57,7 @@ func RunReconciliation(arguments []string, output io.Writer) error { objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout, }) if err != nil { return err diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go index 70cc3e8..3c59e70 100644 --- a/internal/bootstrap/replay.go +++ b/internal/bootstrap/replay.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" ) func RunAttachmentReplay(arguments []string, output io.Writer) error { diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index 473c0b7..d464a5b 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -2,6 +2,9 @@ package bootstrap import ( "context" + "crypto/tls" + "errors" + "fmt" "log/slog" "net" "net/http" @@ -9,22 +12,43 @@ import ( "os/signal" "syscall" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/loginlimit" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) +type serverResult struct { + name string + err error +} + func RunServer(logger *slog.Logger) error { settings, err := config.LoadServer() if err != nil { return err } + apiTLSConfiguration, err := tlsconfig.LoadServer(tlsconfig.ServerConfig{ + CAPath: settings.TLSCAPath, CertificatePath: settings.TLSCertificatePath, + PrivateKeyPath: settings.TLSPrivateKeyPath, AllowedClientURI: settings.TLSAllowedClientURI, + }) + if err != nil { + return err + } + metricsTLSConfiguration, err := tlsconfig.LoadServer(tlsconfig.ServerConfig{ + CAPath: settings.TLSCAPath, CertificatePath: settings.TLSCertificatePath, + PrivateKeyPath: settings.TLSPrivateKeyPath, AllowedClientURI: settings.TLSAllowedMetricsClientURI, + }) + if err != nil { + return err + } instanceID, err := os.Hostname() if err != nil { return err @@ -42,11 +66,27 @@ func RunServer(logger *slog.Logger) error { if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil { return err } - sessions, err := session.New(settings.CacheURL) + cacheTLSConfiguration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: settings.CacheTLS.CAPath, CertificatePath: settings.CacheTLS.CertificatePath, + PrivateKeyPath: settings.CacheTLS.PrivateKeyPath, ServerName: settings.CacheTLS.ServerName, + }) + if err != nil { + return err + } + sessions, err := session.New(settings.CacheURL, cacheTLSConfiguration) if err != nil { return err } defer func() { _ = sessions.Close() }() + loginLimiter, err := loginlimit.New(settings.CacheURL, loginlimit.Config{ + AccountAttempts: int64(settings.LoginRateLimitAccountAttempts), + SourceAttempts: int64(settings.LoginRateLimitSourceAttempts), + Window: settings.LoginRateLimitWindow, + }, cacheTLSConfiguration) + if err != nil { + return err + } + defer func() { _ = loginLimiter.Close() }() passwords, err := password.New(password.Config{ MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, @@ -70,40 +110,85 @@ func RunServer(logger *slog.Logger) error { } app := httpapi.NewServer(logger, httpapi.Dependencies{ Users: repository, Chats: repository, Messages: repository, Attachments: attachments, - Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + Sessions: sessions, Passwords: passwords, LoginLimiter: loginLimiter, + Database: repository, Objects: objects, Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(), }, httpapi.Config{ SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, }) - server := &http.Server{ + apiServer := &http.Server{ Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout, - IdleTimeout: settings.HTTPIdleTimeout, + IdleTimeout: settings.HTTPIdleTimeout, TLSConfig: apiTLSConfiguration, } - listener, err := net.Listen("tcp", server.Addr) + healthServer := &http.Server{ + Addr: settings.HealthAddress, Handler: app.HealthHandler(), + ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, ReadTimeout: settings.HTTPReadTimeout, + WriteTimeout: settings.HTTPWriteTimeout, IdleTimeout: settings.HTTPIdleTimeout, + } + metricsServer := &http.Server{ + Addr: settings.MetricsAddress, Handler: app.MetricsHandler(), + ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, ReadTimeout: settings.HTTPReadTimeout, + WriteTimeout: settings.HTTPWriteTimeout, IdleTimeout: settings.HTTPIdleTimeout, + TLSConfig: metricsTLSConfiguration, + } + apiListener, err := net.Listen("tcp", apiServer.Addr) if err != nil { return err } - serveResult := make(chan error, 1) - go func() { serveResult <- server.Serve(listener) }() - logger.Info("server started", "address", server.Addr) + healthListener, err := net.Listen("tcp", healthServer.Addr) + if err != nil { + _ = apiListener.Close() + return err + } + metricsListener, err := net.Listen("tcp", metricsServer.Addr) + if err != nil { + _ = apiListener.Close() + _ = healthListener.Close() + return err + } + serveResults := make(chan serverResult, 3) + go serveHTTP(serveResults, "api", apiServer, tls.NewListener(apiListener, apiTLSConfiguration)) + go serveHTTP(serveResults, "health", healthServer, healthListener) + go serveHTTP(serveResults, "metrics", metricsServer, tls.NewListener(metricsListener, metricsTLSConfiguration)) + logger.Info("servers started", "api_address", apiServer.Addr, "health_address", healthServer.Addr, "metrics_address", metricsServer.Addr) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() + var serveErr error select { - case err := <-serveResult: - return err + case result := <-serveResults: + serveErr = fmt.Errorf("%s server stopped: %w", result.name, result.err) case <-ctx.Done(): logger.Info("shutdown requested") - app.SetReady(false) - shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) - defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil { - _ = server.Close() - return err - } + } + app.SetReady(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) + defer cancel() + shutdownErr := shutdownServers(shutdownCtx, apiServer, healthServer, metricsServer) + return errors.Join(serveErr, shutdownErr) +} + +func serveHTTP(results chan<- serverResult, name string, server *http.Server, listener net.Listener) { + results <- serverResult{name: name, err: server.Serve(listener)} +} + +func shutdownServers(ctx context.Context, servers ...*http.Server) error { + results := make(chan error, len(servers)) + for _, server := range servers { + go func() { results <- server.Shutdown(ctx) }() + } + var shutdownErr error + for range servers { + shutdownErr = errors.Join(shutdownErr, <-results) + } + if shutdownErr == nil { return nil } + for _, server := range servers { + _ = server.Close() + } + return shutdownErr } func databaseConfig(settings config.ServerConfig) database.Config { @@ -118,6 +203,6 @@ func objectStoreConfig(settings config.ServerConfig) objectstore.Config { return objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout, } } diff --git a/internal/bootstrap/telemetry.go b/internal/bootstrap/telemetry.go index bb1dcd9..e055263 100644 --- a/internal/bootstrap/telemetry.go +++ b/internal/bootstrap/telemetry.go @@ -5,7 +5,7 @@ import ( "log/slog" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) { diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go index 0788997..45fa60a 100644 --- a/internal/broker/redisstream/broker.go +++ b/internal/broker/redisstream/broker.go @@ -16,7 +16,7 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/outbox" ) const ( diff --git a/internal/config/config.go b/internal/config/config.go index 977abc5..b022fc8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,41 +12,53 @@ import ( ) type ServerConfig struct { - Address string - DatabaseURL string - CacheURL string - SessionTTL time.Duration - IdempotencyTTL time.Duration - DBMaxConns int32 - DBMinConns int32 - DBMaxConnectionAge time.Duration - DBMaxConnectionIdle time.Duration - DBHealthCheckPeriod time.Duration - DependencyTimeout time.Duration - RequestTimeout time.Duration - HTTPReadHeaderTimeout time.Duration - HTTPReadTimeout time.Duration - HTTPWriteTimeout time.Duration - HTTPIdleTimeout time.Duration - HTTPShutdownTimeout time.Duration - PasswordMemoryKiB uint32 - PasswordIterations uint32 - PasswordParallelism uint8 - PasswordSaltLength uint32 - PasswordKeyLength uint32 - SwaggerPort string - S3Endpoint string - S3Region string - S3Bucket string - S3AccessKey string - S3SecretKey string - S3UseTLS bool - S3Timeout time.Duration - AttachmentMaxBytes int64 - AttachmentMaxPixels int64 - AttachmentBodyMemory int64 - AttachmentDecodeMemory int64 - PipelineVersion int + Address string + HealthAddress string + MetricsAddress string + TLSCAPath string + TLSCertificatePath string + TLSPrivateKeyPath string + TLSAllowedClientURI string + TLSAllowedMetricsClientURI string + DatabaseURL string + CacheURL string + CacheTLS ClientTLSConfig + SessionTTL time.Duration + IdempotencyTTL time.Duration + LoginRateLimitAccountAttempts int + LoginRateLimitSourceAttempts int + LoginRateLimitWindow time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3CAPath string + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentBodyMemory int64 + AttachmentDecodeMemory int64 + PipelineVersion int } type MigrationConfig struct { @@ -74,6 +86,7 @@ type ReconciliationConfig struct { S3AccessKey string S3SecretKey string S3UseTLS bool + S3CAPath string S3Timeout time.Duration } @@ -94,9 +107,24 @@ type objectStorageConfig struct { accessKey string secretKey string useTLS bool + caPath string timeout time.Duration } +type ClientTLSConfig struct { + CAPath string + CertificatePath string + PrivateKeyPath string + ServerName string +} + +type ServerTLSConfig struct { + CAPath string + CertificatePath string + PrivateKeyPath string + AllowedClientURI string +} + func LoadServer() (ServerConfig, error) { values, err := loadValues() if err != nil { @@ -111,6 +139,34 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + healthAddress, err := loopbackAddress(values, "server.health_address") + if err != nil { + return ServerConfig{}, err + } + metricsAddress, err := nonLoopbackAddress(values, "server.metrics_address") + if err != nil { + return ServerConfig{}, err + } + tlsCAPath, err := requiredString(values, "server.tls.ca_path") + if err != nil { + return ServerConfig{}, err + } + tlsCertificatePath, err := requiredString(values, "server.tls.certificate_path") + if err != nil { + return ServerConfig{}, err + } + tlsPrivateKeyPath, err := requiredString(values, "server.tls.private_key_path") + if err != nil { + return ServerConfig{}, err + } + tlsAllowedClientURI, err := requiredString(values, "server.tls.allowed_client_uri") + if err != nil { + return ServerConfig{}, err + } + tlsAllowedMetricsClientURI, err := requiredString(values, "server.tls.allowed_metrics_client_uri") + if err != nil { + return ServerConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return ServerConfig{}, err @@ -119,6 +175,10 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + cacheTLS, err := loadClientTLSConfig(values, "cache.tls") + if err != nil { + return ServerConfig{}, err + } store, err := requiredString(values, "server.session_store") if err != nil { return ServerConfig{}, err @@ -135,6 +195,18 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + loginRateLimitAccountAttempts, err := integer(values, "server.login_rate_limit.account_attempts", 1, 1_000_000) + if err != nil { + return ServerConfig{}, err + } + loginRateLimitSourceAttempts, err := integer(values, "server.login_rate_limit.source_attempts", 1, 1_000_000) + if err != nil { + return ServerConfig{}, err + } + loginRateLimitWindow, err := duration(values, "server.login_rate_limit.window", time.Second, 24*time.Hour) + if err != nil { + return ServerConfig{}, err + } requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err @@ -216,9 +288,16 @@ func LoadServer() (ServerConfig, error) { } return ServerConfig{ - Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, + Address: net.JoinHostPort(host, port), HealthAddress: healthAddress, MetricsAddress: metricsAddress, + TLSCAPath: tlsCAPath, TLSCertificatePath: tlsCertificatePath, + TLSPrivateKeyPath: tlsPrivateKeyPath, TLSAllowedClientURI: tlsAllowedClientURI, + TLSAllowedMetricsClientURI: tlsAllowedMetricsClientURI, + DatabaseURL: database.url, CacheURL: cacheURL, CacheTLS: cacheTLS, SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, - DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + LoginRateLimitAccountAttempts: loginRateLimitAccountAttempts, + LoginRateLimitSourceAttempts: loginRateLimitSourceAttempts, + LoginRateLimitWindow: loginRateLimitWindow, + DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, RequestTimeout: requestTimeout, @@ -229,7 +308,7 @@ func LoadServer() (ServerConfig, error) { PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, - S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + S3UseTLS: storage.useTLS, S3CAPath: storage.caPath, S3Timeout: storage.timeout, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), AttachmentBodyMemory: int64(attachmentBodyMemory), AttachmentDecodeMemory: int64(attachmentDecodeMemory), @@ -277,7 +356,7 @@ func LoadReconciliation() (ReconciliationConfig, error) { DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, - S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + S3UseTLS: storage.useTLS, S3CAPath: storage.caPath, S3Timeout: storage.timeout, }, nil } @@ -306,13 +385,40 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { if err != nil { return objectStorageConfig{}, err } + caPath, err := requiredString(values, "object_storage.ca_path") + if err != nil { + return objectStorageConfig{}, err + } timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) if err != nil { return objectStorageConfig{}, err } return objectStorageConfig{ endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey, - secretKey: secretKey, useTLS: useTLS, timeout: timeout, + secretKey: secretKey, useTLS: useTLS, caPath: caPath, timeout: timeout, + }, nil +} + +func loadClientTLSConfig(values *viper.Viper, prefix string) (ClientTLSConfig, error) { + caPath, err := requiredString(values, prefix+".ca_path") + if err != nil { + return ClientTLSConfig{}, err + } + certificatePath, err := requiredString(values, prefix+".certificate_path") + if err != nil { + return ClientTLSConfig{}, err + } + privateKeyPath, err := requiredString(values, prefix+".private_key_path") + if err != nil { + return ClientTLSConfig{}, err + } + serverName, err := requiredString(values, prefix+".server_name") + if err != nil { + return ClientTLSConfig{}, err + } + return ClientTLSConfig{ + CAPath: caPath, CertificatePath: certificatePath, + PrivateKeyPath: privateKeyPath, ServerName: serverName, }, nil } @@ -371,6 +477,39 @@ func requiredPort(values *viper.Viper, name string) (string, error) { return value, nil } +func loopbackAddress(values *viper.Viper, name string) (string, error) { + value, err := requiredString(values, name) + if err != nil { + return "", err + } + host, port, err := net.SplitHostPort(value) + if err != nil || host != "127.0.0.1" { + return "", fmt.Errorf("%s: must use 127.0.0.1 with an explicit port", name) + } + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return net.JoinHostPort(host, port), nil +} + +func nonLoopbackAddress(values *viper.Viper, name string) (string, error) { + value, err := requiredString(values, name) + if err != nil { + return "", err + } + host, port, err := net.SplitHostPort(value) + if err != nil || host == "" || strings.EqualFold(host, "localhost") { + return "", fmt.Errorf("%s: must use a non-loopback host with an explicit port", name) + } + if address := net.ParseIP(host); address != nil && address.IsLoopback() { + return "", fmt.Errorf("%s: must use a non-loopback host with an explicit port", name) + } + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return net.JoinHostPort(host, port), nil +} + func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { value, err := requiredString(values, name) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 80d2216..7306499 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -46,6 +46,66 @@ func TestLoadValuesRejectsUnknownVersion(t *testing.T) { require.ErrorContains(t, err, "version: must be 1") } +func TestLoadServerLoadsTLSAndHealthSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + + settings, err := LoadServer() + require.NoError(t, err) + require.Equal(t, "127.0.0.1:8082", settings.HealthAddress) + require.Equal(t, "0.0.0.0:9091", settings.MetricsAddress) + require.Equal(t, "/run/security/internal-ca/ca.crt", settings.TLSCAPath) + require.Equal(t, "spiffe://mtusi/lab7/load-balancer", settings.TLSAllowedClientURI) + require.Equal(t, "spiffe://mtusi/lab7/prometheus", settings.TLSAllowedMetricsClientURI) + require.Equal(t, 5, settings.LoginRateLimitAccountAttempts) + require.Equal(t, 20, settings.LoginRateLimitSourceAttempts) + require.Equal(t, time.Minute, settings.LoginRateLimitWindow) +} + +func TestLoadServerRejectsNonLoopbackHealthAddress(t *testing.T) { + content := strings.Replace(validTestConfig, "health_address: 127.0.0.1:8082", "health_address: 0.0.0.0:8082", 1) + setConfigEnvironment(t, content) + + _, err := LoadServer() + require.ErrorContains(t, err, "server.health_address: must use 127.0.0.1") +} + +func TestLoadServerRejectsLoopbackMetricsAddress(t *testing.T) { + content := strings.Replace(validTestConfig, "metrics_address: 0.0.0.0:9091", "metrics_address: 127.0.0.1:9091", 1) + setConfigEnvironment(t, content) + + _, err := LoadServer() + require.ErrorContains(t, err, "server.metrics_address: must use a non-loopback host") +} + +func TestLoadServerUsesRenderedOpenBaoSecrets(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + path := filepath.Join(t.TempDir(), "secrets.json") + require.NoError(t, os.WriteFile(path, []byte(`{ + "database_url":"postgres://openbao/database", + "cache_url":"rediss://openbao/cache", + "s3_access_key":"openbao-api", + "s3_secret_key":"openbao-secret" +}`), 0o600)) + t.Setenv("MESSENGER_SECRETS_FILE", path) + + settings, err := LoadServer() + require.NoError(t, err) + require.Equal(t, "postgres://openbao/database", settings.DatabaseURL) + require.Equal(t, "rediss://openbao/cache", settings.CacheURL) + require.Equal(t, "openbao-api", settings.S3AccessKey) + require.Equal(t, "openbao-secret", settings.S3SecretKey) +} + +func TestLoadServerRejectsUnknownOpenBaoSecret(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + path := filepath.Join(t.TempDir(), "secrets.json") + require.NoError(t, os.WriteFile(path, []byte(`{"root_token":"forbidden"}`), 0o600)) + t.Setenv("MESSENGER_SECRETS_FILE", path) + + _, err := LoadServer() + require.ErrorContains(t, err, "unsupported field root_token") +} + func setConfigEnvironment(t *testing.T, content string) { t.Helper() path := filepath.Join(t.TempDir(), "messenger.yaml") @@ -64,9 +124,21 @@ func setConfigEnvironment(t *testing.T, content string) { const validTestConfig = `version: 1 server: + health_address: 127.0.0.1:8082 + metrics_address: 0.0.0.0:9091 + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/api/tls.crt + private_key_path: /run/security/api/tls.key + allowed_client_uri: spiffe://mtusi/lab7/load-balancer + allowed_metrics_client_uri: spiffe://mtusi/lab7/prometheus session_store: valkey session_ttl: 1h idempotency_ttl: 24h + login_rate_limit: + account_attempts: 5 + source_attempts: 20 + window: 1m request_timeout: 5s http: read_header_timeout: 5s @@ -87,10 +159,17 @@ database: max_connection_idle: 1m health_check_period: 30s dependency_timeout: 5s +cache: + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/client/tls.crt + private_key_path: /run/security/client/tls.key + server_name: redis-gateway migration: timeout: 30s object_storage: use_tls: false + ca_path: /run/security/internal-ca/ca.crt timeout: 5s attachments: max_bytes: 10485760 @@ -100,6 +179,11 @@ attachments: pipeline: version: 1 broker: + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/client/tls.crt + private_key_path: /run/security/client/tls.key + server_name: redis-gateway stream: attachment-jobs consumer_group: attachment-workers failed_stream: attachment-jobs-failed @@ -128,4 +212,10 @@ worker: memory_bytes: 738197504 monitoring: metrics_address: 0.0.0.0:9091 + health_address: 127.0.0.1:9092 + tls: + ca_path: /run/security/internal-ca/ca.crt + certificate_path: /run/security/metrics/tls.crt + private_key_path: /run/security/metrics/tls.key + allowed_client_uri: spiffe://mtusi/lab7/prometheus ` diff --git a/internal/config/loader.go b/internal/config/loader.go index 6cd115c..5084cd1 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -1,8 +1,10 @@ package config import ( + "encoding/json" "errors" "fmt" + "io" "os" "strings" @@ -12,17 +14,32 @@ import ( const supportedConfigVersion = 1 var environmentKeys = map[string]string{ - "server.host": "APP_HOST", - "server.port": "APP_PORT", - "server.swagger_port": "SWAGGER_PORT", - "database.url": "DATABASE_URL", - "cache.url": "CACHE_URL", - "object_storage.endpoint": "S3_ENDPOINT", - "object_storage.region": "S3_REGION", - "object_storage.bucket": "S3_BUCKET", - "object_storage.access_key": "S3_ACCESS_KEY", - "object_storage.secret_key": "S3_SECRET_KEY", - "broker.url": "BROKER_URL", + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.health_address": "APP_HEALTH_ADDRESS", + "server.metrics_address": "APP_METRICS_ADDRESS", + "server.tls.ca_path": "APP_TLS_CA_PATH", + "server.tls.certificate_path": "APP_TLS_CERTIFICATE_PATH", + "server.tls.private_key_path": "APP_TLS_PRIVATE_KEY_PATH", + "server.tls.allowed_client_uri": "APP_TLS_ALLOWED_CLIENT_URI", + "server.tls.allowed_metrics_client_uri": "APP_TLS_ALLOWED_METRICS_CLIENT_URI", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +var secretKeys = map[string]string{ + "database_url": "database.url", + "cache_url": "cache.url", + "broker_url": "broker.url", + "s3_access_key": "object_storage.access_key", + "s3_secret_key": "object_storage.secret_key", } func loadValues() (*viper.Viper, error) { @@ -43,5 +60,40 @@ func loadValues() (*viper.Viper, error) { return nil, fmt.Errorf("bind %s: %w", name, err) } } + if err := loadSecretFile(values, strings.TrimSpace(os.Getenv("MESSENGER_SECRETS_FILE"))); err != nil { + return nil, err + } return values, nil } + +func loadSecretFile(values *viper.Viper, path string) error { + if path == "" { + return nil + } + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("open messenger secrets: %w", err) + } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, 64<<10)) + var secrets map[string]string + if err := decoder.Decode(&secrets); err != nil { + return fmt.Errorf("decode messenger secrets: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return errors.New("decode messenger secrets: trailing JSON value") + } + for name, value := range secrets { + key, ok := secretKeys[name] + if !ok { + return fmt.Errorf("messenger secrets: unsupported field %s", name) + } + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("messenger secrets: %s is empty", name) + } + values.Set(key, value) + } + return nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go index 76b4f5d..0536ddb 100644 --- a/internal/config/pipeline.go +++ b/internal/config/pipeline.go @@ -18,6 +18,7 @@ type DispatcherConfig struct { DBHealthCheckPeriod time.Duration DependencyTimeout time.Duration BrokerURL string + BrokerTLS ClientTLSConfig Stream string ConsumerGroup string FailedStream string @@ -31,6 +32,8 @@ type DispatcherConfig struct { CleanupInterval time.Duration CleanupBatchSize int MetricsAddress string + HealthAddress string + MetricsTLS ServerTLSConfig } type WorkerConfig struct { @@ -47,8 +50,10 @@ type WorkerConfig struct { S3AccessKey string S3SecretKey string S3UseTLS bool + S3CAPath string S3Timeout time.Duration BrokerURL string + BrokerTLS ClientTLSConfig Stream string ConsumerGroup string FailedStream string @@ -67,6 +72,8 @@ type WorkerConfig struct { AttachmentMaxPixels int64 MemoryBytes int64 MetricsAddress string + HealthAddress string + MetricsTLS ServerTLSConfig } type JobAdminConfig struct { @@ -116,6 +123,10 @@ func LoadDispatcher() (DispatcherConfig, error) { if err != nil { return DispatcherConfig{}, err } + brokerTLS, err := loadClientTLSConfig(values, "broker.tls") + if err != nil { + return DispatcherConfig{}, err + } batchSize, err := integer(values, "outbox.batch_size", 1, 1000) if err != nil { return DispatcherConfig{}, err @@ -152,16 +163,24 @@ func LoadDispatcher() (DispatcherConfig, error) { if err != nil { return DispatcherConfig{}, err } + healthAddress, err := loopbackAddress(values, "monitoring.health_address") + if err != nil { + return DispatcherConfig{}, err + } + metricsTLS, err := loadServerTLSConfig(values, "monitoring.tls") + if err != nil { + return DispatcherConfig{}, err + } return DispatcherConfig{ DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, - BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, + BrokerURL: broker.url, BrokerTLS: brokerTLS, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, - MetricsAddress: metricsAddress, + MetricsAddress: metricsAddress, HealthAddress: healthAddress, MetricsTLS: metricsTLS, }, nil } @@ -182,6 +201,10 @@ func LoadWorker() (WorkerConfig, error) { if err != nil { return WorkerConfig{}, err } + brokerTLS, err := loadClientTLSConfig(values, "broker.tls") + if err != nil { + return WorkerConfig{}, err + } concurrency, err := integer(values, "worker.concurrency", 1, 64) if err != nil { return WorkerConfig{}, err @@ -247,13 +270,22 @@ func LoadWorker() (WorkerConfig, error) { if err != nil { return WorkerConfig{}, err } + healthAddress, err := loopbackAddress(values, "monitoring.health_address") + if err != nil { + return WorkerConfig{}, err + } + metricsTLS, err := loadServerTLSConfig(values, "monitoring.tls") + if err != nil { + return WorkerConfig{}, err + } return WorkerConfig{ DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, - S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream, + S3CAPath: storage.caPath, S3Timeout: storage.timeout, + BrokerURL: broker.url, BrokerTLS: brokerTLS, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, @@ -261,6 +293,7 @@ func LoadWorker() (WorkerConfig, error) { ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, + HealthAddress: healthAddress, MetricsTLS: metricsTLS, }, nil } @@ -279,6 +312,29 @@ func metricsAddress(values *viper.Viper) (string, error) { return address, nil } +func loadServerTLSConfig(values *viper.Viper, prefix string) (ServerTLSConfig, error) { + caPath, err := requiredString(values, prefix+".ca_path") + if err != nil { + return ServerTLSConfig{}, err + } + certificatePath, err := requiredString(values, prefix+".certificate_path") + if err != nil { + return ServerTLSConfig{}, err + } + privateKeyPath, err := requiredString(values, prefix+".private_key_path") + if err != nil { + return ServerTLSConfig{}, err + } + allowedClientURI, err := requiredString(values, prefix+".allowed_client_uri") + if err != nil { + return ServerTLSConfig{}, err + } + return ServerTLSConfig{ + CAPath: caPath, CertificatePath: certificatePath, + PrivateKeyPath: privateKeyPath, AllowedClientURI: allowedClientURI, + }, nil +} + func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { url, err := requiredString(values, "broker.url") if err != nil { diff --git a/internal/database/database.go b/internal/database/database.go index c631b06..0a00149 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" + "gitea.privcloud.tech/dizi/mtusi2026lab7/migrations" ) var ( diff --git a/internal/loginlimit/limiter.go b/internal/loginlimit/limiter.go new file mode 100644 index 0000000..bf7f2b9 --- /dev/null +++ b/internal/loginlimit/limiter.go @@ -0,0 +1,111 @@ +package loginlimit + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "strconv" + "time" + + "github.com/valkey-io/valkey-go" +) + +const keyPrefix = "login-rate:{login}:" + +var ( + ErrInvalidConfig = errors.New("invalid login rate limit configuration") + ErrStore = errors.New("login rate limit store unavailable") +) + +type Config struct { + AccountAttempts int64 + SourceAttempts int64 + Window time.Duration +} + +type Limiter struct { + client valkey.Client + script *valkey.Lua + config Config +} + +func New(rawURL string, config Config, tlsConfiguration ...*tls.Config) (*Limiter, error) { + if config.AccountAttempts < 1 || config.SourceAttempts < 1 || config.Window < time.Second { + return nil, ErrInvalidConfig + } + option, err := valkey.ParseURL(rawURL) + if err != nil { + return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err) + } + if len(tlsConfiguration) > 0 && tlsConfiguration[0] != nil { + option.TLSConfig = tlsConfiguration[0].Clone() + } + option.DisableCache = true + client, err := valkey.NewClient(option) + if err != nil { + return nil, fmt.Errorf("%w: create Valkey client: %w", ErrStore, err) + } + return &Limiter{client: client, script: valkey.NewLuaScript(limitScript), config: config}, nil +} + +func (l *Limiter) Allow(ctx context.Context, username, source string) (bool, time.Duration, error) { + result, err := l.script.Exec(ctx, l.client, + []string{key("account", username), key("source", source)}, + []string{ + strconv.FormatInt(l.config.Window.Milliseconds(), 10), + strconv.FormatInt(l.config.AccountAttempts, 10), + strconv.FormatInt(l.config.SourceAttempts, 10), + }, + ).AsIntSlice() + if err != nil { + return false, 0, fmt.Errorf("%w: account login attempt: %w", ErrStore, err) + } + if len(result) != 2 { + return false, 0, fmt.Errorf("%w: unexpected script result", ErrStore) + } + return result[0] == 1, time.Duration(result[1]) * time.Millisecond, nil +} + +func (l *Limiter) Close() error { + l.client.Close() + return nil +} + +func key(scope, value string) string { + digest := sha256.Sum256([]byte(value)) + return keyPrefix + scope + ":" + hex.EncodeToString(digest[:]) +} + +const limitScript = ` +local function increment(key, window) + local count = redis.call('INCR', key) + if count == 1 then + redis.call('PEXPIRE', key, window) + end + local ttl = redis.call('PTTL', key) + if ttl < 1 then + redis.call('PEXPIRE', key, window) + ttl = window + end + return count, ttl +end + +local window = tonumber(ARGV[1]) +local account_count, account_ttl = increment(KEYS[1], window) +local source_count, source_ttl = increment(KEYS[2], window) +local retry_after = 0 + +if account_count > tonumber(ARGV[2]) then + retry_after = account_ttl +end +if source_count > tonumber(ARGV[3]) and source_ttl > retry_after then + retry_after = source_ttl +end +if retry_after > 0 then + return {0, retry_after} +end +return {1, 0} +` diff --git a/internal/loginlimit/limiter_integration_test.go b/internal/loginlimit/limiter_integration_test.go new file mode 100644 index 0000000..312a20f --- /dev/null +++ b/internal/loginlimit/limiter_integration_test.go @@ -0,0 +1,111 @@ +package loginlimit_test + +import ( + "context" + "crypto/tls" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/loginlimit" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" +) + +func TestLimitIsSharedByInstances(t *testing.T) { + rawURL := testCacheURL(t) + tlsConfiguration := testCacheTLS(t) + config := loginlimit.Config{AccountAttempts: 5, SourceAttempts: 20, Window: time.Minute} + first, err := loginlimit.New(rawURL, config, tlsConfiguration) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.Close()) }) + second, err := loginlimit.New(rawURL, config, tlsConfiguration) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, second.Close()) }) + + username := "rate_" + uuid.NewString() + limiters := []*loginlimit.Limiter{first, second} + results := make(chan bool, 16) + errors := make(chan error, 16) + var group sync.WaitGroup + for i := range 16 { + group.Add(1) + go func() { + defer group.Done() + allowed, retryAfter, err := limiters[i%len(limiters)].Allow(context.Background(), username, "192.0.2.10") + if err != nil { + errors <- err + return + } + if !allowed && retryAfter <= 0 { + errors <- loginlimit.ErrStore + return + } + results <- allowed + }() + } + group.Wait() + close(results) + close(errors) + for err := range errors { + require.NoError(t, err) + } + var allowed int + for result := range results { + if result { + allowed++ + } + } + require.Equal(t, int(config.AccountAttempts), allowed) +} + +func TestSourceLimitCoversDifferentAccounts(t *testing.T) { + limiter, err := loginlimit.New(testCacheURL(t), loginlimit.Config{ + AccountAttempts: 10, + SourceAttempts: 2, + Window: time.Minute, + }, testCacheTLS(t)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, limiter.Close()) }) + + suffix := uuid.NewString() + for i, expected := range []bool{true, true, false} { + allowed, retryAfter, err := limiter.Allow(context.Background(), suffix+string(rune('a'+i)), "198.51.100.20") + require.NoError(t, err) + require.Equal(t, expected, allowed) + if !allowed { + require.Positive(t, retryAfter) + } + } +} + +func testCacheTLS(t *testing.T) *tls.Config { + t.Helper() + configuration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: os.Getenv("TEST_CACHE_TLS_CA_PATH"), + CertificatePath: os.Getenv("TEST_CACHE_TLS_CERTIFICATE_PATH"), + PrivateKeyPath: os.Getenv("TEST_CACHE_TLS_PRIVATE_KEY_PATH"), + ServerName: os.Getenv("TEST_CACHE_TLS_SERVER_NAME"), + }) + require.NoError(t, err) + return configuration +} + +func testCacheURL(t *testing.T) string { + t.Helper() + rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) + if rawURL != "" { + return rawURL + } + path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE")) + if path == "" { + t.Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") + } + data, err := os.ReadFile(path) + require.NoError(t, err) + return strings.TrimSpace(string(data)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index 38e5bdc..3054d5f 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -19,8 +19,8 @@ import ( "github.com/google/uuid" "golang.org/x/sync/semaphore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) type Repository interface { diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index 296f283..448c4ac 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func TestUploadLifecycle(t *testing.T) { diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index 45ff600..6ef0db2 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -23,11 +23,11 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) type Repository interface { diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go index c2ca384..05d1cdd 100644 --- a/internal/messenger/attachment/worker/worker_test.go +++ b/internal/messenger/attachment/worker/worker_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore" ) func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index 2e832d6..84b6a79 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -16,9 +16,9 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go index 29419a1..5f26df8 100644 --- a/internal/messenger/httpapi/auth.go +++ b/internal/messenger/httpapi/auth.go @@ -1,12 +1,16 @@ package httpapi import ( + "context" "errors" "io" + "math" + "net" "net/http" + "strconv" "strings" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { @@ -21,6 +25,15 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError if !ok { return unauthorized() } + allowed, retryAfter, err := s.loginLimiter.Allow(r.Context(), username, loginSource(r)) + if err != nil { + return unavailable() + } + if !allowed { + retrySeconds := max(int64(1), int64(math.Ceil(retryAfter.Seconds()))) + w.Header().Set("Retry-After", strconv.FormatInt(retrySeconds, 10)) + return rateLimited() + } credentials, err := s.userRepository.UserByUsername(r.Context(), username) if errors.Is(err, model.ErrNotFound) { if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { @@ -38,6 +51,9 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError if !verified { return unauthorized() } + if rehashErr := s.rehashPassword(r.Context(), credentials, passwordValue); rehashErr != nil { + return rehashErr + } token, expiresAt, err := s.sessions.Create(r.Context(), credentials.ID, s.sessionTTL) if err != nil { return unavailable() @@ -51,6 +67,57 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError return nil } +func (s *Server) rehashPassword(ctx context.Context, credentials model.UserCredentials, passwordValue string) *apiError { + needsRehash, err := s.passwords.NeedsRehash(credentials.PasswordHash) + if err != nil { + return internalError() + } + if !needsRehash { + return nil + } + passwordHash, err := s.passwords.Hash(passwordValue) + if err != nil { + return internalError() + } + updated, err := s.userRepository.UpdatePasswordHash(ctx, credentials.ID, credentials.PasswordHash, passwordHash) + if err != nil { + return unavailable() + } + if updated { + return nil + } + current, err := s.userRepository.UserByUsername(ctx, credentials.Username) + if errors.Is(err, model.ErrNotFound) { + return unauthorized() + } + if err != nil { + return unavailable() + } + verified, err := s.passwords.Verify(passwordValue, current.PasswordHash) + if err != nil { + return internalError() + } + if !verified { + return unauthorized() + } + return nil +} + +func loginSource(r *http.Request) string { + forwarded := strings.Split(r.Header.Get("X-Forwarded-For"), ",") + // Nginx appends the address it observed last, so preceding client-supplied entries are not trusted. + if address := net.ParseIP(strings.TrimSpace(forwarded[len(forwarded)-1])); address != nil { + return address.String() + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + if address := net.ParseIP(host); address != nil { + return address.String() + } + } + return "unknown" +} + func singleBasicAuth(r *http.Request) (string, string, bool) { if len(r.Header.Values("Authorization")) != 1 { return "", "", false diff --git a/internal/messenger/httpapi/errors.go b/internal/messenger/httpapi/errors.go index 95ffd1c..0567d7f 100644 --- a/internal/messenger/httpapi/errors.go +++ b/internal/messenger/httpapi/errors.go @@ -36,6 +36,10 @@ func unauthorized() *apiError { return &apiError{http.StatusUnauthorized, errorCodeUnauthorized, "authentication required"} } +func rateLimited() *apiError { + return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "too many requests"} +} + func notFound() *apiError { return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"} } diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go index 9084d13..e119c59 100644 --- a/internal/messenger/httpapi/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -9,7 +9,7 @@ import ( "time" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) @@ -32,12 +32,15 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) defer cancel() if err := s.database.Ping(ctx); err != nil { + s.logger.ErrorContext(ctx, "readiness dependency failed", "dependency", "postgres", "error", err) return unavailable() } if err := s.sessions.Ping(ctx); err != nil { + s.logger.ErrorContext(ctx, "readiness dependency failed", "dependency", "valkey", "error", err) return unavailable() } if err := s.objects.Ping(ctx); err != nil { + s.logger.ErrorContext(ctx, "readiness dependency failed", "dependency", "s3", "error", err) return unavailable() } writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go index 9844c25..54d8bf1 100644 --- a/internal/messenger/httpapi/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -4,14 +4,15 @@ import ( "context" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) type UserRepository interface { CreateUser(context.Context, string, string, string) (model.User, error) UserByID(context.Context, string) (model.User, error) UserByUsername(context.Context, string) (model.UserCredentials, error) + UpdatePasswordHash(context.Context, string, string, string) (bool, error) } type ChatRepository interface { @@ -45,5 +46,10 @@ type SessionStore interface { type PasswordService interface { Hash(string) (string, error) Verify(string, string) (bool, error) + NeedsRehash(string) (bool, error) DummyHash() string } + +type LoginLimiter interface { + Allow(context.Context, string, string) (bool, time.Duration, error) +} diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go index 7f3f6dc..366c128 100644 --- a/internal/messenger/httpapi/model.go +++ b/internal/messenger/httpapi/model.go @@ -1,7 +1,7 @@ package httpapi import ( - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" "github.com/google/uuid" ) diff --git a/internal/messenger/httpapi/routes.go b/internal/messenger/httpapi/routes.go index f272767..6912660 100644 --- a/internal/messenger/httpapi/routes.go +++ b/internal/messenger/httpapi/routes.go @@ -14,9 +14,6 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError { handler = s.live case "/health/ready": handler = s.readiness - case "/metrics": - s.metrics.handler.ServeHTTP(w, r) - return nil case "/api/v1/users": handler = s.users case "/api/v1/users/me": diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go index fcc18bf..5371e98 100644 --- a/internal/messenger/httpapi/server.go +++ b/internal/messenger/httpapi/server.go @@ -15,20 +15,21 @@ import ( semconv "go.opentelemetry.io/otel/semconv/v1.34.0" "go.opentelemetry.io/otel/trace" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) type Dependencies struct { - Users UserRepository - Chats ChatRepository - Messages MessageRepository - Attachments AttachmentService - Sessions SessionStore - Passwords PasswordService - Database HealthChecker - Objects HealthChecker - Meter metric.Meter - Metrics http.Handler + Users UserRepository + Chats ChatRepository + Messages MessageRepository + Attachments AttachmentService + Sessions SessionStore + Passwords PasswordService + LoginLimiter LoginLimiter + Database HealthChecker + Objects HealthChecker + Meter metric.Meter + Metrics http.Handler } type Config struct { @@ -46,6 +47,7 @@ type Server struct { attachments AttachmentService sessions SessionStore passwords PasswordService + loginLimiter LoginLimiter database HealthChecker objects HealthChecker sessionTTL time.Duration @@ -77,7 +79,8 @@ func NewServer( userRepository: dependencies.Users, chatRepository: dependencies.Chats, messageRepository: dependencies.Messages, attachments: dependencies.Attachments, sessions: dependencies.Sessions, - passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, + passwords: dependencies.Passwords, loginLimiter: dependencies.LoginLimiter, + database: dependencies.Database, objects: dependencies.Objects, sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(dependencies.Meter, dependencies.Metrics), @@ -94,6 +97,29 @@ func (s *Server) Handler() http.Handler { return http.HandlerFunc(s.serve) } +func (s *Server) HealthHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health/live", func(w http.ResponseWriter, r *http.Request) { + s.serveHealth(w, r, s.live) + }) + mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) { + s.serveHealth(w, r, s.readiness) + }) + return mux +} + +func (s *Server) MetricsHandler() http.Handler { + mux := http.NewServeMux() + mux.Handle("/metrics", s.metrics.handler) + return mux +} + +func (s *Server) serveHealth(w http.ResponseWriter, r *http.Request, handler requestHandler) { + if err := handler(w, r); err != nil { + writeError(w, "", err) + } +} + func (s *Server) serve(w http.ResponseWriter, r *http.Request) { requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) tracer := otel.Tracer("messenger-api/http") diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index c1bf91d..29e51db 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) var errDependency = errors.New("dependency unavailable") @@ -34,14 +34,15 @@ func (s *MessengerSuite) SetupTest() { } type messengerStoreStub struct { - pingErr error - userByID model.User - userByIDErr error - credentials model.UserCredentials - credentialsErr error - message model.Message - messageReplay bool - messageErr error + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error + updatePasswordHash func(context.Context, string, string, string) (bool, error) } func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } @@ -54,6 +55,12 @@ func (s messengerStoreStub) UserByID(context.Context, string) (model.User, error func (s messengerStoreStub) UserByUsername(context.Context, string) (model.UserCredentials, error) { return s.credentials, s.credentialsErr } +func (s messengerStoreStub) UpdatePasswordHash(ctx context.Context, userID, previousHash, passwordHash string) (bool, error) { + if s.updatePasswordHash == nil { + return false, nil + } + return s.updatePasswordHash(ctx, userID, previousHash, passwordHash) +} func (messengerStoreStub) CreateChat(context.Context, string, []string, string) (model.Chat, error) { return model.Chat{}, nil } @@ -109,15 +116,37 @@ func (s sessionStub) UserID(context.Context, string) (string, error) { return s. func (sessionStub) Delete(context.Context, string) error { return nil } type passwordsStub struct { - verified bool - err error + verified bool + err error + hash string + hashErr error + needsRehash bool + needsRehashErr error } -func (passwordsStub) Hash(string) (string, error) { return "hash", nil } -func (passwordsStub) DummyHash() string { return "hash" } +func (s passwordsStub) Hash(string) (string, error) { + if s.hash == "" { + return "hash", s.hashErr + } + return s.hash, s.hashErr +} +func (passwordsStub) DummyHash() string { return "hash" } func (s passwordsStub) Verify(string, string) (bool, error) { return s.verified, s.err } +func (s passwordsStub) NeedsRehash(string) (bool, error) { + return s.needsRehash, s.needsRehashErr +} + +type loginLimiterStub struct { + allowed bool + retryAfter time.Duration + err error +} + +func (s loginLimiterStub) Allow(context.Context, string, string) (bool, time.Duration, error) { + return s.allowed, s.retryAfter, s.err +} func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { t := s.T() @@ -135,7 +164,7 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{}) - response := serveRequest(server, http.MethodGet, "/health/ready", "") + response := serveHealthRequest(server, http.MethodGet, "/health/ready") if response.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) } @@ -143,6 +172,29 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { } } +func (s *MessengerSuite) TestHealthHandlerDoesNotExposeBusinessAPI() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveHealthRequest(server, http.MethodGet, "/api/v1/users/me") + s.Equal(http.StatusNotFound, response.Code) +} + +func (s *MessengerSuite) TestMetricsHandlerOnlyExposesMetrics() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + server.metrics.handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + metrics := httptest.NewRecorder() + server.MetricsHandler().ServeHTTP(metrics, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + s.Equal(http.StatusNoContent, metrics.Code) + for _, path := range []string{"/health/ready", "/api/v1/users/me"} { + response := httptest.NewRecorder() + server.MetricsHandler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + s.Equal(http.StatusNotFound, response.Code, path) + } + s.Equal(http.StatusNotFound, serveRequest(server, http.MethodGet, "/metrics", "").Code) +} + func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() { t := s.T() tests := []struct { @@ -187,6 +239,48 @@ func (s *MessengerSuite) TestAuthenticationChallenges() { } } +func (s *MessengerSuite) TestSessionCreationReturnsRateLimitWithRetryAfter() { + server := s.serverWithLoginLimiter( + messengerStoreStub{}, sessionStub{}, passwordsStub{}, + loginLimiterStub{retryAfter: 1500 * time.Millisecond}, + ) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("student", "T3st-password-123") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + + s.Equal(http.StatusTooManyRequests, response.Code) + s.Equal("2", response.Header().Get("Retry-After")) + s.Contains(response.Body.String(), `"code":"rate_limited"`) +} + +func (s *MessengerSuite) TestSessionCreationRehashesPasswordWithCompareAndSwap() { + var updated []string + store := messengerStoreStub{ + credentials: model.UserCredentials{User: model.User{ID: "user-id"}, PasswordHash: "old-hash"}, + updatePasswordHash: func(_ context.Context, userID, previousHash, passwordHash string) (bool, error) { + updated = []string{userID, previousHash, passwordHash} + return true, nil + }, + } + server := s.newTestServer(store, sessionStub{}, passwordsStub{ + verified: true, needsRehash: true, hash: "new-hash", + }) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("student", "T3st-password-123") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + + s.Equal(http.StatusCreated, response.Code) + s.Equal([]string{"user-id", "old-hash", "new-hash"}, updated) +} + +func (s *MessengerSuite) TestLoginSourceUsesAddressAppendedByIngress() { + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.Header.Set("X-Forwarded-For", "203.0.113.9, 198.51.100.7") + s.Equal("198.51.100.7", loginSource(request)) +} + func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() { server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) response := serveRequest( @@ -357,10 +451,34 @@ func (s *MessengerSuite) server( passwords PasswordService, objects objectStoreStub, attachments AttachmentService, +) *Server { + return s.serverWithDependencies( + store, sessions, passwords, loginLimiterStub{allowed: true}, objects, attachments, + ) +} + +func (s *MessengerSuite) serverWithLoginLimiter( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + limiter LoginLimiter, +) *Server { + return s.serverWithDependencies( + store, sessions, passwords, limiter, objectStoreStub{}, attachmentServiceStub{}, + ) +} + +func (s *MessengerSuite) serverWithDependencies( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + limiter LoginLimiter, + objects objectStoreStub, + attachments AttachmentService, ) *Server { return NewServer(s.logger, Dependencies{ Users: store, Chats: store, Messages: store, Attachments: attachments, - Sessions: sessions, Passwords: passwords, Database: store, Objects: objects, + Sessions: sessions, Passwords: passwords, LoginLimiter: limiter, Database: store, Objects: objects, }, Config{ SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second, InstanceID: "test-instance", SwaggerPort: "8081", @@ -376,3 +494,10 @@ func serveRequest(server *Server, method, path, body string) *httptest.ResponseR server.Handler().ServeHTTP(response, request) return response } + +func serveHealthRequest(server *Server, method, path string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, nil) + response := httptest.NewRecorder() + server.HealthHandler().ServeHTTP(response, request) + return response +} diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go index e2d6ded..d5c214b 100644 --- a/internal/objectstore/store.go +++ b/internal/objectstore/store.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "io" + "net" + "net/http" "net/url" "strings" "time" @@ -18,7 +20,8 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" ) var ErrNotFound = errors.New("object not found") @@ -30,6 +33,7 @@ type Config struct { AccessKey string SecretKey string UseTLS bool + CAPath string Timeout time.Duration } @@ -47,11 +51,27 @@ func New(config Config) (*Store, error) { if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 { return nil, errors.New("invalid object store configuration") } - client, err := minio.New(endpoint, &minio.Options{ + options := &minio.Options{ Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), Secure: config.UseTLS, Region: config.Region, - }) + } + if config.UseTLS { + host, _, err := net.SplitHostPort(endpoint) + if err != nil { + return nil, errors.New("S3 endpoint must include a port") + } + tlsConfiguration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: config.CAPath, ServerName: host, + }) + if err != nil { + return nil, fmt.Errorf("load S3 TLS configuration: %w", err) + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = tlsConfiguration + options.Transport = transport + } + client, err := minio.New(endpoint, options) if err != nil { return nil, fmt.Errorf("create S3 client: %w", err) } @@ -79,12 +99,8 @@ func endpointHost(value string, useTLS bool) (string, error) { func (s *Store) Ping(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() - exists, err := s.client.BucketExists(ctx, s.bucket) - if err != nil { - return fmt.Errorf("check S3 bucket: %w", err) - } - if !exists { - return errors.New("S3 bucket does not exist") + if _, err := s.client.GetBucketLocation(ctx, s.bucket); err != nil { + return fmt.Errorf("check S3 bucket location: %w", err) } return nil } diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go index ea74f61..5b76b75 100644 --- a/internal/outbox/dispatcher.go +++ b/internal/outbox/dispatcher.go @@ -11,8 +11,8 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) var errLeaseLost = errors.New("outbox lease lost") diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go index b9d792c..51c4145 100644 --- a/internal/outbox/dispatcher_test.go +++ b/internal/outbox/dispatcher_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" "github.com/stretchr/testify/require" ) diff --git a/internal/password/password.go b/internal/password/password.go index 5dd8b1b..0d97def 100644 --- a/internal/password/password.go +++ b/internal/password/password.go @@ -68,6 +68,14 @@ func (h *Hasher) Verify(value, encodedHash string) (bool, error) { return subtle.ConstantTimeCompare(actual, digest) == 1, nil } +func (h *Hasher) NeedsRehash(encodedHash string) (bool, error) { + config, _, _, err := decode(encodedHash) + if err != nil { + return false, err + } + return config != h.config, nil +} + func (h *Hasher) DummyHash() string { return h.dummyHash } diff --git a/internal/password/password_test.go b/internal/password/password_test.go index 8e88875..efe63f4 100644 --- a/internal/password/password_test.go +++ b/internal/password/password_test.go @@ -39,6 +39,61 @@ func (s *PasswordSuite) TestVerifyRejectsMalformedHash() { s.Error(err) } +func (s *PasswordSuite) TestNeedsRehashDetectsChangedParameters() { + currentHash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + needsRehash, err := s.hasher.NeedsRehash(currentHash) + s.Require().NoError(err) + s.False(needsRehash) + + legacyConfig := testConfig() + legacyConfig.KeyLength = 16 + legacy, err := New(legacyConfig) + s.Require().NoError(err) + legacyHash, err := legacy.Hash("T3st-password-123") + s.Require().NoError(err) + needsRehash, err = s.hasher.NeedsRehash(legacyHash) + s.Require().NoError(err) + s.True(needsRehash) +} + func testConfig() Config { return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32} } + +func BenchmarkArgon2idHash(b *testing.B) { + hasher, err := New(productionConfig()) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, err := hasher.Hash("T3st-password-123"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkArgon2idVerify(b *testing.B) { + hasher, err := New(productionConfig()) + if err != nil { + b.Fatal(err) + } + hash, err := hasher.Hash("T3st-password-123") + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + verified, err := hasher.Verify("T3st-password-123", hash) + if err != nil || !verified { + b.Fatalf("verify: result=%v error=%v", verified, err) + } + } +} + +func productionConfig() Config { + return Config{MemoryKiB: 64 * 1024, Iterations: 3, Parallelism: 2, SaltLength: 16, KeyLength: 32} +} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 6c959de..314e7fc 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -10,7 +10,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) const attachmentJobTopic = "attachment.jobs" diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go index 9e631a9..261f3b9 100644 --- a/internal/postgres/attachment_lease.go +++ b/internal/postgres/attachment_lease.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func (s *Store) RenewAttachmentJobLease( diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go index e305a79..4774764 100644 --- a/internal/postgres/attachment_recovery.go +++ b/internal/postgres/attachment_recovery.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func (s *Store) RecoverExpiredAttachmentJobs( diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 2765eda..2a5d759 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) const attachmentColumns = ` diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index 39509ad..8c13bb9 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -1,6 +1,6 @@ package postgres -import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +import "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" var ( ErrConflict = model.ErrConflict diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index 4bbb95b..3261e66 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -12,7 +12,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) const expiredIdempotencyCleanupBatch = 100 diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go index 0817f7c..b6b056a 100644 --- a/internal/postgres/outbox.go +++ b/internal/postgres/outbox.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func (s *Store) ClaimOutboxEvents( diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 44980b7..5489fe8 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) type Store struct { @@ -100,6 +100,19 @@ func (s *Store) UserByUsername(ctx context.Context, username string) (model.User return credentials, nil } +func (s *Store) UpdatePasswordHash(ctx context.Context, userID, previousHash, passwordHash string) (bool, error) { + const query = ` + UPDATE users + SET password_hash = $1 + WHERE id = $2 AND password_hash = $3` + + result, err := s.pool.Exec(ctx, query, passwordHash, userID, previousHash) + if err != nil { + return false, fmt.Errorf("update password hash: %w", err) + } + return result.RowsAffected() == 1, nil +} + func (s *Store) CreateChat( ctx context.Context, title string, diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index e4184db..5e3b4f3 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -15,10 +15,10 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/reconcile" ) type PostgresSuite struct { @@ -77,6 +77,27 @@ func (s *PostgresSuite) TestConcurrentUsernameCreation() { } } +func (s *PostgresSuite) TestPasswordHashUpdateUsesCompareAndSwap() { + username := uniqueUsername() + user, err := s.store.CreateUser(context.Background(), username, "Password user", "old-hash") + s.Require().NoError(err) + + updated, err := s.store.UpdatePasswordHash(context.Background(), user.ID, "other-hash", "new-hash") + s.Require().NoError(err) + s.False(updated) + + updated, err = s.store.UpdatePasswordHash(context.Background(), user.ID, "old-hash", "new-hash") + s.Require().NoError(err) + s.True(updated) + updated, err = s.store.UpdatePasswordHash(context.Background(), user.ID, "old-hash", "stale-hash") + s.Require().NoError(err) + s.False(updated) + + credentials, err := s.store.UserByUsername(context.Background(), username) + s.Require().NoError(err) + s.Equal("new-hash", credentials.PasswordHash) +} + func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() { t := s.T() pool := s.pool diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 15607f8..4c04575 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) type MetadataStore interface { diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 36a2955..2236ab8 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) func TestReconcilerLifecycleAndRepeat(t *testing.T) { diff --git a/internal/security/mtlsproxy/proxy.go b/internal/security/mtlsproxy/proxy.go new file mode 100644 index 0000000..ff88d71 --- /dev/null +++ b/internal/security/mtlsproxy/proxy.go @@ -0,0 +1,95 @@ +package mtlsproxy + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "sync" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" +) + +type Config struct { + ListenAddress string + TargetAddress string + CAPath string + CertificatePath string + PrivateKeyPath string + AllowedClientURI string + ALPNProtocols []string + DialTimeout time.Duration +} + +func Run(ctx context.Context, config Config) error { + if config.ListenAddress == "" || config.TargetAddress == "" || config.DialTimeout <= 0 { + return errors.New("invalid mTLS proxy configuration") + } + tlsConfiguration, err := tlsconfig.LoadServer(tlsconfig.ServerConfig{ + CAPath: config.CAPath, CertificatePath: config.CertificatePath, + PrivateKeyPath: config.PrivateKeyPath, AllowedClientURI: config.AllowedClientURI, + }) + if err != nil { + return err + } + tlsConfiguration.NextProtos = config.ALPNProtocols + listener, err := net.Listen("tcp", config.ListenAddress) + if err != nil { + return fmt.Errorf("listen: %w", err) + } + defer listener.Close() + go func() { + <-ctx.Done() + _ = listener.Close() + }() + + var connections sync.WaitGroup + defer connections.Wait() + for { + connection, err := listener.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("accept: %w", err) + } + connections.Add(1) + go func() { + defer connections.Done() + serve(ctx, connection, config, tlsConfiguration) + }() + } +} + +func serve(ctx context.Context, connection net.Conn, config Config, tlsConfiguration *tls.Config) { + client := tls.Server(connection, tlsConfiguration) + defer client.Close() + handshakeCtx, cancel := context.WithTimeout(ctx, config.DialTimeout) + defer cancel() + if err := client.HandshakeContext(handshakeCtx); err != nil { + return + } + target, err := (&net.Dialer{Timeout: config.DialTimeout}).DialContext(ctx, "tcp", config.TargetAddress) + if err != nil { + return + } + defer target.Close() + + result := make(chan struct{}, 2) + copyConnection := func(destination, source net.Conn) { + _, _ = io.Copy(destination, source) + if writer, ok := destination.(interface{ CloseWrite() error }); ok { + _ = writer.CloseWrite() + } + result <- struct{}{} + } + go copyConnection(target, client) + go copyConnection(client, target) + select { + case <-ctx.Done(): + case <-result: + } +} diff --git a/internal/security/tlsconfig/client.go b/internal/security/tlsconfig/client.go new file mode 100644 index 0000000..e0ac0da --- /dev/null +++ b/internal/security/tlsconfig/client.go @@ -0,0 +1,50 @@ +package tlsconfig + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "strings" +) + +type ClientConfig struct { + CAPath string + CertificatePath string + PrivateKeyPath string + ServerName string +} + +func LoadClient(config ClientConfig) (*tls.Config, error) { + if strings.TrimSpace(config.ServerName) == "" { + return nil, errors.New("TLS server name is required") + } + caPEM, err := os.ReadFile(config.CAPath) + if err != nil { + return nil, fmt.Errorf("read server CA: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return nil, errors.New("server CA does not contain a certificate") + } + + var certificates []tls.Certificate + if config.CertificatePath != "" || config.PrivateKeyPath != "" { + if config.CertificatePath == "" || config.PrivateKeyPath == "" { + return nil, errors.New("TLS client certificate and private key must be configured together") + } + certificate, err := tls.LoadX509KeyPair(config.CertificatePath, config.PrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("load client certificate: %w", err) + } + certificates = []tls.Certificate{certificate} + } + + return &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: roots, + Certificates: certificates, + ServerName: config.ServerName, + }, nil +} diff --git a/internal/security/tlsconfig/client_test.go b/internal/security/tlsconfig/client_test.go new file mode 100644 index 0000000..f36bdaa --- /dev/null +++ b/internal/security/tlsconfig/client_test.go @@ -0,0 +1,60 @@ +package tlsconfig + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLoadClientLoadsTrustAndIdentity(t *testing.T) { + caPath, certificatePath, keyPath := writeClientCertificate(t) + + configuration, err := LoadClient(ClientConfig{ + CAPath: caPath, CertificatePath: certificatePath, PrivateKeyPath: keyPath, ServerName: "collector", + }) + require.NoError(t, err) + require.Equal(t, uint16(0x0304), configuration.MinVersion) + require.Equal(t, "collector", configuration.ServerName) + require.Len(t, configuration.Certificates, 1) + require.NotNil(t, configuration.RootCAs) +} + +func TestLoadClientRejectsPartialIdentity(t *testing.T) { + caPath, certificatePath, _ := writeClientCertificate(t) + + _, err := LoadClient(ClientConfig{CAPath: caPath, CertificatePath: certificatePath, ServerName: "collector"}) + require.ErrorContains(t, err, "configured together") +} + +func writeClientCertificate(t *testing.T) (string, string, string) { + t.Helper() + directory := t.TempDir() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + privateKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + caPath := filepath.Join(directory, "ca.crt") + certificatePath := filepath.Join(directory, "tls.crt") + keyPath := filepath.Join(directory, "tls.key") + require.NoError(t, os.WriteFile(caPath, certificate, 0o600)) + require.NoError(t, os.WriteFile(certificatePath, certificate, 0o600)) + require.NoError(t, os.WriteFile(keyPath, privateKey, 0o600)) + return caPath, certificatePath, keyPath +} diff --git a/internal/security/tlsconfig/server.go b/internal/security/tlsconfig/server.go new file mode 100644 index 0000000..c767c26 --- /dev/null +++ b/internal/security/tlsconfig/server.go @@ -0,0 +1,54 @@ +package tlsconfig + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/url" + "os" +) + +type ServerConfig struct { + CAPath string + CertificatePath string + PrivateKeyPath string + AllowedClientURI string +} + +func LoadServer(config ServerConfig) (*tls.Config, error) { + allowedURI, err := url.Parse(config.AllowedClientURI) + if err != nil || allowedURI.Scheme == "" || allowedURI.Host == "" || allowedURI.String() != config.AllowedClientURI { + return nil, errors.New("allowed client URI must be an absolute canonical URI") + } + caPEM, err := os.ReadFile(config.CAPath) + if err != nil { + return nil, fmt.Errorf("read client CA: %w", err) + } + clientCAs := x509.NewCertPool() + if !clientCAs.AppendCertsFromPEM(caPEM) { + return nil, errors.New("client CA does not contain a certificate") + } + certificate, err := tls.LoadX509KeyPair(config.CertificatePath, config.PrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("load server certificate: %w", err) + } + + return &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{certificate}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return errors.New("client certificate is required") + } + for _, identity := range state.PeerCertificates[0].URIs { + if identity.String() == config.AllowedClientURI { + return nil + } + } + return fmt.Errorf("client certificate URI SAN is not allowed: want %s", config.AllowedClientURI) + }, + }, nil +} diff --git a/internal/security/tlsconfig/server_test.go b/internal/security/tlsconfig/server_test.go new file mode 100644 index 0000000..3183b4f --- /dev/null +++ b/internal/security/tlsconfig/server_test.go @@ -0,0 +1,185 @@ +package tlsconfig + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const allowedClientURI = "spiffe://mtusi/lab7/load-balancer" + +func TestServerRequiresAllowedClientIdentity(t *testing.T) { + files := issueTestCertificates(t) + serverConfig, err := LoadServer(ServerConfig{ + CAPath: files.caPath, CertificatePath: files.serverCertificatePath, + PrivateKeyPath: files.serverPrivateKeyPath, AllowedClientURI: allowedClientURI, + }) + require.NoError(t, err) + + tests := []struct { + name string + certificatePath string + privateKeyPath string + wantServerError bool + }{ + {name: "allowed identity", certificatePath: files.allowedCertificatePath, privateKeyPath: files.allowedPrivateKeyPath}, + {name: "other identity from trusted CA", certificatePath: files.otherCertificatePath, privateKeyPath: files.otherPrivateKeyPath, wantServerError: true}, + {name: "missing certificate", wantServerError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientConfig := &tls.Config{RootCAs: files.caPool, ServerName: "api.internal", MinVersion: tls.VersionTLS13} + if tt.certificatePath != "" { + certificate, loadErr := tls.LoadX509KeyPair(tt.certificatePath, tt.privateKeyPath) + require.NoError(t, loadErr) + clientConfig.Certificates = []tls.Certificate{certificate} + } + serverErr, clientErr := handshake(serverConfig, clientConfig) + if tt.wantServerError { + require.Error(t, serverErr) + return + } + require.NoError(t, serverErr) + require.NoError(t, clientErr) + }) + } +} + +func TestLoadServerRejectsInvalidClientCA(t *testing.T) { + directory := t.TempDir() + caPath := filepath.Join(directory, "ca.pem") + require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) + + _, err := LoadServer(ServerConfig{CAPath: caPath, AllowedClientURI: allowedClientURI}) + require.ErrorContains(t, err, "client CA does not contain a certificate") +} + +func handshake(serverConfig, clientConfig *tls.Config) (error, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return err, err + } + defer listener.Close() + serverResult := make(chan error, 1) + go func() { + connection, acceptErr := listener.Accept() + if acceptErr != nil { + serverResult <- acceptErr + return + } + server := tls.Server(connection, serverConfig) + serverResult <- server.Handshake() + _ = server.Close() + }() + client, clientErr := tls.Dial("tcp", listener.Addr().String(), clientConfig) + if client != nil { + _ = client.Close() + } + serverErr := <-serverResult + return serverErr, clientErr +} + +type testCertificateFiles struct { + caPath string + serverCertificatePath string + serverPrivateKeyPath string + allowedCertificatePath string + allowedPrivateKeyPath string + otherCertificatePath string + otherPrivateKeyPath string + caPool *x509.CertPool +} + +func issueTestCertificates(t *testing.T) testCertificateFiles { + t.Helper() + directory := t.TempDir() + caCertificate, caKey := issueCA(t) + caPath := filepath.Join(directory, "ca.pem") + writeCertificate(t, caPath, caCertificate.Raw) + caPool := x509.NewCertPool() + caPool.AddCert(caCertificate) + + serverCertificatePath, serverPrivateKeyPath := issueLeaf(t, directory, caCertificate, caKey, "server", nil, []string{"api.internal"}, x509.ExtKeyUsageServerAuth) + allowed := mustParseURI(t, allowedClientURI) + allowedCertificatePath, allowedPrivateKeyPath := issueLeaf(t, directory, caCertificate, caKey, "allowed", []*url.URL{allowed}, nil, x509.ExtKeyUsageClientAuth) + other := mustParseURI(t, "spiffe://mtusi/lab7/worker") + otherCertificatePath, otherPrivateKeyPath := issueLeaf(t, directory, caCertificate, caKey, "other", []*url.URL{other}, nil, x509.ExtKeyUsageClientAuth) + + return testCertificateFiles{ + caPath: caPath, serverCertificatePath: serverCertificatePath, serverPrivateKeyPath: serverPrivateKeyPath, + allowedCertificatePath: allowedCertificatePath, allowedPrivateKeyPath: allowedPrivateKeyPath, + otherCertificatePath: otherCertificatePath, otherPrivateKeyPath: otherPrivateKeyPath, caPool: caPool, + } +} + +func issueCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + certificate, err := x509.ParseCertificate(der) + require.NoError(t, err) + return certificate, key +} + +func issueLeaf( + t *testing.T, + directory string, + ca *x509.Certificate, + caKey *ecdsa.PrivateKey, + name string, + identities []*url.URL, + dnsNames []string, + usage x509.ExtKeyUsage, +) (string, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage}, + URIs: identities, DNSNames: dnsNames, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) + require.NoError(t, err) + certificatePath := filepath.Join(directory, name+".crt") + privateKeyPath := filepath.Join(directory, name+".key") + writeCertificate(t, certificatePath, der) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + require.NoError(t, os.WriteFile(privateKeyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certificatePath, privateKeyPath +} + +func writeCertificate(t *testing.T, path string, der []byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) +} + +func mustParseURI(t *testing.T, value string) *url.URL { + t.Helper() + identity, err := url.Parse(value) + require.NoError(t, err) + return identity +} diff --git a/internal/session/session.go b/internal/session/session.go index a7df4b9..2a57716 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "crypto/sha256" + "crypto/tls" "encoding/base64" "encoding/hex" "encoding/json" @@ -14,7 +15,7 @@ import ( "github.com/valkey-io/valkey-go" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" ) const ( @@ -40,11 +41,15 @@ type Store struct { client valkey.Client } -func New(rawURL string) (*Store, error) { +func New(rawURL string, tlsConfiguration ...*tls.Config) (*Store, error) { option, err := valkey.ParseURL(rawURL) if err != nil { return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err) } + if len(tlsConfiguration) > 0 && tlsConfiguration[0] != nil { + option.TLSConfig = tlsConfiguration[0].Clone() + } + option.DisableCache = true client, err := valkey.NewClient(option) if err != nil { diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go index a82e5eb..f231828 100644 --- a/internal/session/session_integration_test.go +++ b/internal/session/session_integration_test.go @@ -11,7 +11,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/session" ) type SessionSuite struct { @@ -34,7 +35,14 @@ func (s *SessionSuite) SetupSuite() { s.Require().NoError(err) rawURL = strings.TrimSpace(string(data)) } - store, err := session.New(rawURL) + tlsConfiguration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: os.Getenv("TEST_CACHE_TLS_CA_PATH"), + CertificatePath: os.Getenv("TEST_CACHE_TLS_CERTIFICATE_PATH"), + PrivateKeyPath: os.Getenv("TEST_CACHE_TLS_PRIVATE_KEY_PATH"), + ServerName: os.Getenv("TEST_CACHE_TLS_SERVER_NAME"), + }) + s.Require().NoError(err) + store, err := session.New(rawURL, tlsConfiguration) s.Require().NoError(err) s.store = store } diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go index 90267e6..2eb7642 100644 --- a/internal/telemetry/runtime.go +++ b/internal/telemetry/runtime.go @@ -26,6 +26,9 @@ import ( "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "google.golang.org/grpc/credentials" + + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig" ) const ( @@ -34,13 +37,17 @@ const ( ) type Config struct { - ServiceName string - Version string - InstanceID string - Endpoint string - Environment string - Namespace string - Insecure bool + ServiceName string + Version string + InstanceID string + Endpoint string + Environment string + Namespace string + Insecure bool + TLSCAPath string + TLSCertificatePath string + TLSPrivateKeyPath string + TLSServerName string } type Runtime struct { @@ -54,13 +61,17 @@ type Runtime struct { func ConfigFromEnvironment(serviceName, instanceID string) Config { resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES")) return Config{ - ServiceName: serviceName, - Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), - InstanceID: instanceID, - Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), - Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), - Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), - Insecure: true, + ServiceName: serviceName, + Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), + InstanceID: instanceID, + Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), + Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), + Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), + Insecure: strings.EqualFold(strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_INSECURE")), "true"), + TLSCAPath: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_CERTIFICATE")), + TLSCertificatePath: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE")), + TLSPrivateKeyPath: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_CLIENT_KEY")), + TLSServerName: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_SERVER_NAME")), } } @@ -187,6 +198,12 @@ func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, e } if config.Insecure { options = append(options, otlptracegrpc.WithInsecure()) + } else { + transportCredentials, err := telemetryCredentials(config) + if err != nil { + return nil, err + } + options = append(options, otlptracegrpc.WithTLSCredentials(transportCredentials)) } exporter, err := otlptracegrpc.New(ctx, options...) if err != nil { @@ -204,6 +221,12 @@ func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Expo } if config.Insecure { options = append(options, otlpmetricgrpc.WithInsecure()) + } else { + transportCredentials, err := telemetryCredentials(config) + if err != nil { + return nil, err + } + options = append(options, otlpmetricgrpc.WithTLSCredentials(transportCredentials)) } exporter, err := otlpmetricgrpc.New(ctx, options...) if err != nil { @@ -221,6 +244,12 @@ func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, } if config.Insecure { options = append(options, otlploggrpc.WithInsecure()) + } else { + transportCredentials, err := telemetryCredentials(config) + if err != nil { + return nil, err + } + options = append(options, otlploggrpc.WithTLSCredentials(transportCredentials)) } exporter, err := otlploggrpc.New(ctx, options...) if err != nil { @@ -229,6 +258,17 @@ func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, return exporter, nil } +func telemetryCredentials(config Config) (credentials.TransportCredentials, error) { + tlsConfiguration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{ + CAPath: config.TLSCAPath, CertificatePath: config.TLSCertificatePath, + PrivateKeyPath: config.TLSPrivateKeyPath, ServerName: config.TLSServerName, + }) + if err != nil { + return nil, fmt.Errorf("load OTLP TLS configuration: %w", err) + } + return credentials.NewTLS(tlsConfiguration), nil +} + func (r *Runtime) Logger() *slog.Logger { return r.logger } -- 2.54.0 From ee547e5523724ecc2ea1b2578072b6c429114de2 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 6 Nov 2026 21:14:49 +0300 Subject: [PATCH 3/5] ops: configure PKI and scoped OpenBao identities --- .dockerignore | 1 + .gitignore | 2 + Dockerfile | 24 +- compose.yaml | 802 +++++++++++++++--- infra/minio/bootstrap.sh | 113 ++- infra/minio/client.Dockerfile | 4 + infra/openbao/README.md | 46 +- infra/openbao/agent/api-secrets.ctmpl | 4 + infra/openbao/agent/api.hcl | 44 + infra/openbao/agent/worker-secrets.ctmpl | 4 + infra/openbao/agent/worker.hcl | 44 + infra/openbao/config.hcl | 20 + infra/openbao/install-identity.sh | 21 + infra/openbao/policies/messenger-api.hcl | 32 + infra/openbao/policies/messenger-worker.hcl | 32 + infra/otel/collector.yaml | 25 +- infra/otel/gateway.yaml | 133 +++ infra/pki/bootstrap.sh | 158 ++++ infra/pki/install.sh | 141 +++ infra/pki/templates/api-client.cnf | 6 + infra/pki/templates/api.cnf | 10 + infra/pki/templates/ca.cnf | 12 + infra/pki/templates/collector-client.cnf | 6 + infra/pki/templates/dispatcher-client.cnf | 6 + infra/pki/templates/forbidden-client.cnf | 9 + infra/pki/templates/grafana-client.cnf | 6 + infra/pki/templates/grafana-server.cnf | 11 + infra/pki/templates/ingress.cnf | 11 + infra/pki/templates/load-balancer.cnf | 9 + infra/pki/templates/migrate-client.cnf | 6 + infra/pki/templates/minio.cnf | 6 + infra/pki/templates/openbao.cnf | 11 + infra/pki/templates/otel-gateway.cnf | 6 + infra/pki/templates/pipeline-metrics.cnf | 6 + infra/pki/templates/postgres-admin-client.cnf | 6 + infra/pki/templates/postgres-exporter.cnf | 6 + infra/pki/templates/postgres.cnf | 6 + infra/pki/templates/prometheus.cnf | 4 + infra/pki/templates/reconciler-client.cnf | 6 + infra/pki/templates/redis-broker.cnf | 6 + infra/pki/templates/redis-exporter.cnf | 6 + infra/pki/templates/redis-gateway-client.cnf | 6 + infra/pki/templates/redis-gateway.cnf | 6 + infra/pki/templates/telemetry-gateway.cnf | 6 + infra/pki/templates/valkey-exporter.cnf | 6 + infra/pki/templates/valkey.cnf | 6 + infra/pki/templates/worker-client.cnf | 6 + infra/postgres/bootstrap-roles.sh | 38 + infra/postgres/grants.sql | 25 + infra/postgres/pg_hba.conf | 15 + infra/postgres/postgresql.conf | 11 + infra/postgres/roles.sql | 38 + infra/redis/bootstrap-acl.sh | 66 ++ infra/redis/gateway.yaml | 136 +++ infra/redis/redis-broker.conf | 17 + infra/redis/valkey.conf | 16 + infra/security/test-dependency-matrix.sh | 199 +++++ lab6.env.example | 93 -- lab7.env.example | 121 +++ ...essenger-lab6.json => messenger-lab7.json} | 4 +- .../datasources/observability.yml | 18 +- monitoring/prometheus.yml | 23 +- nginx/nginx.conf | 20 +- nginx/upstream.conf | 2 +- 64 files changed, 2454 insertions(+), 235 deletions(-) mode change 100644 => 100755 infra/minio/bootstrap.sh create mode 100644 infra/openbao/agent/api-secrets.ctmpl create mode 100644 infra/openbao/agent/api.hcl create mode 100644 infra/openbao/agent/worker-secrets.ctmpl create mode 100644 infra/openbao/agent/worker.hcl create mode 100644 infra/openbao/config.hcl create mode 100644 infra/openbao/install-identity.sh create mode 100644 infra/openbao/policies/messenger-api.hcl create mode 100644 infra/openbao/policies/messenger-worker.hcl create mode 100644 infra/otel/gateway.yaml create mode 100755 infra/pki/bootstrap.sh create mode 100644 infra/pki/install.sh create mode 100644 infra/pki/templates/api-client.cnf create mode 100644 infra/pki/templates/api.cnf create mode 100644 infra/pki/templates/ca.cnf create mode 100644 infra/pki/templates/collector-client.cnf create mode 100644 infra/pki/templates/dispatcher-client.cnf create mode 100644 infra/pki/templates/forbidden-client.cnf create mode 100644 infra/pki/templates/grafana-client.cnf create mode 100644 infra/pki/templates/grafana-server.cnf create mode 100644 infra/pki/templates/ingress.cnf create mode 100644 infra/pki/templates/load-balancer.cnf create mode 100644 infra/pki/templates/migrate-client.cnf create mode 100644 infra/pki/templates/minio.cnf create mode 100644 infra/pki/templates/openbao.cnf create mode 100644 infra/pki/templates/otel-gateway.cnf create mode 100644 infra/pki/templates/pipeline-metrics.cnf create mode 100644 infra/pki/templates/postgres-admin-client.cnf create mode 100644 infra/pki/templates/postgres-exporter.cnf create mode 100644 infra/pki/templates/postgres.cnf create mode 100644 infra/pki/templates/prometheus.cnf create mode 100644 infra/pki/templates/reconciler-client.cnf create mode 100644 infra/pki/templates/redis-broker.cnf create mode 100644 infra/pki/templates/redis-exporter.cnf create mode 100644 infra/pki/templates/redis-gateway-client.cnf create mode 100644 infra/pki/templates/redis-gateway.cnf create mode 100644 infra/pki/templates/telemetry-gateway.cnf create mode 100644 infra/pki/templates/valkey-exporter.cnf create mode 100644 infra/pki/templates/valkey.cnf create mode 100644 infra/pki/templates/worker-client.cnf create mode 100755 infra/postgres/bootstrap-roles.sh create mode 100644 infra/postgres/grants.sql create mode 100644 infra/postgres/pg_hba.conf create mode 100644 infra/postgres/postgresql.conf create mode 100644 infra/postgres/roles.sql create mode 100755 infra/redis/bootstrap-acl.sh create mode 100644 infra/redis/gateway.yaml create mode 100644 infra/redis/redis-broker.conf create mode 100644 infra/redis/valkey.conf create mode 100755 infra/security/test-dependency-matrix.sh delete mode 100644 lab6.env.example create mode 100644 lab7.env.example rename monitoring/grafana/dashboards/{messenger-lab6.json => messenger-lab7.json} (99%) diff --git a/.dockerignore b/.dockerignore index ddd2c73..59cb48d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ .env .env.* !.env.example +.runtime secrets certs volumes diff --git a/.gitignore b/.gitignore index 7913498..d7b71d6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env .env.* !.env.example +.runtime/ secrets/ certs/private/ *.key @@ -32,3 +33,4 @@ coverage/ # Inspect git diff --cached: ignore rules are not a secret scanner. evidence/runtime/ +.runtime/ diff --git a/Dockerfile b/Dockerfile index ab37655..324c2f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,23 @@ -FROM golang:1.26.5-alpine3.24 AS build +FROM alpine:3.24.2 AS security-test + +RUN apk add --no-cache curl openssl +WORKDIR /src +COPY scripts/test-tls-matrix.sh /src/scripts/test-tls-matrix.sh +CMD ["/bin/sh", "/src/scripts/test-tls-matrix.sh"] + +FROM golang:1.26.10-alpine3.24 AS build + +ARG GOPROXY=https://proxy.golang.org WORKDIR /src ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB +RUN apk add --no-cache git COPY go.mod go.sum ./ -RUN go mod download +RUN GOPROXY="$GOPROXY" go mod download COPY . . RUN go build -p=1 -o /out/messenger ./cmd/messenger RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment +RUN go build -p=1 -o /out/mtlsproxy ./cmd/mtlsproxy FROM build AS test @@ -27,19 +38,22 @@ FROM grafana/grafana:13.2.4 AS grafana COPY monitoring/grafana/provisioning /etc/grafana/provisioning COPY monitoring/grafana/dashboards /etc/grafana/dashboards -FROM nginx:1.30.5-alpine AS loadbalancer +FROM nginx:1.30.6-alpine AS loadbalancer + +RUN apk upgrade --no-cache COPY nginx/nginx.conf /etc/nginx/nginx.conf COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf FROM alpine:3.24.2 -RUN addgroup -S app && adduser -S -G app -u 10001 app +RUN apk upgrade --no-cache && addgroup -S app && adduser -S -G app -u 10001 app COPY --from=build --chown=app:app /out/messenger /app/messenger COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment +COPY --from=build --chown=app:app /out/mtlsproxy /app/mtlsproxy USER app WORKDIR /app HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \ - CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 + CMD wget -q -O /dev/null "http://127.0.0.1:8082/health/ready" || exit 1 CMD ["/app/messenger"] diff --git a/compose.yaml b/compose.yaml index d21c2eb..fa8ba0c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,17 +7,98 @@ x-messenger-config-volume: &messenger-config-volume x-telemetry-environment: &telemetry-environment OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL} + OTEL_EXPORTER_OTLP_INSECURE: "false" + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/security/internal-ca/ca.crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/security/client/tls.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/security/client/tls.key + OTEL_EXPORTER_OTLP_SERVER_NAME: otel-gateway OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION} +x-app-hardening: &app-hardening + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + services: + security-bootstrap: + image: alpine:3.24.2 + entrypoint: ["/bin/sh", "/install.sh"] + volumes: + - ./.runtime/security:/source:ro + - ./infra/pki/install.sh:/install.sh:ro + - api_security:/target/api + - nginx_security:/target/nginx + - prometheus_security:/target/prometheus + - client_security:/target/client + - postgres_security:/target/postgres + - valkey_security:/target/valkey + - redis_broker_security:/target/redis-broker + - redis_gateway_security:/target/redis-gateway + - minio_security:/target/minio + - otel_gateway_security:/target/otel-gateway + - dispatcher_security:/target/dispatcher + - worker_security:/target/worker + - migrate_security:/target/migrate + - reconciler_security:/target/reconciler + - postgres_bootstrap_security:/target/postgres-bootstrap + - postgres_exporter_security:/target/postgres-exporter + - valkey_exporter_security:/target/valkey-exporter + - redis_exporter_security:/target/redis-exporter + - s3_bootstrap_security:/target/s3-bootstrap + - openbao_security:/target/openbao + - openbao_agent_security:/target/openbao-agent + - telemetry_gateway_security:/target/telemetry-gateway + - collector_security:/target/collector + - grafana_security:/target/grafana + healthcheck: + disable: true + restart: "no" + network_mode: none + + acl-bootstrap: + image: alpine:3.24.2 + entrypoint: ["/bin/sh", "-c"] + command: + - | + /bootstrap-acl.sh valkey /target/valkey/valkey/users.acl + /bootstrap-acl.sh broker /target/redis/redis-broker/users.acl + chown 999:999 /target/valkey/valkey/users.acl /target/redis/redis-broker/users.acl + environment: + VALKEY_API_PASSWORD: ${VALKEY_API_PASSWORD} + VALKEY_EXPORTER_PASSWORD: ${VALKEY_EXPORTER_PASSWORD} + VALKEY_HEALTH_PASSWORD: ${VALKEY_HEALTH_PASSWORD} + REDIS_DISPATCHER_PASSWORD: ${REDIS_DISPATCHER_PASSWORD} + REDIS_WORKER_PASSWORD: ${REDIS_WORKER_PASSWORD} + REDIS_EXPORTER_PASSWORD: ${REDIS_EXPORTER_PASSWORD} + REDIS_HEALTH_PASSWORD: ${REDIS_HEALTH_PASSWORD} + volumes: + - ./infra/redis/bootstrap-acl.sh:/bootstrap-acl.sh:ro + - valkey_security:/target/valkey + - redis_broker_security:/target/redis + depends_on: + security-bootstrap: + condition: service_completed_successfully + healthcheck: + disable: true + restart: "no" + network_mode: none + postgres: image: postgres:17.6-alpine3.22 environment: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"] volumes: - postgres_data:/var/lib/postgresql/data + - postgres_security:/run/security:ro + - ./infra/postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro + - ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] interval: 5s @@ -26,50 +107,73 @@ services: cpus: ${POSTGRES_CPUS} mem_limit: ${POSTGRES_MEMORY} restart: unless-stopped + depends_on: + security-bootstrap: + condition: service_completed_successfully networks: [data] valkey: image: valkey/valkey:8.1.4-alpine - command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"] + command: ["valkey-server", "/etc/valkey/valkey.conf"] environment: - VALKEY_PASSWORD: ${VALKEY_PASSWORD} + VALKEY_HEALTH_PASSWORD: ${VALKEY_HEALTH_PASSWORD} volumes: - valkey_data:/data + - valkey_security:/run/security:ro + - ./infra/redis/valkey.conf:/etc/valkey/valkey.conf:ro healthcheck: - test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"] + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -s /tmp/valkey.sock --user valkey_health -a \"$$VALKEY_HEALTH_PASSWORD\" ping | grep -q PONG"] interval: 5s timeout: 3s retries: 20 cpus: ${VALKEY_CPUS} mem_limit: ${VALKEY_MEMORY} restart: unless-stopped - networks: [data] + depends_on: + acl-bootstrap: + condition: service_completed_successfully + networks: [data_backend] redis-broker: image: redis:8.2.2-alpine - command: - - redis-server - - --appendonly - - "yes" - - --appendfsync - - always - - --maxmemory-policy - - noeviction - - --requirepass - - ${REDIS_BROKER_PASSWORD} + command: ["redis-server", "/etc/redis/redis.conf"] environment: - REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_HEALTH_PASSWORD: ${REDIS_HEALTH_PASSWORD} volumes: - redis_broker_data:/data + - redis_broker_security:/run/security:ro + - ./infra/redis/redis-broker.conf:/etc/redis/redis.conf:ro healthcheck: - test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"] + test: ["CMD-SHELL", "redis-cli --no-auth-warning -s /tmp/redis.sock --user redis_health -a \"$$REDIS_HEALTH_PASSWORD\" ping | grep -q PONG"] interval: 5s timeout: 3s retries: 20 cpus: ${REDIS_BROKER_CPUS} mem_limit: ${REDIS_BROKER_MEMORY} restart: unless-stopped - networks: [data] + depends_on: + acl-bootstrap: + condition: service_completed_successfully + networks: [data_backend] + + redis-gateway: + image: envoyproxy/envoy:v1.35.2 + command: ["-c", "/etc/envoy/gateway.yaml", "--log-level", "warning"] + volumes: + - ./infra/redis/gateway.yaml:/etc/envoy/gateway.yaml:ro + - redis_gateway_security:/run/security:ro + expose: ["16379", "26379", "36379"] + depends_on: + valkey: + condition: service_healthy + redis-broker: + condition: service_healthy + healthcheck: + disable: true + cpus: ${REDIS_GATEWAY_CPUS} + mem_limit: ${REDIS_GATEWAY_MEMORY} + restart: unless-stopped + networks: [data, data_backend, secure_clients] s3: build: @@ -78,24 +182,29 @@ services: args: MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z - command: ["server", "/data", "--address", ":9000"] + command: ["server", "/data", "--address", ":9000", "--certs-dir", "/run/security/minio"] environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} MINIO_BROWSER: "off" MINIO_PROMETHEUS_AUTH_TYPE: public + SSL_CERT_FILE: /run/security/internal-ca/ca.crt volumes: - s3_data:/data + - minio_security:/run/security:ro expose: - "9000" healthcheck: - test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] + test: ["CMD", "wget", "-q", "-O", "/dev/null", "https://s3:9000/minio/health/ready"] interval: 5s timeout: 3s retries: 20 cpus: ${S3_CPUS} mem_limit: ${S3_MEMORY} restart: unless-stopped + depends_on: + security-bootstrap: + condition: service_completed_successfully networks: [data] s3-bootstrap: @@ -110,8 +219,17 @@ services: MINIO_ROOT_USER: ${MINIO_ROOT_USER} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} S3_BUCKET: ${S3_BUCKET} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_API_ACCESS_KEY: ${S3_API_ACCESS_KEY} + S3_API_SECRET_KEY: ${S3_API_SECRET_KEY} + S3_WORKER_ACCESS_KEY: ${S3_WORKER_ACCESS_KEY} + S3_WORKER_SECRET_KEY: ${S3_WORKER_SECRET_KEY} + S3_RECONCILER_ACCESS_KEY: ${S3_RECONCILER_ACCESS_KEY} + S3_RECONCILER_SECRET_KEY: ${S3_RECONCILER_SECRET_KEY} + SSL_CERT_FILE: /run/security/internal-ca/ca.crt + MC_CONFIG_DIR: /tmp/mc-bootstrap + volumes: + - s3_bootstrap_security:/run/security:ro + - ./infra/minio/bootstrap.sh:/usr/local/bin/bootstrap.sh:ro depends_on: s3: condition: service_healthy @@ -122,54 +240,229 @@ services: restart: "no" networks: [data] - migrate: - image: ${API_IMAGE:-mtusi-messenger:lab6} - command: ["/app/messenger", "migrate"] - environment: - MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml - DATABASE_URL: ${DATABASE_URL} + postgres-roles: + image: postgres:17.6-alpine3.22 + command: ["/bootstrap/bootstrap-roles.sh", "create"] + environment: &postgres-role-environment + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_API_PASSWORD: ${POSTGRES_API_PASSWORD} + POSTGRES_DISPATCHER_PASSWORD: ${POSTGRES_DISPATCHER_PASSWORD} + POSTGRES_WORKER_PASSWORD: ${POSTGRES_WORKER_PASSWORD} + POSTGRES_RECONCILER_PASSWORD: ${POSTGRES_RECONCILER_PASSWORD} + POSTGRES_EXPORTER_PASSWORD: ${POSTGRES_EXPORTER_PASSWORD} + POSTGRES_MIGRATE_PASSWORD: ${POSTGRES_MIGRATE_PASSWORD} volumes: - - *messenger-config-volume + - ./infra/postgres:/bootstrap:ro + - postgres_bootstrap_security:/run/security:ro depends_on: postgres: condition: service_healthy healthcheck: disable: true + restart: "no" + networks: [data] + + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/messenger", "migrate"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_MIGRATE_URL} + volumes: + - *messenger-config-volume + - migrate_security:/run/security:ro + depends_on: + postgres-roles: + condition: service_completed_successfully + healthcheck: + disable: true cpus: ${MIGRATE_CPUS} mem_limit: ${MIGRATE_MEMORY} restart: "no" networks: [data] + postgres-grants: + image: postgres:17.6-alpine3.22 + command: ["/bootstrap/bootstrap-roles.sh", "grant"] + environment: *postgres-role-environment + volumes: + - ./infra/postgres:/bootstrap:ro + - postgres_bootstrap_security:/run/security:ro + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + restart: "no" + networks: [data] + + openbao-permissions: + image: alpine:3.24.2 + command: ["/bin/sh", "-c", "chown -R 100:100 /openbao/data && chown -R 10001:10001 /secrets/api /secrets/worker"] + volumes: + - openbao_data:/openbao/data + - openbao_api_secrets:/secrets/api + - openbao_worker_secrets:/secrets/worker + depends_on: + security-bootstrap: + condition: service_completed_successfully + healthcheck: + disable: true + restart: "no" + network_mode: none + + openbao: + image: openbao/openbao:2.7.1 + command: ["server", "-config=/etc/openbao/config.hcl"] + environment: + BAO_ENABLE_FILE_PERMISSIONS_CHECK: "false" + volumes: + - ./infra/openbao/config.hcl:/etc/openbao/config.hcl:ro + - openbao_data:/openbao/data + - openbao_security:/run/security:ro + ports: + - "${OPENBAO_BIND_ADDRESS}:8200" + depends_on: + openbao-permissions: + condition: service_completed_successfully + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + tmpfs: + - /tmp:rw,noexec,nosuid,size=16m + - /run/openbao:rw,noexec,nosuid,size=8m + cpus: ${OPENBAO_CPUS} + mem_limit: ${OPENBAO_MEMORY} + restart: unless-stopped + networks: [secrets, edge] + + openbao-cli: + profiles: ["tools"] + image: openbao/openbao:2.7.1 + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/security/openbao/ca.crt + BAO_TOKEN: ${BAO_TOKEN:-} + volumes: + - openbao_agent_security:/run/security:ro + depends_on: + - openbao + networks: [secrets] + + openbao-identity-install: + profiles: ["tools"] + image: alpine:3.24.2 + entrypoint: ["/bin/sh", "/install-identity.sh"] + volumes: + - ./infra/openbao/install-identity.sh:/install-identity.sh:ro + - ./.runtime/openbao/approle:/source:ro + - openbao_api_identity:/identities/api + - openbao_worker_identity:/identities/worker + network_mode: none + restart: "no" + + openbao-agent-api: + image: openbao/openbao:2.7.1 + user: "10001:10001" + command: ["agent", "-config=/etc/openbao-agent/agent.hcl"] + volumes: + - ./infra/openbao/agent/api.hcl:/etc/openbao-agent/agent.hcl:ro + - ./infra/openbao/agent/api-secrets.ctmpl:/etc/openbao-agent/api-secrets.ctmpl:ro + - openbao_api_identity:/run/openbao-identity:ro + - openbao_api_secrets:/run/openbao-secrets + - openbao_agent_security:/run/security:ro + depends_on: + openbao-permissions: + condition: service_completed_successfully + openbao: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "test -s /run/openbao-secrets/secrets.json && BAO_ADDR=https://openbao:8200 BAO_CACERT=/run/security/openbao/ca.crt BAO_TOKEN=$$(cat /run/openbao-agent/token) bao token lookup >/dev/null 2>&1"] + interval: 2s + timeout: 1s + retries: 60 + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: ["ALL"] + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + - /run/openbao-agent:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700 + restart: unless-stopped + networks: [secrets] + + openbao-agent-worker: + image: openbao/openbao:2.7.1 + user: "10001:10001" + command: ["agent", "-config=/etc/openbao-agent/agent.hcl"] + volumes: + - ./infra/openbao/agent/worker.hcl:/etc/openbao-agent/agent.hcl:ro + - ./infra/openbao/agent/worker-secrets.ctmpl:/etc/openbao-agent/worker-secrets.ctmpl:ro + - openbao_worker_identity:/run/openbao-identity:ro + - openbao_worker_secrets:/run/openbao-secrets + - openbao_agent_security:/run/security:ro + depends_on: + openbao-permissions: + condition: service_completed_successfully + openbao: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "test -s /run/openbao-secrets/secrets.json && BAO_ADDR=https://openbao:8200 BAO_CACERT=/run/security/openbao/ca.crt BAO_TOKEN=$$(cat /run/openbao-agent/token) bao token lookup >/dev/null 2>&1"] + interval: 2s + timeout: 1s + retries: 60 + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: ["ALL"] + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + - /run/openbao-agent:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700 + restart: unless-stopped + networks: [secrets] + api: + <<: *app-hardening build: . - image: ${API_IMAGE:-mtusi-messenger:lab6} + image: ${API_IMAGE:-mtusi-messenger:lab7} environment: <<: *telemetry-environment OTEL_SERVICE_NAME: messenger-api MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} - DATABASE_URL: ${DATABASE_URL} - CACHE_URL: ${CACHE_URL} + APP_HEALTH_ADDRESS: ${APP_HEALTH_ADDRESS} + MESSENGER_SECRETS_FILE: /run/openbao-secrets/secrets.json SWAGGER_PORT: ${SWAGGER_PORT} - S3_ENDPOINT: ${S3_ENDPOINT} + S3_ENDPOINT: https://${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_SECRET_KEY: ${S3_SECRET_KEY} volumes: - *messenger-config-volume + - api_security:/run/security:ro + - openbao_api_secrets:/run/openbao-secrets:ro expose: - - "8080" + - "8443" + - "9091" depends_on: + security-bootstrap: + condition: service_completed_successfully migrate: condition: service_completed_successfully - valkey: - condition: service_healthy + postgres-grants: + condition: service_completed_successfully + redis-gateway: + condition: service_started s3-bootstrap: condition: service_completed_successfully + openbao-agent-api: + condition: service_healthy healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8082/health/ready || exit 1"] interval: 2s timeout: 1s start_period: 5s @@ -180,28 +473,32 @@ services: mem_limit: ${API_MEMORY} stop_grace_period: 8s restart: unless-stopped - networks: [application, data] + networks: [application, data, monitoring] dispatcher: - image: ${API_IMAGE:-mtusi-messenger:lab6} + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} command: ["/app/messenger", "dispatch"] environment: <<: *telemetry-environment OTEL_SERVICE_NAME: messenger-dispatcher MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml - DATABASE_URL: ${DATABASE_URL} - BROKER_URL: ${BROKER_URL} + DATABASE_URL: ${DATABASE_DISPATCHER_URL} + BROKER_URL: ${BROKER_DISPATCHER_URL} volumes: - *messenger-config-volume + - dispatcher_security:/run/security:ro expose: - "9091" depends_on: migrate: condition: service_completed_successfully - redis-broker: - condition: service_healthy + postgres-grants: + condition: service_completed_successfully + redis-gateway: + condition: service_started healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9092/health/live || exit 1"] interval: 5s timeout: 2s retries: 20 @@ -211,32 +508,36 @@ services: networks: [data, monitoring] worker: - image: ${API_IMAGE:-mtusi-messenger:lab6} + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} command: ["/app/messenger", "worker"] environment: <<: *telemetry-environment OTEL_SERVICE_NAME: messenger-worker MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml - DATABASE_URL: ${DATABASE_URL} - S3_ENDPOINT: ${S3_ENDPOINT} + MESSENGER_SECRETS_FILE: /run/openbao-secrets/secrets.json + S3_ENDPOINT: https://${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_SECRET_KEY: ${S3_SECRET_KEY} - BROKER_URL: ${BROKER_URL} volumes: - *messenger-config-volume + - worker_security:/run/security:ro + - openbao_worker_secrets:/run/openbao-secrets:ro expose: - "9091" depends_on: migrate: condition: service_completed_successfully - redis-broker: - condition: service_healthy + postgres-grants: + condition: service_completed_successfully + redis-gateway: + condition: service_started s3-bootstrap: condition: service_completed_successfully + openbao-agent-worker: + condition: service_healthy healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9092/health/live || exit 1"] interval: 5s timeout: 2s retries: 20 @@ -246,14 +547,43 @@ services: restart: unless-stopped networks: [data, monitoring] + reconciler: + profiles: ["maintenance"] + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/messenger", "reconcile-attachments", "--grace-period=1h", "--page-size=100"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_RECONCILER_URL} + S3_ENDPOINT: https://${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_RECONCILER_ACCESS_KEY} + S3_SECRET_KEY: ${S3_RECONCILER_SECRET_KEY} + volumes: + - *messenger-config-volume + - reconciler_security:/run/security:ro + depends_on: + postgres-grants: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + disable: true + networks: [data] + nginx: build: context: . target: loadbalancer - image: ${NGINX_IMAGE:-mtusi-nginx:lab6} + image: ${NGINX_IMAGE:-mtusi-nginx:lab7} ports: - - "${API_BIND_ADDRESS}:8080" + - "${API_BIND_ADDRESS}:8443" + volumes: + - nginx_security:/run/security:ro depends_on: + security-bootstrap: + condition: service_completed_successfully api: condition: service_healthy healthcheck: @@ -266,11 +596,62 @@ services: restart: unless-stopped networks: [edge, application, monitoring] + security-tests: + profiles: ["security"] + build: + context: . + target: security-test + environment: + PUBLIC_TLS_HOST: nginx + PUBLIC_TLS_PORT: "8443" + API_TLS_ADDRESS: api:8443 + METRICS_TLS_ADDRESS: api:9091 + SECURITY_DIR: /src/.runtime/security + volumes: + - ./.runtime/security:/src/.runtime/security:ro + depends_on: + nginx: + condition: service_healthy + networks: [edge, application] + + dependency-tests: + profiles: ["security"] + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + user: "0:0" + entrypoint: ["/bin/sh", "/test-dependency-matrix.sh"] + environment: + SECURITY_DIR: /run/security-source + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_API_PASSWORD: ${POSTGRES_API_PASSWORD} + POSTGRES_WORKER_PASSWORD: ${POSTGRES_WORKER_PASSWORD} + POSTGRES_DISPATCHER_PASSWORD: ${POSTGRES_DISPATCHER_PASSWORD} + VALKEY_API_PASSWORD: ${VALKEY_API_PASSWORD} + REDIS_DISPATCHER_PASSWORD: ${REDIS_DISPATCHER_PASSWORD} + REDIS_WORKER_PASSWORD: ${REDIS_WORKER_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + S3_API_ACCESS_KEY: ${S3_API_ACCESS_KEY} + S3_API_SECRET_KEY: ${S3_API_SECRET_KEY} + S3_WORKER_ACCESS_KEY: ${S3_WORKER_ACCESS_KEY} + S3_WORKER_SECRET_KEY: ${S3_WORKER_SECRET_KEY} + volumes: + - ./infra/security/test-dependency-matrix.sh:/test-dependency-matrix.sh:ro + - ./.runtime/security:/run/security-source:ro + depends_on: + api: + condition: service_healthy + dispatcher: + condition: service_healthy + worker: + condition: service_healthy + otel-gateway: + condition: service_started + networks: [data, data_backend, secure_clients] + swagger: build: context: . target: swagger - image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6} + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab7} ports: - "${SWAGGER_BIND_ADDRESS}:8080" depends_on: @@ -287,32 +668,40 @@ services: context: . target: test environment: - TEST_DATABASE_URL: ${DATABASE_URL} + TEST_DATABASE_URL: postgres://messenger_migrate:${POSTGRES_MIGRATE_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/database-security/internal-ca/ca.crt&sslcert=/run/database-security/client/tls.crt&sslkey=/run/database-security/client/tls.key TEST_CACHE_URL: ${CACHE_URL} + TEST_CACHE_TLS_CA_PATH: /run/cache-security/internal-ca/ca.crt + TEST_CACHE_TLS_CERTIFICATE_PATH: /run/cache-security/client/tls.crt + TEST_CACHE_TLS_PRIVATE_KEY_PATH: /run/cache-security/client/tls.key + TEST_CACHE_TLS_SERVER_NAME: redis-gateway + volumes: + - migrate_security:/run/database-security:ro + - api_security:/run/cache-security:ro depends_on: - migrate: + postgres-grants: condition: service_completed_successfully - valkey: - condition: service_healthy + redis-gateway: + condition: service_started cpus: ${TEST_CPUS} mem_limit: ${TEST_MEMORY} networks: [data] lab5-experiment: profiles: ["experiments"] - image: ${API_IMAGE:-mtusi-messenger:lab6} + image: ${API_IMAGE:-mtusi-messenger:lab7} command: ["/app/lab5experiment"] environment: MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml - DATABASE_URL: ${DATABASE_URL} - S3_ENDPOINT: ${S3_ENDPOINT} + DATABASE_URL: ${DATABASE_WORKER_URL} + S3_ENDPOINT: https://${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_SECRET_KEY: ${S3_SECRET_KEY} - BROKER_URL: ${BROKER_URL} + S3_ACCESS_KEY: ${S3_WORKER_ACCESS_KEY} + S3_SECRET_KEY: ${S3_WORKER_SECRET_KEY} + BROKER_URL: ${BROKER_WORKER_URL} volumes: - *messenger-config-volume + - worker_security:/run/security:ro depends_on: migrate: condition: service_completed_successfully @@ -327,16 +716,15 @@ services: postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 environment: - DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable - DATA_SOURCE_USER: ${POSTGRES_USER} - DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + DATA_SOURCE_NAME: postgres://messenger_exporter:${POSTGRES_EXPORTER_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key PG_EXPORTER_COLLECTION_TIMEOUT: 5s PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml volumes: - ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro + - postgres_exporter_security:/run/security:ro depends_on: - postgres: - condition: service_healthy + postgres-grants: + condition: service_completed_successfully cpus: ${EXPORTER_CPUS} mem_limit: ${EXPORTER_MEMORY} restart: unless-stopped @@ -345,31 +733,43 @@ services: valkey-exporter: image: oliver006/redis_exporter:v1.89.0-alpine environment: - REDIS_ADDR: redis://valkey:6379 - REDIS_USER: default - REDIS_PASSWORD: ${VALKEY_PASSWORD} + REDIS_ADDR: rediss://valkey:6379 + REDIS_USER: valkey_exporter + REDIS_PASSWORD: ${VALKEY_EXPORTER_PASSWORD} + REDIS_EXPORTER_TLS_CLIENT_KEY_FILE: /run/security/client/tls.key + REDIS_EXPORTER_TLS_CLIENT_CERT_FILE: /run/security/client/tls.crt + REDIS_EXPORTER_TLS_CA_CERT_FILE: /run/security/internal-ca/ca.crt + REDIS_EXPORTER_TLS_SERVER_NAME: valkey + volumes: + - valkey_exporter_security:/run/security:ro depends_on: valkey: condition: service_healthy cpus: ${EXPORTER_CPUS} mem_limit: ${EXPORTER_MEMORY} restart: unless-stopped - networks: [data, monitoring] + networks: [data_backend, monitoring] redis-broker-exporter: image: oliver006/redis_exporter:v1.89.0-alpine environment: - REDIS_ADDR: redis://redis-broker:6379 - REDIS_USER: default - REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_ADDR: rediss://redis-broker:6379 + REDIS_USER: redis_exporter + REDIS_PASSWORD: ${REDIS_EXPORTER_PASSWORD} + REDIS_EXPORTER_TLS_CLIENT_KEY_FILE: /run/security/client/tls.key + REDIS_EXPORTER_TLS_CLIENT_CERT_FILE: /run/security/client/tls.crt + REDIS_EXPORTER_TLS_CA_CERT_FILE: /run/security/internal-ca/ca.crt + REDIS_EXPORTER_TLS_SERVER_NAME: redis-broker REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed + volumes: + - redis_exporter_security:/run/security:ro depends_on: redis-broker: condition: service_healthy cpus: ${EXPORTER_CPUS} mem_limit: ${EXPORTER_MEMORY} restart: unless-stopped - networks: [data, monitoring] + networks: [data_backend, monitoring] nginx-exporter: image: nginx/nginx-prometheus-exporter:1.5.1 @@ -386,7 +786,7 @@ services: build: context: . target: prometheus - image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6} + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab7} command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus @@ -394,6 +794,7 @@ services: - --web.enable-remote-write-receiver volumes: - prometheus_data:/prometheus + - prometheus_security:/run/security:ro ports: - "${PROMETHEUS_BIND_ADDRESS}:9090" healthcheck: @@ -402,49 +803,71 @@ services: timeout: 3s retries: 20 depends_on: - - api - - nginx-exporter - - postgres-exporter - - valkey-exporter - - redis-broker-exporter - - dispatcher - - worker - - s3 - - otel-collector - - loki - - tempo + security-bootstrap: + condition: service_completed_successfully + api: + condition: service_healthy + nginx-exporter: + condition: service_started + postgres-exporter: + condition: service_started + valkey-exporter: + condition: service_started + redis-broker-exporter: + condition: service_started + dispatcher: + condition: service_healthy + worker: + condition: service_healthy + s3: + condition: service_healthy + otel-collector: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} restart: unless-stopped - networks: [edge, application, data, monitoring] + networks: [application, data, monitoring, telemetry_storage] grafana: build: context: . target: grafana - image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6} + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab7} environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} GF_USERS_ALLOW_SIGN_UP: "false" GF_EXPLORE_ENABLED: "true" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/security/grafana-server.crt + GF_SERVER_CERT_KEY: /run/security/grafana-server.key + GF_SERVER_DOMAIN: localhost + SSL_CERT_FILE: /run/security/internal-ca/ca.crt volumes: - grafana_data:/var/lib/grafana + - grafana_security:/run/security:ro ports: - "${GRAFANA_BIND_ADDRESS}:3000" depends_on: - - prometheus - - loki - - tempo + prometheus-grafana-gateway: + condition: service_started + loki-grafana-gateway: + condition: service_started + tempo-grafana-gateway: + condition: service_started healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"] + test: ["CMD-SHELL", "wget -q -O /dev/null https://localhost:3000/api/health || exit 1"] interval: 5s timeout: 3s retries: 20 cpus: ${GRAFANA_CPUS} mem_limit: ${GRAFANA_MEMORY} restart: unless-stopped - networks: [edge, monitoring] + networks: [edge, secure_clients] loki: image: grafana/loki:3.7.9 @@ -452,8 +875,6 @@ services: volumes: - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro - loki_data:/loki - ports: - - "${LOKI_BIND_ADDRESS}:3100" healthcheck: test: ["CMD", "/usr/bin/loki", "-version"] interval: 10s @@ -462,7 +883,7 @@ services: cpus: ${LOKI_CPUS} mem_limit: ${LOKI_MEMORY} restart: unless-stopped - networks: [edge, monitoring] + networks: [telemetry_storage] tempo: image: grafana/tempo:2.10.10 @@ -470,8 +891,6 @@ services: volumes: - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo_data:/var/tempo - ports: - - "${TEMPO_BIND_ADDRESS}:3200" healthcheck: test: ["CMD", "/tempo", "-version"] interval: 10s @@ -480,13 +899,145 @@ services: cpus: ${TEMPO_CPUS} mem_limit: ${TEMPO_MEMORY} restart: unless-stopped - networks: [edge, monitoring] + networks: [telemetry_storage] + + otel-gateway: + image: envoyproxy/envoy:v1.35.2 + command: ["-c", "/etc/envoy/gateway.yaml", "--log-level", "warning"] + volumes: + - ./infra/otel/gateway.yaml:/etc/envoy/gateway.yaml:ro + - otel_gateway_security:/run/security:ro + expose: ["4317", "4318"] + depends_on: + security-bootstrap: + condition: service_completed_successfully + otel-collector: + condition: service_healthy + cpus: ${OTEL_GATEWAY_CPUS} + mem_limit: ${OTEL_GATEWAY_MEMORY} + restart: unless-stopped + networks: [data, telemetry_backend] + + tempo-collector-gateway: + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/mtlsproxy"] + environment: + PROXY_LISTEN_ADDRESS: 0.0.0.0:8443 + PROXY_TARGET_ADDRESS: tempo:4317 + PROXY_TLS_CA_PATH: /run/security/internal-ca/ca.crt + PROXY_TLS_CERTIFICATE_PATH: /run/security/telemetry-gateway/tls.crt + PROXY_TLS_PRIVATE_KEY_PATH: /run/security/telemetry-gateway/tls.key + PROXY_ALLOWED_CLIENT_URI: spiffe://mtusi/lab7/collector + PROXY_ALPN_PROTOCOLS: h2 + volumes: + - telemetry_gateway_security:/run/security:ro + depends_on: + security-bootstrap: + condition: service_completed_successfully + tempo: + condition: service_healthy + healthcheck: + disable: true + restart: unless-stopped + networks: [secure_clients, telemetry_storage] + + loki-collector-gateway: + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/mtlsproxy"] + environment: + PROXY_LISTEN_ADDRESS: 0.0.0.0:8443 + PROXY_TARGET_ADDRESS: loki:3100 + PROXY_TLS_CA_PATH: /run/security/internal-ca/ca.crt + PROXY_TLS_CERTIFICATE_PATH: /run/security/telemetry-gateway/tls.crt + PROXY_TLS_PRIVATE_KEY_PATH: /run/security/telemetry-gateway/tls.key + PROXY_ALLOWED_CLIENT_URI: spiffe://mtusi/lab7/collector + PROXY_ALPN_PROTOCOLS: http/1.1 + volumes: + - telemetry_gateway_security:/run/security:ro + depends_on: + security-bootstrap: + condition: service_completed_successfully + loki: + condition: service_healthy + healthcheck: + disable: true + restart: unless-stopped + networks: [secure_clients, telemetry_storage] + + prometheus-grafana-gateway: + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/mtlsproxy"] + environment: + PROXY_LISTEN_ADDRESS: 0.0.0.0:8443 + PROXY_TARGET_ADDRESS: prometheus:9090 + PROXY_TLS_CA_PATH: /run/security/internal-ca/ca.crt + PROXY_TLS_CERTIFICATE_PATH: /run/security/telemetry-gateway/tls.crt + PROXY_TLS_PRIVATE_KEY_PATH: /run/security/telemetry-gateway/tls.key + PROXY_ALLOWED_CLIENT_URI: spiffe://mtusi/lab7/grafana + PROXY_ALPN_PROTOCOLS: http/1.1 + volumes: + - telemetry_gateway_security:/run/security:ro + depends_on: + prometheus: + condition: service_healthy + healthcheck: + disable: true + restart: unless-stopped + networks: [secure_clients, telemetry_storage] + + loki-grafana-gateway: + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/mtlsproxy"] + environment: + PROXY_LISTEN_ADDRESS: 0.0.0.0:8443 + PROXY_TARGET_ADDRESS: loki:3100 + PROXY_TLS_CA_PATH: /run/security/internal-ca/ca.crt + PROXY_TLS_CERTIFICATE_PATH: /run/security/telemetry-gateway/tls.crt + PROXY_TLS_PRIVATE_KEY_PATH: /run/security/telemetry-gateway/tls.key + PROXY_ALLOWED_CLIENT_URI: spiffe://mtusi/lab7/grafana + PROXY_ALPN_PROTOCOLS: http/1.1 + volumes: + - telemetry_gateway_security:/run/security:ro + depends_on: + loki: + condition: service_healthy + healthcheck: + disable: true + restart: unless-stopped + networks: [secure_clients, telemetry_storage] + + tempo-grafana-gateway: + <<: *app-hardening + image: ${API_IMAGE:-mtusi-messenger:lab7} + command: ["/app/mtlsproxy"] + environment: + PROXY_LISTEN_ADDRESS: 0.0.0.0:8443 + PROXY_TARGET_ADDRESS: tempo:3200 + PROXY_TLS_CA_PATH: /run/security/internal-ca/ca.crt + PROXY_TLS_CERTIFICATE_PATH: /run/security/telemetry-gateway/tls.crt + PROXY_TLS_PRIVATE_KEY_PATH: /run/security/telemetry-gateway/tls.key + PROXY_ALLOWED_CLIENT_URI: spiffe://mtusi/lab7/grafana + PROXY_ALPN_PROTOCOLS: http/1.1 + volumes: + - telemetry_gateway_security:/run/security:ro + depends_on: + tempo: + condition: service_healthy + healthcheck: + disable: true + restart: unless-stopped + networks: [secure_clients, telemetry_storage] otel-collector: image: otel/opentelemetry-collector-contrib:0.160.0 command: ["--config=/etc/otelcol-contrib/collector.yaml"] volumes: - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + - collector_security:/run/security:ro expose: - "4317" - "4318" @@ -504,9 +1055,38 @@ services: cpus: ${OTEL_COLLECTOR_CPUS} mem_limit: ${OTEL_COLLECTOR_MEMORY} restart: unless-stopped - networks: [application, data, monitoring] + networks: [telemetry_backend, secure_clients, monitoring] volumes: + api_security: + nginx_security: + prometheus_security: + client_security: + postgres_security: + valkey_security: + redis_broker_security: + redis_gateway_security: + minio_security: + otel_gateway_security: + dispatcher_security: + worker_security: + migrate_security: + reconciler_security: + postgres_bootstrap_security: + postgres_exporter_security: + valkey_exporter_security: + redis_exporter_security: + s3_bootstrap_security: + openbao_security: + openbao_agent_security: + telemetry_gateway_security: + collector_security: + grafana_security: + openbao_api_secrets: + openbao_worker_secrets: + openbao_api_identity: + openbao_worker_identity: + openbao_data: postgres_data: valkey_data: redis_broker_data: @@ -522,5 +1102,15 @@ networks: internal: true data: internal: true + data_backend: + internal: true monitoring: internal: true + telemetry_backend: + internal: true + telemetry_storage: + internal: true + secure_clients: + internal: true + secrets: + internal: true diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh old mode 100644 new mode 100755 index 68a2888..2df3ceb --- a/infra/minio/bootstrap.sh +++ b/infra/minio/bootstrap.sh @@ -2,35 +2,124 @@ set -eu -alias_name=bootstrap -policy_name=messenger-attachments -policy_file=/tmp/messenger-attachments-policy.json +: "${MINIO_ROOT_USER:?MINIO_ROOT_USER is required}" +: "${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}" +: "${S3_BUCKET:?S3_BUCKET is required}" +: "${S3_API_ACCESS_KEY:?S3_API_ACCESS_KEY is required}" +: "${S3_API_SECRET_KEY:?S3_API_SECRET_KEY is required}" +: "${S3_WORKER_ACCESS_KEY:?S3_WORKER_ACCESS_KEY is required}" +: "${S3_WORKER_SECRET_KEY:?S3_WORKER_SECRET_KEY is required}" +: "${S3_RECONCILER_ACCESS_KEY:?S3_RECONCILER_ACCESS_KEY is required}" +: "${S3_RECONCILER_SECRET_KEY:?S3_RECONCILER_SECRET_KEY is required}" -mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +export SSL_CERT_FILE=${SSL_CERT_FILE:-/run/security/internal-ca/ca.crt} + +alias_name=bootstrap +policy_dir=/tmp/minio-policies +mkdir -p "$policy_dir" + +mc alias set "$alias_name" https://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" mc mb --ignore-existing "$alias_name/$S3_BUCKET" mc anonymous set none "$alias_name/$S3_BUCKET" -cat >"$policy_file" <"$policy_dir/messenger-api.json" <"$policy_dir/messenger-worker.json" </dev/null +cat >"$policy_dir/messenger-reconciler.json" </dev/null diff --git a/infra/minio/client.Dockerfile b/infra/minio/client.Dockerfile index fe8ee0a..1be182f 100644 --- a/infra/minio/client.Dockerfile +++ b/infra/minio/client.Dockerfile @@ -10,7 +10,11 @@ RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ FROM alpine:3.22.2 +RUN apk add --no-cache curl openssl postgresql17-client redis && \ + addgroup -g 10001 messenger && \ + adduser -D -H -u 10001 -G messenger messenger COPY --from=build /out/mc /usr/local/bin/mc COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh +USER 10001:10001 ENTRYPOINT ["mc"] diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..2b75368 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -1,7 +1,45 @@ -# OpenBao — реализовать конфигурацию и политики +# OpenBao -Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key. +OpenBao работает вне dev-mode с single-node integrated Raft. Данные сохраняются в `/openbao/data`, API принимает только TLS 1.3 на `8200`, cluster transport использует `8201`. Сертификат и ключ передаются в `/run/security/openbao`, а не входят в образ. -Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. +## Идентичности -Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. +`messenger-api` и `messenger-worker` — разные AppRole. Каждая роль: + +- читает только свой `secret/data/messenger/` в KV v2; +- получает renewable token на 15 минут с максимальным TTL 1 час; +- использует короткоживущий `SecretID` с TTL один час; +- не получает policy `default`, доступ к чужому secret или управление policies. + +OpenBao Agent читает `RoleID` и короткоживущий `SecretID`, renew-ит token и повторно аутентифицируется после его истечения или рестарта. SecretID действует один час и хранится в operator-controlled каталоге с правами `0600`; после истечения оператор выдаёт новый через `openbao-issue-approle.sh`. KV v2 secret рендерится в защищённый Docker volume `/run/openbao-secrets/secrets.json` с правами `0600`; токен Agent хранится отдельно в tmpfs `/run/openbao-agent`. Старый rendered secret не считается признаком здоровья: healthcheck также проверяет token и процесс Agent. + +## Bootstrap оператора + +Все команды выполняются с проверкой CA и hostname. `BAO_ADDR` по умолчанию равен `https://localhost:8200`, `BAO_CACERT` — `.runtime/security/openbao/ca.crt`. + +```sh +scripts/openbao-init.sh +scripts/openbao-unseal.sh +chmod 0600 .runtime/openbao/seed/api.json .runtime/openbao/seed/worker.json +scripts/openbao-bootstrap.sh \ + .runtime/openbao/seed/api.json \ + .runtime/openbao/seed/worker.json +scripts/openbao-test-policy.sh +``` + +Seed-файлы — непустые JSON-объекты с фактическими credentials сервиса. Скрипт не выводит значения в консоль. `operator init` создаёт три unseal share с threshold 2 и сохраняет единственную локальную копию в `.runtime/openbao/operator/init.json` с правами `0600`. Этот каталог игнорируется Git. + +Для учебного локального стенда оператор хранит init-файл вне контейнера. В реальном окружении shares нужно разделить между операторами либо защитить KMS/HSM, а root token отозвать после настройки recovery-доступа. API, worker и Agent не получают root token или unseal shares. + +После обычного перезапуска данные остаются в Raft volume, но процесс остаётся sealed до ручного `scripts/openbao-unseal.sh`. Перезапуск API и worker не требует unseal. + +## Ротация + +```sh +chmod 0600 .runtime/openbao/seed/api-next.json +scripts/openbao-rotate.sh api .runtime/openbao/seed/api-next.json +``` + +Ротация создаёт новую версию KV v2 и новый короткоживущий `SecretID`; предыдущий accessor отзывается. Agent перечитывает статический secret не позднее пяти минут. Уже выданный token может жить до своего TTL, поэтому при компрометации дополнительно отзывают его accessor или отключают роль и перезапускают Agent с новой идентичностью. + +`scripts/openbao-test-policy.sh` выдаёт временные AppRole credentials, проверяет разрешённое чтение собственного пути, запрет чужого пути, записи и управления policies, а также renewal. Временные tokens отзываются при завершении. diff --git a/infra/openbao/agent/api-secrets.ctmpl b/infra/openbao/agent/api-secrets.ctmpl new file mode 100644 index 0000000..3e8892b --- /dev/null +++ b/infra/openbao/agent/api-secrets.ctmpl @@ -0,0 +1,4 @@ +{{- with secret "secret/data/messenger/api" -}} +{{ .Data.data | toJSON }} +{{- end -}} + diff --git a/infra/openbao/agent/api.hcl b/infra/openbao/agent/api.hcl new file mode 100644 index 0000000..964d993 --- /dev/null +++ b/infra/openbao/agent/api.hcl @@ -0,0 +1,44 @@ +pid_file = "/run/openbao-agent/agent.pid" + +vault { + address = "https://openbao:8200" + ca_cert = "/run/security/openbao/ca.crt" + tls_server_name = "openbao" +} + +auto_auth { + method { + type = "approle" + mount_path = "auth/approle" + + config = { + role_id_file_path = "/run/openbao-identity/role-id" + secret_id_file_path = "/run/openbao-identity/secret-id" + remove_secret_id_file_after_reading = false + } + } + + sink { + type = "file" + + config = { + path = "/run/openbao-agent/token" + mode = 384 + } + } +} + +template_config { + exit_on_retry_failure = true + static_secret_render_interval = "5m" +} + +template { + source = "/etc/openbao-agent/api-secrets.ctmpl" + destination = "/run/openbao-secrets/secrets.json" + create_dest_dirs = true + error_on_missing_key = true + perms = "0600" + backup = false +} + diff --git a/infra/openbao/agent/worker-secrets.ctmpl b/infra/openbao/agent/worker-secrets.ctmpl new file mode 100644 index 0000000..9e2f36e --- /dev/null +++ b/infra/openbao/agent/worker-secrets.ctmpl @@ -0,0 +1,4 @@ +{{- with secret "secret/data/messenger/worker" -}} +{{ .Data.data | toJSON }} +{{- end -}} + diff --git a/infra/openbao/agent/worker.hcl b/infra/openbao/agent/worker.hcl new file mode 100644 index 0000000..66567de --- /dev/null +++ b/infra/openbao/agent/worker.hcl @@ -0,0 +1,44 @@ +pid_file = "/run/openbao-agent/agent.pid" + +vault { + address = "https://openbao:8200" + ca_cert = "/run/security/openbao/ca.crt" + tls_server_name = "openbao" +} + +auto_auth { + method { + type = "approle" + mount_path = "auth/approle" + + config = { + role_id_file_path = "/run/openbao-identity/role-id" + secret_id_file_path = "/run/openbao-identity/secret-id" + remove_secret_id_file_after_reading = false + } + } + + sink { + type = "file" + + config = { + path = "/run/openbao-agent/token" + mode = 384 + } + } +} + +template_config { + exit_on_retry_failure = true + static_secret_render_interval = "5m" +} + +template { + source = "/etc/openbao-agent/worker-secrets.ctmpl" + destination = "/run/openbao-secrets/secrets.json" + create_dest_dirs = true + error_on_missing_key = true + perms = "0600" + backup = false +} + diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl new file mode 100644 index 0000000..939484f --- /dev/null +++ b/infra/openbao/config.hcl @@ -0,0 +1,20 @@ +ui = false + +api_addr = "https://openbao:8200" +cluster_addr = "https://openbao:8201" + +storage "raft" { + path = "/openbao/data" + node_id = "messenger-openbao-1" +} + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + tls_cert_file = "/run/security/openbao/tls.crt" + tls_key_file = "/run/security/openbao/tls.key" + tls_min_version = "tls13" +} + +log_level = "info" + diff --git a/infra/openbao/install-identity.sh b/infra/openbao/install-identity.sh new file mode 100644 index 0000000..6c61ff2 --- /dev/null +++ b/infra/openbao/install-identity.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -eu +umask 077 + +case "${SERVICE:-}" in + api|worker) ;; + *) echo "SERVICE must be api or worker" >&2; exit 2 ;; +esac + +source_dir="/source/$SERVICE" +target_dir="/identities/$SERVICE" +mkdir -p "$target_dir" +for file in role-id secret-id secret-id-accessor; do + cp "$source_dir/$file" "$target_dir/$file.next" + chown 10001:10001 "$target_dir/$file.next" + chmod 0600 "$target_dir/$file.next" + mv "$target_dir/$file.next" "$target_dir/$file" +done +chown 10001:10001 "$target_dir" +chmod 0700 "$target_dir" diff --git a/infra/openbao/policies/messenger-api.hcl b/infra/openbao/policies/messenger-api.hcl new file mode 100644 index 0000000..aaebd73 --- /dev/null +++ b/infra/openbao/policies/messenger-api.hcl @@ -0,0 +1,32 @@ +path "secret/data/messenger/api" { + capabilities = ["read"] +} + +path "secret/metadata/messenger/api" { + capabilities = ["read"] +} + +path "secret/data/messenger/worker" { + capabilities = ["deny"] +} + +path "secret/metadata/messenger/worker" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +path "sys/policies/*" { + capabilities = ["deny"] +} + diff --git a/infra/openbao/policies/messenger-worker.hcl b/infra/openbao/policies/messenger-worker.hcl new file mode 100644 index 0000000..8357e16 --- /dev/null +++ b/infra/openbao/policies/messenger-worker.hcl @@ -0,0 +1,32 @@ +path "secret/data/messenger/worker" { + capabilities = ["read"] +} + +path "secret/metadata/messenger/worker" { + capabilities = ["read"] +} + +path "secret/data/messenger/api" { + capabilities = ["deny"] +} + +path "secret/metadata/messenger/api" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +path "sys/policies/*" { + capabilities = ["deny"] +} + diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index e912195..6f7c90f 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -6,9 +6,9 @@ receivers: otlp: protocols: grpc: - endpoint: 0.0.0.0:4317 + endpoint: 0.0.0.0:14317 http: - endpoint: 0.0.0.0:4318 + endpoint: 0.0.0.0:14318 processors: memory_limiter: @@ -29,9 +29,12 @@ exporters: - service.version - deployment.environment.name otlp_grpc/tempo: - endpoint: tempo:4317 + endpoint: tempo-collector-gateway:8443 tls: - insecure: true + ca_file: /run/security/internal-ca/ca.crt + cert_file: /run/security/client/tls.crt + key_file: /run/security/client/tls.key + server_name_override: telemetry-gateway sending_queue: enabled: true num_consumers: 2 @@ -42,7 +45,12 @@ exporters: max_interval: 5s max_elapsed_time: 30s otlp_http/loki: - endpoint: http://loki:3100/otlp + endpoint: https://loki-collector-gateway:8443/otlp + tls: + ca_file: /run/security/internal-ca/ca.crt + cert_file: /run/security/client/tls.crt + key_file: /run/security/client/tls.key + server_name_override: telemetry-gateway sending_queue: enabled: true num_consumers: 2 @@ -58,6 +66,13 @@ service: telemetry: logs: level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 pipelines: metrics: receivers: [otlp] diff --git a/infra/otel/gateway.yaml b/infra/otel/gateway.yaml new file mode 100644 index 0000000..7871fa4 --- /dev/null +++ b/infra/otel/gateway.yaml @@ -0,0 +1,133 @@ +static_resources: + listeners: + - name: otlp_grpc_mtls + address: + socket_address: + address: 0.0.0.0 + port_value: 4317 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: otlp_grpc + codec_type: HTTP2 + route_config: + name: otlp_grpc + virtual_hosts: + - name: collector + domains: ["*"] + routes: + - match: {prefix: "/"} + route: + cluster: collector_grpc + timeout: 0s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_3 + tls_maximum_protocol_version: TLSv1_3 + alpn_protocols: [h2] + tls_certificates: + - certificate_chain: {filename: /run/security/otel-gateway/tls.crt} + private_key: {filename: /run/security/otel-gateway/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: URI + matcher: + safe_regex: + regex: "^spiffe://mtusi/lab7/(api|dispatcher|worker)$" + + - name: otlp_http_mtls + address: + socket_address: + address: 0.0.0.0 + port_value: 4318 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: otlp_http + codec_type: AUTO + route_config: + name: otlp_http + virtual_hosts: + - name: collector + domains: ["*"] + routes: + - match: {prefix: "/"} + route: + cluster: collector_http + timeout: 30s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_3 + tls_maximum_protocol_version: TLSv1_3 + alpn_protocols: [h2, http/1.1] + tls_certificates: + - certificate_chain: {filename: /run/security/otel-gateway/tls.crt} + private_key: {filename: /run/security/otel-gateway/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: URI + matcher: + safe_regex: + regex: "^spiffe://mtusi/lab7/(api|dispatcher|worker)$" + + clusters: + - name: collector_grpc + connect_timeout: 2s + type: STRICT_DNS + load_assignment: + cluster_name: collector_grpc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: otel-collector + port_value: 14317 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + + - name: collector_http + connect_timeout: 2s + type: STRICT_DNS + load_assignment: + cluster_name: collector_http + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: otel-collector + port_value: 14318 + +admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 diff --git a/infra/pki/bootstrap.sh b/infra/pki/bootstrap.sh new file mode 100755 index 0000000..297809a --- /dev/null +++ b/infra/pki/bootstrap.sh @@ -0,0 +1,158 @@ +#!/bin/sh + +set -eu + +umask 077 + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd) +security_dir="$repo_dir/.runtime/security" +staging_dir="$security_dir/.bootstrap.$$" + +cleanup() { + rm -rf "$staging_dir" +} + +trap cleanup EXIT HUP INT TERM + +if [ "${1:-}" = "--force" ]; then + force=1 +elif [ "$#" -eq 0 ]; then + force=0 +else + echo "usage: $0 [--force]" >&2 + exit 2 +fi + +if [ -d "$security_dir/public-ca" ] || [ -d "$security_dir/internal-ca" ]; then + if [ "$force" -ne 1 ]; then + echo "security material already exists; use --force to replace it" >&2 + exit 1 + fi +fi + +mkdir -p "$security_dir" "$staging_dir" +chmod 0700 "$security_dir" "$staging_dir" + +create_ca() { + name=$1 + common_name=$2 + directory="$staging_dir/$name" + + mkdir -p "$directory" + openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/ca.key" + openssl req -new -x509 -sha256 -days 3650 \ + -key "$directory/ca.key" \ + -out "$directory/ca.crt" \ + -subj "/CN=$common_name" \ + -config "$script_dir/templates/ca.cnf" \ + -extensions v3_ca +} + +create_leaf() { + name=$1 + common_name=$2 + issuer=$3 + template=$4 + directory="$staging_dir/$name" + issuer_directory="$staging_dir/$issuer" + + mkdir -p "$directory" + openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/tls.key" + openssl req -new -sha256 \ + -key "$directory/tls.key" \ + -out "$directory/tls.csr" \ + -subj "/CN=$common_name" + openssl x509 -req -sha256 -days 30 \ + -in "$directory/tls.csr" \ + -CA "$issuer_directory/ca.crt" \ + -CAkey "$issuer_directory/ca.key" \ + -CAcreateserial \ + -out "$directory/tls.crt" \ + -extfile "$script_dir/templates/$template" + rm "$directory/tls.csr" +} + +create_ca public-ca "MTUSI Lab 7 Public CA" +create_ca internal-ca "MTUSI Lab 7 Internal CA" +create_ca foreign-ca "MTUSI Lab 7 Foreign CA" + +create_leaf ingress localhost public-ca ingress.cnf +create_leaf api api internal-ca api.cnf +create_leaf load-balancer load-balancer internal-ca load-balancer.cnf +create_leaf prometheus prometheus internal-ca prometheus.cnf +create_leaf postgres postgres internal-ca postgres.cnf +create_leaf valkey valkey internal-ca valkey.cnf +create_leaf redis-broker redis-broker internal-ca redis-broker.cnf +create_leaf redis-gateway redis-gateway internal-ca redis-gateway.cnf +create_leaf redis-gateway-client redis-gateway internal-ca redis-gateway-client.cnf +create_leaf minio s3 internal-ca minio.cnf +create_leaf otel-gateway otel-gateway internal-ca otel-gateway.cnf +create_leaf openbao openbao internal-ca openbao.cnf +cp "$staging_dir/internal-ca/ca.crt" "$staging_dir/openbao/ca.crt" +create_leaf telemetry-gateway telemetry-gateway internal-ca telemetry-gateway.cnf +create_leaf collector-client collector internal-ca collector-client.cnf +create_leaf grafana-client grafana internal-ca grafana-client.cnf +create_leaf grafana-server grafana internal-ca grafana-server.cnf +create_leaf pipeline-metrics pipeline-metrics internal-ca pipeline-metrics.cnf +create_leaf api-client messenger_api internal-ca api-client.cnf +create_leaf dispatcher-client messenger_dispatcher internal-ca dispatcher-client.cnf +create_leaf worker-client messenger_worker internal-ca worker-client.cnf +create_leaf migrate-client messenger_migrate internal-ca migrate-client.cnf +create_leaf reconciler-client messenger_reconciler internal-ca reconciler-client.cnf +create_leaf postgres-exporter messenger_exporter internal-ca postgres-exporter.cnf +create_leaf postgres-admin-client messenger internal-ca postgres-admin-client.cnf +create_leaf valkey-exporter valkey_exporter internal-ca valkey-exporter.cnf +create_leaf redis-exporter redis_exporter internal-ca redis-exporter.cnf +create_leaf forbidden-client worker internal-ca forbidden-client.cnf +create_leaf foreign-client load-balancer foreign-ca load-balancer.cnf + +openssl verify -purpose sslserver -verify_hostname localhost \ + -CAfile "$staging_dir/public-ca/ca.crt" "$staging_dir/ingress/tls.crt" >/dev/null +openssl verify -purpose sslserver -verify_hostname api \ + -CAfile "$staging_dir/internal-ca/ca.crt" "$staging_dir/api/tls.crt" >/dev/null +for server in postgres valkey redis-broker redis-gateway minio otel-gateway openbao telemetry-gateway grafana-server pipeline-metrics; do + case "$server" in + minio) hostname=s3 ;; + grafana-server) hostname=grafana ;; + *) hostname=$server ;; + esac + openssl verify -purpose sslserver -verify_hostname "$hostname" \ + -CAfile "$staging_dir/internal-ca/ca.crt" "$staging_dir/$server/tls.crt" >/dev/null +done +openssl verify -purpose sslclient \ + -CAfile "$staging_dir/internal-ca/ca.crt" \ + "$staging_dir/load-balancer/tls.crt" \ + "$staging_dir/prometheus/tls.crt" \ + "$staging_dir/api-client/tls.crt" \ + "$staging_dir/dispatcher-client/tls.crt" \ + "$staging_dir/worker-client/tls.crt" \ + "$staging_dir/migrate-client/tls.crt" \ + "$staging_dir/reconciler-client/tls.crt" \ + "$staging_dir/postgres-exporter/tls.crt" \ + "$staging_dir/postgres-admin-client/tls.crt" \ + "$staging_dir/valkey-exporter/tls.crt" \ + "$staging_dir/redis-exporter/tls.crt" \ + "$staging_dir/redis-gateway-client/tls.crt" \ + "$staging_dir/collector-client/tls.crt" \ + "$staging_dir/grafana-client/tls.crt" \ + "$staging_dir/forbidden-client/tls.crt" >/dev/null +openssl verify -purpose sslclient \ + -CAfile "$staging_dir/foreign-ca/ca.crt" "$staging_dir/foreign-client/tls.crt" >/dev/null + +find "$staging_dir" -type d -exec chmod 0700 {} \; +find "$staging_dir" -type f -exec chmod 0600 {} \; + +for directory in \ + public-ca internal-ca foreign-ca ingress api load-balancer prometheus \ + postgres valkey redis-broker redis-gateway redis-gateway-client minio otel-gateway openbao \ + telemetry-gateway collector-client grafana-client grafana-server \ + pipeline-metrics \ + api-client dispatcher-client worker-client migrate-client reconciler-client \ + postgres-exporter postgres-admin-client valkey-exporter redis-exporter \ + forbidden-client foreign-client; do + rm -rf "$security_dir/$directory" + mv "$staging_dir/$directory" "$security_dir/$directory" +done + +echo "local PKI created in .runtime/security" diff --git a/infra/pki/install.sh b/infra/pki/install.sh new file mode 100644 index 0000000..0a96544 --- /dev/null +++ b/infra/pki/install.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +set -eu + +umask 077 + +rm -rf /target/api/internal-ca /target/api/api /target/api/client +mkdir -p /target/api/internal-ca /target/api/api /target/api/client +cp /source/internal-ca/ca.crt /target/api/internal-ca/ca.crt +cp /source/api/tls.crt /target/api/api/tls.crt +cp /source/api/tls.key /target/api/api/tls.key +cp /source/api-client/tls.crt /target/api/client/tls.crt +cp /source/api-client/tls.key /target/api/client/tls.key +chown -R 10001:10001 /target/api +find /target/api -type d -exec chmod 0700 {} \; +find /target/api -type f -exec chmod 0600 {} \; + +rm -rf /target/nginx/ingress /target/nginx/public-ca /target/nginx/internal-ca /target/nginx/load-balancer +mkdir -p /target/nginx/ingress /target/nginx/public-ca /target/nginx/internal-ca /target/nginx/load-balancer +cp /source/ingress/tls.crt /target/nginx/ingress/tls.crt +cp /source/ingress/tls.key /target/nginx/ingress/tls.key +cp /source/public-ca/ca.crt /target/nginx/public-ca/ca.crt +cp /source/internal-ca/ca.crt /target/nginx/internal-ca/ca.crt +cp /source/load-balancer/tls.crt /target/nginx/load-balancer/tls.crt +cp /source/load-balancer/tls.key /target/nginx/load-balancer/tls.key +chown -R 0:0 /target/nginx +find /target/nginx -type d -exec chmod 0700 {} \; +find /target/nginx -type f -exec chmod 0600 {} \; + +rm -rf /target/prometheus/internal-ca /target/prometheus/prometheus +mkdir -p /target/prometheus/internal-ca /target/prometheus/prometheus +cp /source/internal-ca/ca.crt /target/prometheus/internal-ca/ca.crt +cp /source/prometheus/tls.crt /target/prometheus/prometheus/tls.crt +cp /source/prometheus/tls.key /target/prometheus/prometheus/tls.key +chown -R 65534:65534 /target/prometheus +find /target/prometheus -type d -exec chmod 0700 {} \; +find /target/prometheus -type f -exec chmod 0600 {} \; + +rm -rf /target/client/public-ca +mkdir -p /target/client/public-ca +cp /source/public-ca/ca.crt /target/client/public-ca/ca.crt +chown -R 10001:10001 /target/client +find /target/client -type d -exec chmod 0700 {} \; +find /target/client -type f -exec chmod 0600 {} \; + +install_server() { + target=$1 + identity=$2 + owner=$3 + + rm -rf "/target/$target/internal-ca" "/target/$target/$identity" + mkdir -p "/target/$target/internal-ca" "/target/$target/$identity" + cp /source/internal-ca/ca.crt "/target/$target/internal-ca/ca.crt" + cp "/source/$identity/tls.crt" "/target/$target/$identity/tls.crt" + cp "/source/$identity/tls.key" "/target/$target/$identity/tls.key" + chown -R "$owner" "/target/$target" + find "/target/$target" -type d -exec chmod 0700 {} \; + find "/target/$target" -type f -exec chmod 0600 {} \; +} + +install_client() { + target=$1 + identity=$2 + owner=$3 + + rm -rf "/target/$target/internal-ca" "/target/$target/public-ca" "/target/$target/client" + mkdir -p "/target/$target/internal-ca" "/target/$target/public-ca" "/target/$target/client" + cp /source/internal-ca/ca.crt "/target/$target/internal-ca/ca.crt" + cp /source/public-ca/ca.crt "/target/$target/public-ca/ca.crt" + cp "/source/$identity/tls.crt" "/target/$target/client/tls.crt" + cp "/source/$identity/tls.key" "/target/$target/client/tls.key" + chown -R "$owner" "/target/$target" + find "/target/$target" -type d -exec chmod 0700 {} \; + find "/target/$target" -type f -exec chmod 0600 {} \; +} + +install_server postgres postgres 70:70 +install_server valkey valkey 999:999 +install_server redis-broker redis-broker 999:999 +install_server minio minio 100:101 +cp /source/minio/tls.crt /target/minio/minio/public.crt +cp /source/minio/tls.key /target/minio/minio/private.key +chown 100:101 /target/minio/minio/public.crt /target/minio/minio/private.key +chmod 0600 /target/minio/minio/public.crt /target/minio/minio/private.key +install_server otel-gateway otel-gateway 101:101 +install_server openbao openbao 100:100 +cp /source/internal-ca/ca.crt /target/openbao/openbao/ca.crt +chown 100:100 /target/openbao/openbao/ca.crt +chmod 0600 /target/openbao/openbao/ca.crt + +rm -rf /target/openbao-agent/openbao +mkdir -p /target/openbao-agent/openbao +cp /source/internal-ca/ca.crt /target/openbao-agent/openbao/ca.crt +chown -R 10001:10001 /target/openbao-agent +find /target/openbao-agent -type d -exec chmod 0700 {} \; +find /target/openbao-agent -type f -exec chmod 0600 {} \; +chmod 0644 /target/openbao-agent/openbao/ca.crt +chmod 0755 /target/openbao-agent /target/openbao-agent/openbao + +install_server telemetry-gateway telemetry-gateway 10001:10001 +install_client collector collector-client 10001:10001 +install_client grafana grafana-client 472:472 +cp /source/grafana-server/tls.crt /target/grafana/grafana-server.crt +cp /source/grafana-server/tls.key /target/grafana/grafana-server.key +chown 472:472 /target/grafana/grafana-server.crt /target/grafana/grafana-server.key +chmod 0600 /target/grafana/grafana-server.crt /target/grafana/grafana-server.key + +rm -rf /target/redis-gateway/internal-ca /target/redis-gateway/redis-gateway /target/redis-gateway/client +mkdir -p /target/redis-gateway/internal-ca /target/redis-gateway/redis-gateway /target/redis-gateway/client +cp /source/internal-ca/ca.crt /target/redis-gateway/internal-ca/ca.crt +cp /source/redis-gateway/tls.crt /target/redis-gateway/redis-gateway/tls.crt +cp /source/redis-gateway/tls.key /target/redis-gateway/redis-gateway/tls.key +cp /source/redis-gateway-client/tls.crt /target/redis-gateway/client/tls.crt +cp /source/redis-gateway-client/tls.key /target/redis-gateway/client/tls.key +chown -R 101:101 /target/redis-gateway +find /target/redis-gateway -type d -exec chmod 0700 {} \; +find /target/redis-gateway -type f -exec chmod 0600 {} \; + +install_client dispatcher dispatcher-client 10001:10001 +install_client worker worker-client 10001:10001 +for target in dispatcher worker; do + mkdir -p "/target/$target/metrics" + cp /source/pipeline-metrics/tls.crt "/target/$target/metrics/tls.crt" + cp /source/pipeline-metrics/tls.key "/target/$target/metrics/tls.key" + chown -R 10001:10001 "/target/$target/metrics" + find "/target/$target/metrics" -type d -exec chmod 0700 {} \; + find "/target/$target/metrics" -type f -exec chmod 0600 {} \; +done +install_client migrate migrate-client 10001:10001 +install_client reconciler reconciler-client 10001:10001 +install_client postgres-exporter postgres-exporter 65534:65534 +install_client postgres-bootstrap postgres-admin-client 0:0 +install_client valkey-exporter valkey-exporter 59000:59000 +install_client redis-exporter redis-exporter 59000:59000 + +rm -rf /target/s3-bootstrap/internal-ca +mkdir -p /target/s3-bootstrap/internal-ca +cp /source/internal-ca/ca.crt /target/s3-bootstrap/internal-ca/ca.crt +chown -R 10001:10001 /target/s3-bootstrap +find /target/s3-bootstrap -type d -exec chmod 0700 {} \; +find /target/s3-bootstrap -type f -exec chmod 0600 {} \; diff --git a/infra/pki/templates/api-client.cnf b/infra/pki/templates/api-client.cnf new file mode 100644 index 0000000..0184b3e --- /dev/null +++ b/infra/pki/templates/api-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/api +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/api.cnf b/infra/pki/templates/api.cnf new file mode 100644 index 0000000..db5c7c5 --- /dev/null +++ b/infra/pki/templates/api.cnf @@ -0,0 +1,10 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +DNS.1 = api +URI.1 = spiffe://mtusi/lab7/api diff --git a/infra/pki/templates/ca.cnf b/infra/pki/templates/ca.cnf new file mode 100644 index 0000000..eafaf88 --- /dev/null +++ b/infra/pki/templates/ca.cnf @@ -0,0 +1,12 @@ +[req] +distinguished_name = distinguished_name +prompt = no + +[distinguished_name] +CN = local-ca + +[v3_ca] +basicConstraints = critical,CA:TRUE,pathlen:0 +keyUsage = critical,keyCertSign,cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always diff --git a/infra/pki/templates/collector-client.cnf b/infra/pki/templates/collector-client.cnf new file mode 100644 index 0000000..98344b7 --- /dev/null +++ b/infra/pki/templates/collector-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/collector +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/dispatcher-client.cnf b/infra/pki/templates/dispatcher-client.cnf new file mode 100644 index 0000000..5b0bbfc --- /dev/null +++ b/infra/pki/templates/dispatcher-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/dispatcher +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/forbidden-client.cnf b/infra/pki/templates/forbidden-client.cnf new file mode 100644 index 0000000..2c0a192 --- /dev/null +++ b/infra/pki/templates/forbidden-client.cnf @@ -0,0 +1,9 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +URI.1 = spiffe://mtusi/lab7/worker diff --git a/infra/pki/templates/grafana-client.cnf b/infra/pki/templates/grafana-client.cnf new file mode 100644 index 0000000..b65e4e0 --- /dev/null +++ b/infra/pki/templates/grafana-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/grafana +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/grafana-server.cnf b/infra/pki/templates/grafana-server.cnf new file mode 100644 index 0000000..4d5646d --- /dev/null +++ b/infra/pki/templates/grafana-server.cnf @@ -0,0 +1,11 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +DNS.1 = grafana +DNS.2 = localhost +IP.1 = 127.0.0.1 diff --git a/infra/pki/templates/ingress.cnf b/infra/pki/templates/ingress.cnf new file mode 100644 index 0000000..0f50b42 --- /dev/null +++ b/infra/pki/templates/ingress.cnf @@ -0,0 +1,11 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +DNS.1 = localhost +DNS.2 = nginx +IP.1 = 127.0.0.1 diff --git a/infra/pki/templates/load-balancer.cnf b/infra/pki/templates/load-balancer.cnf new file mode 100644 index 0000000..6b7565b --- /dev/null +++ b/infra/pki/templates/load-balancer.cnf @@ -0,0 +1,9 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +URI.1 = spiffe://mtusi/lab7/load-balancer diff --git a/infra/pki/templates/migrate-client.cnf b/infra/pki/templates/migrate-client.cnf new file mode 100644 index 0000000..548c604 --- /dev/null +++ b/infra/pki/templates/migrate-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/migrate +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/minio.cnf b/infra/pki/templates/minio.cnf new file mode 100644 index 0000000..8d8eb9d --- /dev/null +++ b/infra/pki/templates/minio.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:s3 +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/openbao.cnf b/infra/pki/templates/openbao.cnf new file mode 100644 index 0000000..16eef4e --- /dev/null +++ b/infra/pki/templates/openbao.cnf @@ -0,0 +1,11 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @subject_alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +[subject_alt_names] +DNS.1 = openbao +DNS.2 = localhost +IP.1 = 127.0.0.1 diff --git a/infra/pki/templates/otel-gateway.cnf b/infra/pki/templates/otel-gateway.cnf new file mode 100644 index 0000000..96d68d6 --- /dev/null +++ b/infra/pki/templates/otel-gateway.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:otel-gateway +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/pipeline-metrics.cnf b/infra/pki/templates/pipeline-metrics.cnf new file mode 100644 index 0000000..e31124f --- /dev/null +++ b/infra/pki/templates/pipeline-metrics.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:pipeline-metrics +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/postgres-admin-client.cnf b/infra/pki/templates/postgres-admin-client.cnf new file mode 100644 index 0000000..35c28f0 --- /dev/null +++ b/infra/pki/templates/postgres-admin-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/postgres-bootstrap +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/postgres-exporter.cnf b/infra/pki/templates/postgres-exporter.cnf new file mode 100644 index 0000000..ac68c22 --- /dev/null +++ b/infra/pki/templates/postgres-exporter.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/postgres-exporter +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/postgres.cnf b/infra/pki/templates/postgres.cnf new file mode 100644 index 0000000..2c52161 --- /dev/null +++ b/infra/pki/templates/postgres.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:postgres +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/prometheus.cnf b/infra/pki/templates/prometheus.cnf new file mode 100644 index 0000000..8d924dc --- /dev/null +++ b/infra/pki/templates/prometheus.cnf @@ -0,0 +1,4 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/prometheus diff --git a/infra/pki/templates/reconciler-client.cnf b/infra/pki/templates/reconciler-client.cnf new file mode 100644 index 0000000..d43a4e4 --- /dev/null +++ b/infra/pki/templates/reconciler-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/reconciler +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/redis-broker.cnf b/infra/pki/templates/redis-broker.cnf new file mode 100644 index 0000000..1fd32ab --- /dev/null +++ b/infra/pki/templates/redis-broker.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:redis-broker +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/redis-exporter.cnf b/infra/pki/templates/redis-exporter.cnf new file mode 100644 index 0000000..583a914 --- /dev/null +++ b/infra/pki/templates/redis-exporter.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/redis-exporter +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/redis-gateway-client.cnf b/infra/pki/templates/redis-gateway-client.cnf new file mode 100644 index 0000000..8312453 --- /dev/null +++ b/infra/pki/templates/redis-gateway-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/redis-gateway +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/redis-gateway.cnf b/infra/pki/templates/redis-gateway.cnf new file mode 100644 index 0000000..f63842f --- /dev/null +++ b/infra/pki/templates/redis-gateway.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:redis-gateway +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/telemetry-gateway.cnf b/infra/pki/templates/telemetry-gateway.cnf new file mode 100644 index 0000000..970e9a7 --- /dev/null +++ b/infra/pki/templates/telemetry-gateway.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:telemetry-gateway +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/valkey-exporter.cnf b/infra/pki/templates/valkey-exporter.cnf new file mode 100644 index 0000000..faf5ecd --- /dev/null +++ b/infra/pki/templates/valkey-exporter.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/valkey-exporter +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/valkey.cnf b/infra/pki/templates/valkey.cnf new file mode 100644 index 0000000..d6d5aa8 --- /dev/null +++ b/infra/pki/templates/valkey.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature,keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:valkey +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/pki/templates/worker-client.cnf b/infra/pki/templates/worker-client.cnf new file mode 100644 index 0000000..25af4e6 --- /dev/null +++ b/infra/pki/templates/worker-client.cnf @@ -0,0 +1,6 @@ +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = clientAuth +subjectAltName = URI:spiffe://mtusi/lab7/worker +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer diff --git a/infra/postgres/bootstrap-roles.sh b/infra/postgres/bootstrap-roles.sh new file mode 100755 index 0000000..d4c5ddc --- /dev/null +++ b/infra/postgres/bootstrap-roles.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -eu + +case "${1:-}" in + create) sql_file=/bootstrap/roles.sql ;; + grant) sql_file=/bootstrap/grants.sql ;; + *) echo "usage: $0 create|grant" >&2; exit 2 ;; +esac + +: "${POSTGRES_DB:?POSTGRES_DB is required}" +: "${POSTGRES_USER:?POSTGRES_USER is required}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}" +: "${POSTGRES_API_PASSWORD:?POSTGRES_API_PASSWORD is required}" +: "${POSTGRES_DISPATCHER_PASSWORD:?POSTGRES_DISPATCHER_PASSWORD is required}" +: "${POSTGRES_WORKER_PASSWORD:?POSTGRES_WORKER_PASSWORD is required}" +: "${POSTGRES_RECONCILER_PASSWORD:?POSTGRES_RECONCILER_PASSWORD is required}" +: "${POSTGRES_EXPORTER_PASSWORD:?POSTGRES_EXPORTER_PASSWORD is required}" +: "${POSTGRES_MIGRATE_PASSWORD:?POSTGRES_MIGRATE_PASSWORD is required}" + +export PGPASSWORD=$POSTGRES_PASSWORD +export PGSSLROOTCERT=${PGSSLROOTCERT:-/run/security/internal-ca/ca.crt} +export PGSSLCERT=${PGSSLCERT:-/run/security/client/tls.crt} +export PGSSLKEY=${PGSSLKEY:-/run/security/client/tls.key} +export PGSSLMODE=verify-full + +psql \ + --host=postgres \ + --username="$POSTGRES_USER" \ + --dbname="$POSTGRES_DB" \ + --set=ON_ERROR_STOP=1 \ + --set=api_password="$POSTGRES_API_PASSWORD" \ + --set=dispatcher_password="$POSTGRES_DISPATCHER_PASSWORD" \ + --set=worker_password="$POSTGRES_WORKER_PASSWORD" \ + --set=reconciler_password="$POSTGRES_RECONCILER_PASSWORD" \ + --set=exporter_password="$POSTGRES_EXPORTER_PASSWORD" \ + --set=migrate_password="$POSTGRES_MIGRATE_PASSWORD" \ + --file="$sql_file" diff --git a/infra/postgres/grants.sql b/infra/postgres/grants.sql new file mode 100644 index 0000000..02fca4d --- /dev/null +++ b/infra/postgres/grants.sql @@ -0,0 +1,25 @@ +REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; + +GRANT SELECT, INSERT, UPDATE, DELETE ON + users, + chats, + chat_members, + messages, + message_idempotency, + attachments, + message_attachments, + attachment_jobs, + outbox_events +TO messenger_api; +GRANT SELECT ON attachment_variants TO messenger_api; + +GRANT SELECT, UPDATE, DELETE ON outbox_events TO messenger_dispatcher; + +GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_worker; +GRANT SELECT, INSERT, UPDATE, DELETE ON attachment_variants TO messenger_worker; +GRANT INSERT ON attachment_replay_audit TO messenger_worker; +GRANT INSERT ON outbox_events TO messenger_worker; + +GRANT SELECT, UPDATE, DELETE ON attachments TO messenger_reconciler; +GRANT SELECT ON attachment_variants, message_attachments TO messenger_reconciler; +GRANT pg_monitor TO messenger_exporter; diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..33d20f3 --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,15 @@ +# Local bootstrap and health checks stay inside the PostgreSQL container. +local all all trust + +# TLS verifies the server. clientcert=verify-full additionally binds the +# certificate CN to the database role; SCRAM remains a second factor. +hostssl messenger messenger_api all scram-sha-256 clientcert=verify-full +hostssl messenger messenger_dispatcher all scram-sha-256 clientcert=verify-full +hostssl messenger messenger_worker all scram-sha-256 clientcert=verify-full +hostssl messenger messenger_reconciler all scram-sha-256 clientcert=verify-full +hostssl messenger messenger_exporter all scram-sha-256 clientcert=verify-full +hostssl messenger messenger_migrate all scram-sha-256 clientcert=verify-full +hostssl messenger messenger all scram-sha-256 clientcert=verify-full + +hostnossl all all all reject +hostssl all all all reject diff --git a/infra/postgres/postgresql.conf b/infra/postgres/postgresql.conf new file mode 100644 index 0000000..e27a93c --- /dev/null +++ b/infra/postgres/postgresql.conf @@ -0,0 +1,11 @@ +listen_addresses = '*' +port = 5432 + +ssl = on +ssl_cert_file = '/run/security/postgres/tls.crt' +ssl_key_file = '/run/security/postgres/tls.key' +ssl_ca_file = '/run/security/internal-ca/ca.crt' +ssl_min_protocol_version = 'TLSv1.3' + +hba_file = '/etc/postgresql/pg_hba.conf' +password_encryption = 'scram-sha-256' diff --git a/infra/postgres/roles.sql b/infra/postgres/roles.sql new file mode 100644 index 0000000..2d6ba99 --- /dev/null +++ b/infra/postgres/roles.sql @@ -0,0 +1,38 @@ +SELECT format('CREATE ROLE %I LOGIN', role_name) +FROM unnest(ARRAY[ + 'messenger_api', + 'messenger_dispatcher', + 'messenger_worker', + 'messenger_reconciler', + 'messenger_exporter', + 'messenger_migrate' +]) AS roles(role_name) +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = role_name) +\gexec + +ALTER ROLE messenger_api PASSWORD :'api_password'; +ALTER ROLE messenger_dispatcher PASSWORD :'dispatcher_password'; +ALTER ROLE messenger_worker PASSWORD :'worker_password'; +ALTER ROLE messenger_reconciler PASSWORD :'reconciler_password'; +ALTER ROLE messenger_exporter PASSWORD :'exporter_password'; +ALTER ROLE messenger_migrate PASSWORD :'migrate_password'; + +REVOKE CREATE ON SCHEMA public FROM PUBLIC; + +GRANT CONNECT ON DATABASE messenger TO + messenger_api, + messenger_dispatcher, + messenger_worker, + messenger_reconciler, + messenger_exporter, + messenger_migrate; +GRANT USAGE ON SCHEMA public TO + messenger_api, + messenger_dispatcher, + messenger_worker, + messenger_reconciler, + messenger_exporter; +GRANT USAGE, CREATE ON SCHEMA public TO messenger_migrate; + +ALTER DEFAULT PRIVILEGES FOR ROLE messenger_migrate IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO messenger_api; diff --git a/infra/redis/bootstrap-acl.sh b/infra/redis/bootstrap-acl.sh new file mode 100755 index 0000000..b5133c2 --- /dev/null +++ b/infra/redis/bootstrap-acl.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -eu + +usage() { + echo "usage: $0 valkey|broker OUTPUT" >&2 + exit 2 +} + +valid_secret() { + value=$1 + case "$value" in + ""|*[!A-Za-z0-9_@%+=:,./-]*) return 1 ;; + *) return 0 ;; + esac +} + +write_valkey_acl() { + : "${VALKEY_API_PASSWORD:?VALKEY_API_PASSWORD is required}" + : "${VALKEY_EXPORTER_PASSWORD:?VALKEY_EXPORTER_PASSWORD is required}" + : "${VALKEY_HEALTH_PASSWORD:?VALKEY_HEALTH_PASSWORD is required}" + valid_secret "$VALKEY_API_PASSWORD" || exit 1 + valid_secret "$VALKEY_EXPORTER_PASSWORD" || exit 1 + valid_secret "$VALKEY_HEALTH_PASSWORD" || exit 1 + + cat >"$output" <$VALKEY_API_PASSWORD ~session:* ~login-rate:* +ping +get +set +del +unlink +expire +pexpire +ttl +pttl +incr +eval +evalsha +script|load +user valkey_exporter reset on >$VALKEY_EXPORTER_PASSWORD ~* +ping +info +client +config|get +scan +dbsize +slowlog +user valkey_health reset on >$VALKEY_HEALTH_PASSWORD +ping +EOF +} + +write_broker_acl() { + : "${REDIS_DISPATCHER_PASSWORD:?REDIS_DISPATCHER_PASSWORD is required}" + : "${REDIS_WORKER_PASSWORD:?REDIS_WORKER_PASSWORD is required}" + : "${REDIS_EXPORTER_PASSWORD:?REDIS_EXPORTER_PASSWORD is required}" + : "${REDIS_HEALTH_PASSWORD:?REDIS_HEALTH_PASSWORD is required}" + valid_secret "$REDIS_DISPATCHER_PASSWORD" || exit 1 + valid_secret "$REDIS_WORKER_PASSWORD" || exit 1 + valid_secret "$REDIS_EXPORTER_PASSWORD" || exit 1 + valid_secret "$REDIS_HEALTH_PASSWORD" || exit 1 + + cat >"$output" <$REDIS_DISPATCHER_PASSWORD ~attachment-jobs +ping +xadd +user messenger_worker reset on >$REDIS_WORKER_PASSWORD ~attachment-jobs ~attachment-jobs-failed +ping +xgroup +xreadgroup +xautoclaim +xpending +xack +xdel +xadd +eval +evalsha +script|load +user redis_exporter reset on >$REDIS_EXPORTER_PASSWORD ~* +ping +info +client +config|get +scan +dbsize +slowlog +xinfo +user redis_health reset on >$REDIS_HEALTH_PASSWORD +ping +EOF +} + +[ "$#" -eq 2 ] || usage +mode=$1 +output=$2 + +umask 077 +mkdir -p "$(dirname -- "$output")" + +case "$mode" in + valkey) write_valkey_acl ;; + broker) write_broker_acl ;; + *) usage ;; +esac + +chmod 0600 "$output" diff --git a/infra/redis/gateway.yaml b/infra/redis/gateway.yaml new file mode 100644 index 0000000..93ef23e --- /dev/null +++ b/infra/redis/gateway.yaml @@ -0,0 +1,136 @@ +static_resources: + listeners: + - name: valkey_api + address: + socket_address: {address: 0.0.0.0, port_value: 16379} + filter_chains: + - filters: + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: valkey_api + cluster: valkey + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_params: {tls_minimum_protocol_version: TLSv1_3, tls_maximum_protocol_version: TLSv1_3} + tls_certificates: + - certificate_chain: {filename: /run/security/redis-gateway/tls.crt} + private_key: {filename: /run/security/redis-gateway/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: URI + matcher: {exact: "spiffe://mtusi/lab7/api"} + + - name: broker_dispatcher + address: + socket_address: {address: 0.0.0.0, port_value: 26379} + filter_chains: + - filters: + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: broker_dispatcher + cluster: redis_broker + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_params: {tls_minimum_protocol_version: TLSv1_3, tls_maximum_protocol_version: TLSv1_3} + tls_certificates: + - certificate_chain: {filename: /run/security/redis-gateway/tls.crt} + private_key: {filename: /run/security/redis-gateway/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: URI + matcher: {exact: "spiffe://mtusi/lab7/dispatcher"} + + - name: broker_worker + address: + socket_address: {address: 0.0.0.0, port_value: 36379} + filter_chains: + - filters: + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: broker_worker + cluster: redis_broker + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_params: {tls_minimum_protocol_version: TLSv1_3, tls_maximum_protocol_version: TLSv1_3} + tls_certificates: + - certificate_chain: {filename: /run/security/redis-gateway/tls.crt} + private_key: {filename: /run/security/redis-gateway/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: URI + matcher: {exact: "spiffe://mtusi/lab7/worker"} + + clusters: + - name: valkey + connect_timeout: 2s + type: STRICT_DNS + load_assignment: + cluster_name: valkey + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: {address: valkey, port_value: 6379} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: valkey + common_tls_context: + tls_params: {tls_minimum_protocol_version: TLSv1_3, tls_maximum_protocol_version: TLSv1_3} + tls_certificates: + - certificate_chain: {filename: /run/security/client/tls.crt} + private_key: {filename: /run/security/client/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: DNS + matcher: {exact: valkey} + + - name: redis_broker + connect_timeout: 2s + type: STRICT_DNS + load_assignment: + cluster_name: redis_broker + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: {address: redis-broker, port_value: 6379} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: redis-broker + common_tls_context: + tls_params: {tls_minimum_protocol_version: TLSv1_3, tls_maximum_protocol_version: TLSv1_3} + tls_certificates: + - certificate_chain: {filename: /run/security/client/tls.crt} + private_key: {filename: /run/security/client/tls.key} + validation_context: + trusted_ca: {filename: /run/security/internal-ca/ca.crt} + match_typed_subject_alt_names: + - san_type: DNS + matcher: {exact: redis-broker} + +admin: + address: + socket_address: {address: 127.0.0.1, port_value: 9901} diff --git a/infra/redis/redis-broker.conf b/infra/redis/redis-broker.conf new file mode 100644 index 0000000..4124875 --- /dev/null +++ b/infra/redis/redis-broker.conf @@ -0,0 +1,17 @@ +bind 0.0.0.0 +protected-mode yes +port 0 +tls-port 6379 +tls-cert-file /run/security/redis-broker/tls.crt +tls-key-file /run/security/redis-broker/tls.key +tls-ca-cert-file /run/security/internal-ca/ca.crt +tls-auth-clients yes +tls-protocols "TLSv1.3" +unixsocket /tmp/redis.sock +unixsocketperm 700 + +aclfile /run/security/redis-broker/users.acl +appendonly yes +appendfsync always +maxmemory-policy noeviction +dir /data diff --git a/infra/redis/valkey.conf b/infra/redis/valkey.conf new file mode 100644 index 0000000..14b2aa5 --- /dev/null +++ b/infra/redis/valkey.conf @@ -0,0 +1,16 @@ +bind 0.0.0.0 +protected-mode yes +port 0 +tls-port 6379 +tls-cert-file /run/security/valkey/tls.crt +tls-key-file /run/security/valkey/tls.key +tls-ca-cert-file /run/security/internal-ca/ca.crt +tls-auth-clients yes +tls-protocols "TLSv1.3" +unixsocket /tmp/valkey.sock +unixsocketperm 700 + +aclfile /run/security/valkey/users.acl +appendonly yes +appendfsync everysec +dir /data diff --git a/infra/security/test-dependency-matrix.sh b/infra/security/test-dependency-matrix.sh new file mode 100755 index 0000000..06c96c5 --- /dev/null +++ b/infra/security/test-dependency-matrix.sh @@ -0,0 +1,199 @@ +#!/bin/sh + +set -eu + +: "${SECURITY_DIR:=/run/security-source}" +: "${POSTGRES_HOST:=postgres}" +: "${VALKEY_HOST:=redis-gateway}" +: "${VALKEY_PORT:=16379}" +: "${REDIS_HOST:=redis-gateway}" +: "${REDIS_DISPATCHER_PORT:=26379}" +: "${REDIS_WORKER_PORT:=36379}" +: "${S3_HOST:=s3}" +: "${OTEL_HOST:=otel-gateway}" +: "${POSTGRES_DB:=messenger}" +: "${S3_BUCKET:=messenger-attachments}" + +required_commands="curl mc openssl psql redis-cli" +for command in $required_commands; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 1 + } +done + +ca=$SECURITY_DIR/internal-ca/ca.crt + +expect_failure() { + description=$1 + shift + if "$@" >/dev/null 2>&1; then + echo "expected failure: $description" >&2 + exit 1 + fi +} + +expect_redis_denied() { + description=$1 + shift + output=$("$@" 2>&1 || true) + case "$output" in + *NOPERM*|*DENIED*) ;; + *) echo "expected Redis ACL denial: $description" >&2; exit 1 ;; + esac +} + +expect_redis_allowed() { + description=$1 + shift + output=$("$@" 2>&1) || { + echo "expected Redis ACL allow: $description: $output" >&2 + exit 1 + } + case "$output" in + *NOPERM*|*DENIED*) echo "expected Redis ACL allow: $description: $output" >&2; exit 1 ;; + esac +} + +check_server() { + host=$1 + port=$2 + identity=${3:-} + client_options= + if [ -n "$identity" ]; then + client_options="-cert $SECURITY_DIR/$identity/tls.crt -key $SECURITY_DIR/$identity/tls.key" + fi + # client_options contains only paths assembled by this script. + # shellcheck disable=SC2086 + printf '' | openssl s_client $client_options \ + -connect "$host:$port" \ + -servername "$host" \ + -verify_hostname "$host" \ + -verify_return_error \ + -CAfile "$ca" >/dev/null 2>&1 +} + +check_server "$POSTGRES_HOST" 5432 api-client +check_server "$VALKEY_HOST" "$VALKEY_PORT" api-client +check_server "$REDIS_HOST" "$REDIS_WORKER_PORT" worker-client +check_server "$S3_HOST" 9000 + +: "${POSTGRES_API_PASSWORD:?POSTGRES_API_PASSWORD is required}" +: "${POSTGRES_WORKER_PASSWORD:?POSTGRES_WORKER_PASSWORD is required}" +: "${POSTGRES_DISPATCHER_PASSWORD:?POSTGRES_DISPATCHER_PASSWORD is required}" + +pg_url() { + role=$1 + cert=$2 + password=$3 + printf 'postgresql://%s:%s@%s:5432/%s?sslmode=verify-full&sslrootcert=%s&sslcert=%s&sslkey=%s' \ + "$role" "$password" "$POSTGRES_HOST" "$POSTGRES_DB" "$ca" \ + "$SECURITY_DIR/$cert/tls.crt" "$SECURITY_DIR/$cert/tls.key" +} + +api_db=$(pg_url messenger_api api-client "$POSTGRES_API_PASSWORD") +worker_db=$(pg_url messenger_worker worker-client "$POSTGRES_WORKER_PASSWORD") +dispatcher_db=$(pg_url messenger_dispatcher dispatcher-client "$POSTGRES_DISPATCHER_PASSWORD") +api_with_worker_cert=$(pg_url messenger_api worker-client "$POSTGRES_API_PASSWORD") + +psql "$api_db" --set=ON_ERROR_STOP=1 --command='SELECT count(*) FROM users' >/dev/null +psql "$worker_db" --set=ON_ERROR_STOP=1 --command='SELECT count(*) FROM attachment_jobs' >/dev/null +psql "$worker_db" --set=ON_ERROR_STOP=1 --command='UPDATE attachment_jobs SET updated_at = updated_at WHERE false' >/dev/null +psql "$dispatcher_db" --set=ON_ERROR_STOP=1 --command='SELECT count(*) FROM outbox_events' >/dev/null +expect_failure "worker reads users" psql "$worker_db" --set=ON_ERROR_STOP=1 --command='SELECT count(*) FROM users' +expect_failure "dispatcher reads attachments" psql "$dispatcher_db" --set=ON_ERROR_STOP=1 --command='SELECT count(*) FROM attachments' +expect_failure "API role uses worker certificate" psql "$api_with_worker_cert" --set=ON_ERROR_STOP=1 --command='SELECT 1' + +: "${VALKEY_API_PASSWORD:?VALKEY_API_PASSWORD is required}" +: "${REDIS_DISPATCHER_PASSWORD:?REDIS_DISPATCHER_PASSWORD is required}" +: "${REDIS_WORKER_PASSWORD:?REDIS_WORKER_PASSWORD is required}" + +expect_failure "broker without client certificate" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" PING +for identity in foreign-client api-client; do + expect_failure "broker rejects $identity" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/$identity/tls.crt" \ + --key "$SECURITY_DIR/$identity/tls.key" --user messenger_worker \ + -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" PING +done + +redis_tls() { + identity=$1 + shift + redis-cli --tls --cacert "$ca" \ + --cert "$SECURITY_DIR/$identity/tls.crt" \ + --key "$SECURITY_DIR/$identity/tls.key" "$@" +} + +REDISCLI_AUTH=$VALKEY_API_PASSWORD redis_tls api-client \ + --user messenger_api -h "$VALKEY_HOST" -p "$VALKEY_PORT" SET session:acl-test value EX 30 >/dev/null +expect_redis_denied "session identity writes an arbitrary key" env REDISCLI_AUTH="$VALKEY_API_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/api-client/tls.crt" \ + --key "$SECURITY_DIR/api-client/tls.key" --user messenger_api -h "$VALKEY_HOST" -p "$VALKEY_PORT" SET forbidden value + +REDISCLI_AUTH=$REDIS_DISPATCHER_PASSWORD redis_tls dispatcher-client \ + --user messenger_dispatcher -h "$REDIS_HOST" -p "$REDIS_DISPATCHER_PORT" XADD attachment-jobs '*' payload acl-test >/dev/null +expect_redis_denied "dispatcher consumes jobs" env REDISCLI_AUTH="$REDIS_DISPATCHER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/dispatcher-client/tls.crt" \ + --key "$SECURITY_DIR/dispatcher-client/tls.key" --user messenger_dispatcher -h "$REDIS_HOST" -p "$REDIS_DISPATCHER_PORT" \ + XREAD COUNT 1 STREAMS attachment-jobs 0 +REDISCLI_AUTH=$REDIS_WORKER_PASSWORD redis_tls worker-client \ + --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" XGROUP DESTROY attachment-jobs acl-worker-check >/dev/null 2>&1 || true +expect_redis_allowed "worker creates its consumer group" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/worker-client/tls.crt" \ + --key "$SECURITY_DIR/worker-client/tls.key" --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" \ + XGROUP CREATE attachment-jobs acl-worker-check '$' MKSTREAM +expect_redis_allowed "worker consumes its stream" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/worker-client/tls.crt" \ + --key "$SECURITY_DIR/worker-client/tls.key" --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" \ + XREADGROUP GROUP acl-worker-check acl-worker-consumer COUNT 1 STREAMS attachment-jobs '>' +expect_redis_allowed "worker acknowledges its stream" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/worker-client/tls.crt" \ + --key "$SECURITY_DIR/worker-client/tls.key" --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" \ + XACK attachment-jobs acl-worker-check 0-1 +REDISCLI_AUTH=$REDIS_WORKER_PASSWORD redis_tls worker-client \ + --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" XGROUP DESTROY attachment-jobs acl-worker-check >/dev/null +expect_redis_denied "worker accesses an unrelated stream" env REDISCLI_AUTH="$REDIS_WORKER_PASSWORD" \ + redis-cli --tls --cacert "$ca" --cert "$SECURITY_DIR/worker-client/tls.crt" \ + --key "$SECURITY_DIR/worker-client/tls.key" --user messenger_worker -h "$REDIS_HOST" -p "$REDIS_WORKER_PORT" XADD unrelated '*' payload denied + +: "${S3_API_ACCESS_KEY:?S3_API_ACCESS_KEY is required}" +: "${S3_API_SECRET_KEY:?S3_API_SECRET_KEY is required}" +: "${S3_WORKER_ACCESS_KEY:?S3_WORKER_ACCESS_KEY is required}" +: "${S3_WORKER_SECRET_KEY:?S3_WORKER_SECRET_KEY is required}" + +export SSL_CERT_FILE=$ca +export MC_CONFIG_DIR=/tmp/mc-acl-test +test_file=/tmp/acl-test-object +printf 'acl-test' >"$test_file" +mc alias set api "https://$S3_HOST:9000" "$S3_API_ACCESS_KEY" "$S3_API_SECRET_KEY" >/dev/null +mc alias set worker "https://$S3_HOST:9000" "$S3_WORKER_ACCESS_KEY" "$S3_WORKER_SECRET_KEY" >/dev/null +mc cp "$test_file" "api/$S3_BUCKET/staging/acl-test" >/dev/null +expect_failure "API writes processed objects" mc cp "$test_file" "api/$S3_BUCKET/attachments/acl-test" +mc cp "$test_file" "worker/$S3_BUCKET/attachments/acl-test" >/dev/null +expect_failure "worker writes outside attachment prefixes" mc cp "$test_file" "worker/$S3_BUCKET/private/acl-test" +mc rm "api/$S3_BUCKET/staging/acl-test" >/dev/null +mc rm "worker/$S3_BUCKET/attachments/acl-test" >/dev/null + +expect_failure "OTel without client certificate" curl --silent --show-error \ + --cacert "$ca" "https://$OTEL_HOST:4318/" +expect_failure "OTel with foreign CA client certificate" curl --silent --show-error \ + --cacert "$ca" --cert "$SECURITY_DIR/foreign-client/tls.crt" \ + --key "$SECURITY_DIR/foreign-client/tls.key" "https://$OTEL_HOST:4318/" + +curl --silent --show-error --fail-with-body \ + --cacert "$ca" \ + --cert "$SECURITY_DIR/api-client/tls.crt" \ + --key "$SECURITY_DIR/api-client/tls.key" \ + "https://$OTEL_HOST:4318/" >/dev/null 2>&1 || status=$? +case "${status:-0}" in + 0|22) ;; + *) echo "allowed OTel identity failed TLS" >&2; exit 1 ;; +esac +expect_failure "Prometheus same-CA identity reaches OTel" curl --silent --show-error \ + --cacert "$ca" \ + --cert "$SECURITY_DIR/prometheus/tls.crt" \ + --key "$SECURITY_DIR/prometheus/tls.key" \ + "https://$OTEL_HOST:4318/" + +echo "dependency TLS and least-privilege matrix passed" diff --git a/lab6.env.example b/lab6.env.example deleted file mode 100644 index ca34a40..0000000 --- a/lab6.env.example +++ /dev/null @@ -1,93 +0,0 @@ -# Изолированное имя Compose-проекта и его volumes/networks. -COMPOSE_PROJECT_NAME=mtusi2026lab6 - -# Адрес и внутренний порт HTTP-сервера. -APP_HOST=0.0.0.0 -APP_PORT=8080 -MESSENGER_CONFIG_PATH=config/messenger.yaml - -# Адреса публикации сервисов на локальной машине. -API_BIND_ADDRESS=127.0.0.1:8080 -SWAGGER_BIND_ADDRESS=127.0.0.1:8081 -PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 -GRAFANA_BIND_ADDRESS=127.0.0.1:3000 -LOKI_BIND_ADDRESS=127.0.0.1:3100 -TEMPO_BIND_ADDRESS=127.0.0.1:3200 - -# Ресурсные атрибуты и внутренний OTLP endpoint приложений. -DEPLOYMENT_ENVIRONMENT=lab -SERVICE_VERSION=lab6 -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 -OTEL_EXPORTER_OTLP_PROTOCOL=grpc - -# Число API-реплик за Nginx. -API_REPLICAS=2 - -# Учётные данные локального PostgreSQL и строка подключения API. -POSTGRES_USER=messenger -POSTGRES_DB=messenger -POSTGRES_PASSWORD=local-postgres-password -DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable - -# Пароль локального Valkey и строка подключения API. -VALKEY_PASSWORD=local-valkey-password -CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 - -# Redis Streams используется только как устойчивый брокер заданий. -REDIS_BROKER_PASSWORD=local-redis-broker-password -BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 - -# Административные данные MinIO используются только при bootstrap. -MINIO_ROOT_USER=local-minio-admin -MINIO_ROOT_PASSWORD=local-minio-admin-password - -# API использует отдельного пользователя с доступом только к bucket вложений. -S3_ENDPOINT=s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY=messenger-app -S3_SECRET_KEY=local-messenger-s3-password - -# Порт Swagger внутри конфигурации CORS API. -SWAGGER_PORT=8081 - -# Ограничения ресурсов контейнеров. -API_CPUS=1 -API_MEMORY=512m -MIGRATE_CPUS=0.5 -MIGRATE_MEMORY=128m -NGINX_CPUS=0.5 -NGINX_MEMORY=128m -SWAGGER_CPUS=0.25 -SWAGGER_MEMORY=128m -TEST_CPUS=2 -TEST_MEMORY=512m -EXPORTER_CPUS=0.25 -EXPORTER_MEMORY=128m -PROMETHEUS_CPUS=1 -PROMETHEUS_MEMORY=512m -GRAFANA_CPUS=1 -GRAFANA_MEMORY=512m -OTEL_COLLECTOR_CPUS=0.5 -OTEL_COLLECTOR_MEMORY=256m -LOKI_CPUS=1 -LOKI_MEMORY=512m -TEMPO_CPUS=1 -TEMPO_MEMORY=512m -POSTGRES_CPUS=2 -POSTGRES_MEMORY=1g -VALKEY_CPUS=1 -VALKEY_MEMORY=256m -REDIS_BROKER_CPUS=1 -REDIS_BROKER_MEMORY=256m -DISPATCHER_CPUS=0.5 -DISPATCHER_MEMORY=128m -WORKER_CPUS=2 -WORKER_MEMORY=1g -S3_CPUS=1 -S3_MEMORY=512m -S3_BOOTSTRAP_CPUS=0.25 -S3_BOOTSTRAP_MEMORY=256m - -# Пароль локального администратора Grafana. -GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/lab7.env.example b/lab7.env.example new file mode 100644 index 0000000..0dcc587 --- /dev/null +++ b/lab7.env.example @@ -0,0 +1,121 @@ +# Изолированное имя Compose-проекта и его volumes/networks. +COMPOSE_PROJECT_NAME=mtusi2026lab7 + +# Адреса HTTPS API и локального health-only listener. +APP_HOST=0.0.0.0 +APP_PORT=8443 +APP_HEALTH_ADDRESS=127.0.0.1:8082 +MESSENGER_CONFIG_PATH=config/messenger.yaml + +# Адреса публикации сервисов на локальной машине. +API_BIND_ADDRESS=127.0.0.1:8443 +SWAGGER_BIND_ADDRESS=127.0.0.1:8081 +PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 +GRAFANA_BIND_ADDRESS=127.0.0.1:3000 +LOKI_BIND_ADDRESS=127.0.0.1:3100 +TEMPO_BIND_ADDRESS=127.0.0.1:3200 + +# Ресурсные атрибуты и внутренний OTLP endpoint приложений. +DEPLOYMENT_ENVIRONMENT=lab +SERVICE_VERSION=lab7 +OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-gateway:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +# Число API-реплик за Nginx. +API_REPLICAS=2 + +# Учётные данные локального PostgreSQL и строка подключения API. +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD=local-postgres-password +POSTGRES_API_PASSWORD=local-postgres-api-password +POSTGRES_DISPATCHER_PASSWORD=local-postgres-dispatcher-password +POSTGRES_WORKER_PASSWORD=local-postgres-worker-password +POSTGRES_RECONCILER_PASSWORD=local-postgres-reconciler-password +POSTGRES_EXPORTER_PASSWORD=local-postgres-exporter-password +POSTGRES_MIGRATE_PASSWORD=local-postgres-migrate-password +DATABASE_URL=postgres://messenger_api:local-postgres-api-password@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key +DATABASE_MIGRATE_URL=postgres://messenger_migrate:local-postgres-migrate-password@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key +DATABASE_DISPATCHER_URL=postgres://messenger_dispatcher:local-postgres-dispatcher-password@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key +DATABASE_WORKER_URL=postgres://messenger_worker:local-postgres-worker-password@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key +DATABASE_RECONCILER_URL=postgres://messenger_reconciler:local-postgres-reconciler-password@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security/internal-ca/ca.crt&sslcert=/run/security/client/tls.crt&sslkey=/run/security/client/tls.key + +# Пароль локального Valkey и строка подключения API. +VALKEY_API_PASSWORD=local-valkey-api-password +VALKEY_EXPORTER_PASSWORD=local-valkey-exporter-password +VALKEY_HEALTH_PASSWORD=local-valkey-health-password +CACHE_URL=rediss://messenger_api:local-valkey-api-password@redis-gateway:16379/0 + +# Redis Streams используется только как устойчивый брокер заданий. +REDIS_DISPATCHER_PASSWORD=local-redis-dispatcher-password +REDIS_WORKER_PASSWORD=local-redis-worker-password +REDIS_EXPORTER_PASSWORD=local-redis-exporter-password +REDIS_HEALTH_PASSWORD=local-redis-health-password +BROKER_DISPATCHER_URL=rediss://messenger_dispatcher:local-redis-dispatcher-password@redis-gateway:26379/0 +BROKER_WORKER_URL=rediss://messenger_worker:local-redis-worker-password@redis-gateway:36379/0 + +# Административные данные MinIO используются только при bootstrap. +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=local-minio-admin-password + +# API использует отдельного пользователя с доступом только к bucket вложений. +S3_ENDPOINT=s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_API_ACCESS_KEY=messenger-api +S3_API_SECRET_KEY=local-messenger-api-s3-password +S3_WORKER_ACCESS_KEY=messenger-worker +S3_WORKER_SECRET_KEY=local-messenger-worker-s3-password +S3_RECONCILER_ACCESS_KEY=messenger-reconciler +S3_RECONCILER_SECRET_KEY=local-messenger-reconciler-s3-password + +# Порт Swagger внутри конфигурации CORS API. +SWAGGER_PORT=8081 + +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=512m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=1 +GRAFANA_MEMORY=512m +OTEL_COLLECTOR_CPUS=0.5 +OTEL_COLLECTOR_MEMORY=256m +OTEL_GATEWAY_CPUS=0.25 +OTEL_GATEWAY_MEMORY=128m +REDIS_GATEWAY_CPUS=0.25 +REDIS_GATEWAY_MEMORY=128m +OPENBAO_CPUS=0.5 +OPENBAO_MEMORY=256m +OPENBAO_BIND_ADDRESS=127.0.0.1:8200 +LOKI_CPUS=1 +LOKI_MEMORY=512m +TEMPO_CPUS=1 +TEMPO_MEMORY=512m +POSTGRES_CPUS=2 +POSTGRES_MEMORY=1g +VALKEY_CPUS=1 +VALKEY_MEMORY=256m +REDIS_BROKER_CPUS=1 +REDIS_BROKER_MEMORY=256m +DISPATCHER_CPUS=0.5 +DISPATCHER_MEMORY=128m +WORKER_CPUS=2 +WORKER_MEMORY=1g +S3_CPUS=1 +S3_MEMORY=512m +S3_BOOTSTRAP_CPUS=0.25 +S3_BOOTSTRAP_MEMORY=256m + +# Пароль локального администратора Grafana. +GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/monitoring/grafana/dashboards/messenger-lab6.json b/monitoring/grafana/dashboards/messenger-lab7.json similarity index 99% rename from monitoring/grafana/dashboards/messenger-lab6.json rename to monitoring/grafana/dashboards/messenger-lab7.json index f20cabe..7d03bec 100644 --- a/monitoring/grafana/dashboards/messenger-lab6.json +++ b/monitoring/grafana/dashboards/messenger-lab7.json @@ -1471,8 +1471,8 @@ "to": "now" }, "timezone": "browser", - "title": "Messenger lab6 observability", - "uid": "messenger-lab6", + "title": "Messenger lab7 security and observability", + "uid": "messenger-lab7", "version": 1, "links": [ { diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml index 99af68b..2eb6a09 100644 --- a/monitoring/grafana/provisioning/datasources/observability.yml +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -13,38 +13,49 @@ datasources: type: prometheus uid: prometheus access: proxy - url: http://prometheus:9090 + url: https://prometheus-grafana-gateway:8443 isDefault: true editable: false jsonData: httpMethod: POST + tlsAuthWithCACert: true + serverName: telemetry-gateway exemplarTraceIdDestinations: - name: trace_id datasourceUid: tempo + secureJsonData: &mtls + tlsCACert: $__file{/run/security/internal-ca/ca.crt} + tlsClientCert: $__file{/run/security/client/tls.crt} + tlsClientKey: $__file{/run/security/client/tls.key} - name: Loki type: loki uid: loki access: proxy - url: http://loki:3100 + url: https://loki-grafana-gateway:8443 editable: false jsonData: maxLines: 1000 + tlsAuthWithCACert: true + serverName: telemetry-gateway derivedFields: - name: TraceID matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})' datasourceUid: tempo url: '$${__value.raw}' + secureJsonData: *mtls - name: Tempo type: tempo uid: tempo access: proxy - url: http://tempo:3200 + url: https://tempo-grafana-gateway:8443 editable: false jsonData: nodeGraph: enabled: true + tlsAuthWithCACert: true + serverName: telemetry-gateway search: hide: false serviceMap: @@ -72,3 +83,4 @@ datasources: query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))' - name: Error rate query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))' + secureJsonData: *mtls diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 70b5dfd..1bef4f1 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -12,8 +12,15 @@ scrape_configs: dns_sd_configs: - names: ["api"] type: A - port: 8080 + port: 9091 refresh_interval: 2s + scheme: https + tls_config: + ca_file: /run/security/internal-ca/ca.crt + cert_file: /run/security/prometheus/tls.crt + key_file: /run/security/prometheus/tls.key + server_name: api + min_version: TLS13 relabel_configs: - target_label: service_name replacement: messenger-api @@ -44,6 +51,13 @@ scrape_configs: type: A port: 9091 refresh_interval: 2s + scheme: https + tls_config: &pipeline_tls + ca_file: /run/security/internal-ca/ca.crt + cert_file: /run/security/prometheus/tls.crt + key_file: /run/security/prometheus/tls.key + server_name: pipeline-metrics + min_version: TLS13 relabel_configs: - target_label: service_name replacement: messenger-worker @@ -55,6 +69,8 @@ scrape_configs: - job_name: outbox-dispatcher static_configs: - targets: ["dispatcher:9091"] + scheme: https + tls_config: *pipeline_tls relabel_configs: - target_label: service_name replacement: messenger-dispatcher @@ -65,6 +81,11 @@ scrape_configs: - job_name: minio metrics_path: /minio/v2/metrics/cluster + scheme: https + tls_config: + ca_file: /run/security/internal-ca/ca.crt + server_name: s3 + min_version: TLS13 static_configs: - targets: ["s3:9000"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index d62dcd6..8ff6fde 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -25,18 +25,32 @@ http { } server { - listen 8080; + listen 8443 ssl; client_max_body_size 11m; + ssl_certificate /run/security/ingress/tls.crt; + ssl_certificate_key /run/security/ingress/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_tickets off; + location / { - proxy_pass http://messenger; + proxy_pass https://messenger; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto https; proxy_request_buffering off; + proxy_ssl_server_name on; + proxy_ssl_name api; + proxy_ssl_trusted_certificate /run/security/internal-ca/ca.crt; + proxy_ssl_verify on; + proxy_ssl_verify_depth 1; + proxy_ssl_certificate /run/security/load-balancer/tls.crt; + proxy_ssl_certificate_key /run/security/load-balancer/tls.key; + proxy_ssl_session_reuse off; + proxy_connect_timeout 1s; proxy_send_timeout 10s; proxy_read_timeout 10s; diff --git a/nginx/upstream.conf b/nginx/upstream.conf index f6131cb..3e60dfe 100644 --- a/nginx/upstream.conf +++ b/nginx/upstream.conf @@ -1 +1 @@ -server api:8080 resolve max_fails=1 fail_timeout=3s; +server api:8443 resolve max_fails=1 fail_timeout=3s; -- 2.54.0 From 5ac64f51faecefc09ebc795f62dc73d81410fd75 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 6 Nov 2026 21:17:03 +0300 Subject: [PATCH 4/5] test: verify security boundaries and recovery --- .gitea/workflows/ci.yml | 56 +++++--- .gitea/workflows/deploy.yml | 64 +++++---- .gitea/workflows/incidents.yml | 20 ++- Makefile | 37 +++++- scripts/bao-docker.sh | 10 ++ scripts/bootstrap-local.sh | 56 ++++++++ scripts/incident-database.sh | 6 +- scripts/incident-telemetry.sh | 8 +- scripts/incident-worker.sh | 6 +- scripts/lab6-traffic.py | 11 +- scripts/lab7-worker-compromise.sh | 209 ++++++++++++++++++++++++++++++ scripts/openbao-bootstrap.sh | 93 +++++++++++++ scripts/openbao-common.sh | 91 +++++++++++++ scripts/openbao-init.sh | 46 +++++++ scripts/openbao-issue-approle.sh | 87 +++++++++++++ scripts/openbao-rotate.sh | 42 ++++++ scripts/openbao-seed.sh | 62 +++++++++ scripts/openbao-test-policy.sh | 140 ++++++++++++++++++++ scripts/openbao-unseal.sh | 51 ++++++++ scripts/rolling-update.sh | 13 +- scripts/scan-security.sh | 132 +++++++++++++++++++ scripts/test-lab6.sh | 14 +- scripts/test-leaf-rotation.sh | 160 +++++++++++++++++++++++ scripts/test-login-rate-limit.py | 66 ++++++++++ scripts/test-openbao-lifecycle.sh | 177 +++++++++++++++++++++++++ scripts/test-tls-matrix.sh | 163 +++++++++++++++++++++++ 26 files changed, 1752 insertions(+), 68 deletions(-) create mode 100755 scripts/bao-docker.sh create mode 100755 scripts/bootstrap-local.sh create mode 100755 scripts/lab7-worker-compromise.sh create mode 100755 scripts/openbao-bootstrap.sh create mode 100755 scripts/openbao-common.sh create mode 100755 scripts/openbao-init.sh create mode 100755 scripts/openbao-issue-approle.sh create mode 100755 scripts/openbao-rotate.sh create mode 100755 scripts/openbao-seed.sh create mode 100755 scripts/openbao-test-policy.sh create mode 100755 scripts/openbao-unseal.sh create mode 100755 scripts/scan-security.sh create mode 100755 scripts/test-leaf-rotation.sh create mode 100644 scripts/test-login-rate-limit.py create mode 100755 scripts/test-openbao-lifecycle.sh create mode 100755 scripts/test-tls-matrix.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 33018c7..36db92a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -14,15 +14,16 @@ concurrency: jobs: verify: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-${{ github.run_id }} API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 steps: - name: Checkout exact commit run: | @@ -48,8 +49,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh - python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -65,28 +66,48 @@ jobs: - name: Prepare local stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Integration checks run: | - set -a - . ./.env - set +a - test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" - make test BASE_URL=http://nginx:8080 + test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" + make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-tls-matrix ENV_FILE=.env + ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh + docker compose --profile security run --rm dependency-tests + OPENBAO_ADDRESS=$(docker compose port openbao 8200) + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests - go run ./cmd/dependencycheck -base-url=http://nginx:8080 - make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 - make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose up -d --scale api=2 --wait --wait-timeout 90 make test-lab5-acceptance - make test-lab5 - ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + make test-worker-compromise ENV_FILE=.env + make scan-security + ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 + + - name: Upload security evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: lab7-security-${{ github.sha }} + path: | + evidence/lab7-*.json + evidence/runtime/security-scan + if-no-files-found: warn + retention-days: 14 - name: Container logs if: failure() @@ -97,5 +118,8 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index b1b9a29..7fdf0aa 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,7 +12,7 @@ jobs: name: Verify selected commit if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 outputs: api_image: ${{ steps.publish.outputs.api_image }} nginx_image: ${{ steps.publish.outputs.nginx_image }} @@ -22,7 +22,7 @@ jobs: s3_image: ${{ steps.publish.outputs.s3_image }} s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-deploy-${{ github.run_id }} COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 @@ -30,13 +30,14 @@ jobs: GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 - API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }} - NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }} - SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }} - PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }} - GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }} - S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }} - S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }} + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio-client:${{ github.sha }} steps: - name: Checkout exact commit run: | @@ -62,8 +63,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh - python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -79,27 +80,35 @@ jobs: - name: Prepare local stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Integration checks run: | - set -a - . ./.env - set +a - test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" - make test BASE_URL=http://nginx:8080 + test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" + make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-tls-matrix ENV_FILE=.env + ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh + docker compose --profile security run --rm dependency-tests + OPENBAO_ADDRESS=$(docker compose port openbao 8200) + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests - go run ./cmd/dependencycheck -base-url=http://nginx:8080 - make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 - make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-lab5-acceptance - make test-lab5 - ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + make test-worker-compromise ENV_FILE=.env + ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 - name: Publish verified images id: publish @@ -137,7 +146,10 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi deploy: @@ -177,17 +189,17 @@ jobs: run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin" - name: Upload image manifest - run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env" + run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab7-${{ github.run_id }}.env" - name: Deploy and verify run: | ssh "$DEPLOY_USER@$DEPLOY_HOST" " set -eu - cd ~/mtusi2026lab6 + cd ~/mtusi2026lab7 git fetch origin main git cat-file -e '$DEPLOY_SHA^{commit}' git reset --hard '$DEPLOY_SHA' - manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env' + manifest='/tmp/mtusi2026lab7-${{ github.run_id }}.env' if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then rm -f \"\$manifest\" else diff --git a/.gitea/workflows/incidents.yml b/.gitea/workflows/incidents.yml index 244f11e..57c4b52 100644 --- a/.gitea/workflows/incidents.yml +++ b/.gitea/workflows/incidents.yml @@ -1,10 +1,10 @@ -name: Lab 6 incidents +name: Lab 7 inherited incidents on: workflow_dispatch: concurrency: - group: lab6-incidents-${{ github.repository }}-${{ github.ref }} + group: lab7-incidents-${{ github.repository }}-${{ github.ref }} cancel-in-progress: false jobs: @@ -12,13 +12,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 35 env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-incidents-${{ github.run_id }} API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 steps: - name: Checkout exact commit run: | @@ -34,17 +35,21 @@ jobs: - name: Prepare stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 180 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Reproduce SLO incidents run: | ENV_FILE=.env \ - BASE_URL=http://nginx:8080 \ + BASE_URL=https://nginx:8443 \ + CA_FILE=.runtime/security/public-ca/ca.crt \ PROMETHEUS_URL=http://prometheus:9090 \ TEMPO_URL=http://tempo:3200 \ LOKI_URL=http://loki:3100 \ @@ -69,5 +74,8 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true docker compose down -v --remove-orphans diff --git a/Makefile b/Makefile index 5ad43ad..fcbd5c7 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,10 @@ PYTHON ?= python3 BASE_URL ?= https://localhost:8443 CA_FILE ?= ACTION_SCRIPT ?= +ENV_FILE ?= $(if $(wildcard .env),.env,lab7.env.example) TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-security +.PHONY: help check validate test load test-persistence test-failover test-security test-tls-matrix test-leaf-rotation test-openbao-lifecycle test-login-rate-limit test-worker-compromise scan-security test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -31,3 +32,37 @@ test-failover: test-security: $(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-tls-matrix: + docker compose --env-file "$(ENV_FILE)" --profile security run --rm security-tests + +test-leaf-rotation: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" ./scripts/test-leaf-rotation.sh + +test-openbao-lifecycle: + ENV_FILE="$(ENV_FILE)" ./scripts/test-openbao-lifecycle.sh + +test-login-rate-limit: + @test -n "$(CA_FILE)" || (echo "Set CA_FILE"; exit 2) + $(PYTHON) scripts/test-login-rate-limit.py --base-url "$(BASE_URL)" --ca-file "$(CA_FILE)" + +test-worker-compromise: + ENV_FILE="$(ENV_FILE)" ./scripts/lab7-worker-compromise.sh + +scan-security: + ./scripts/scan-security.sh + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh + +lab6-load: + $(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" $(TLS_ARGS) --mode mixed --duration 60 --concurrency 4 --wait-ready + +test-lab6-smoke: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh smoke + +test-lab6: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh full diff --git a/scripts/bao-docker.sh b/scripts/bao-docker.sh new file mode 100755 index 0000000..303b988 --- /dev/null +++ b/scripts/bao-docker.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} + +exec docker compose --project-directory "$repo_dir" --env-file "$env_file" run --rm --no-deps -T \ + -v "$repo_dir:$repo_dir:ro" openbao-cli "$@" diff --git a/scripts/bootstrap-local.sh b/scripts/bootstrap-local.sh new file mode 100755 index 0000000..07c6233 --- /dev/null +++ b/scripts/bootstrap-local.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${1:-$repo_dir/lab7.env.example} + +if [ ! -f "$env_file" ]; then + echo "environment file not found: $env_file" >&2 + exit 2 +fi + +compose() { + docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@" +} + +if [ ! -f "$repo_dir/.runtime/security/openbao/tls.crt" ]; then + "$repo_dir/infra/pki/bootstrap.sh" +fi + +"$script_dir/openbao-seed.sh" "$env_file" +compose up -d security-bootstrap openbao-permissions openbao +openbao_address=$(compose port openbao 8200) +BAO_ADDR="https://$openbao_address" +BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" +export BAO_ADDR BAO_CACERT ENV_FILE="$env_file" + +remaining=60 +until curl --silent --show-error --cacert "$BAO_CACERT" \ + --output /dev/null "$BAO_ADDR/v1/sys/health"; do + remaining=$((remaining - 1)) + [ "$remaining" -gt 0 ] || { + echo "OpenBao did not start" >&2 + exit 1 + } + sleep 1 +done + +ENV_FILE="$env_file" "$script_dir/openbao-init.sh" +ENV_FILE="$env_file" "$script_dir/openbao-unseal.sh" +remaining=60 +until curl --fail --silent --show-error --cacert "$BAO_CACERT" \ + --output /dev/null "$BAO_ADDR/v1/sys/health"; do + remaining=$((remaining - 1)) + [ "$remaining" -gt 0 ] || { + echo "OpenBao did not become active after unseal" >&2 + exit 1 + } + sleep 1 +done +ENV_FILE="$env_file" "$script_dir/openbao-bootstrap.sh" \ + "$repo_dir/.runtime/openbao/seed/api.json" \ + "$repo_dir/.runtime/openbao/seed/worker.json" + +compose up -d --build --scale api=2 --wait --wait-timeout 240 diff --git a/scripts/incident-database.sh b/scripts/incident-database.sh index bc3c6e0..6bc3cc6 100755 --- a/scripts/incident-database.sh +++ b/scripts/incident-database.sh @@ -2,8 +2,9 @@ set -eu -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} loki_url=${LOKI_URL:-http://127.0.0.1:3100} @@ -33,6 +34,7 @@ trap restore EXIT INT TERM python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode messages \ --duration "$((outage_seconds + 20))" \ --concurrency 4 \ diff --git a/scripts/incident-telemetry.sh b/scripts/incident-telemetry.sh index 1f0cc50..716414a 100755 --- a/scripts/incident-telemetry.sh +++ b/scripts/incident-telemetry.sh @@ -2,8 +2,9 @@ set -eu -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} evidence_root=${EVIDENCE_DIR:-evidence/runtime} outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30} @@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose stop tempo loki >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode mixed \ --duration "$outage_seconds" \ --concurrency 4 \ @@ -41,7 +43,7 @@ load_pid=$! sleep "$outage_seconds" wait "$load_pid" load_pid= -health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready") +health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready") python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose up -d --wait --wait-timeout 90 tempo loki >/dev/null diff --git a/scripts/incident-worker.sh b/scripts/incident-worker.sh index 016df05..f5477de 100755 --- a/scripts/incident-worker.sh +++ b/scripts/incident-worker.sh @@ -3,8 +3,9 @@ set -eu mode=${1:-short} -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} evidence_root=${EVIDENCE_DIR:-evidence/runtime} require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} @@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose stop worker >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode attachments \ --duration "$load_seconds" \ --requests "$requests" \ diff --git a/scripts/lab6-traffic.py b/scripts/lab6-traffic.py index 244d53b..e87ece1 100755 --- a/scripts/lab6-traffic.py +++ b/scripts/lab6-traffic.py @@ -7,6 +7,7 @@ import json import math from pathlib import Path import secrets +import ssl import struct import threading import time @@ -16,6 +17,7 @@ import urllib.request import zlib MAX_RESPONSE_BYTES = 1 << 20 +TLS_CONTEXT = None def png_fixture(width=96, height=64): @@ -37,7 +39,7 @@ def png_fixture(width=96, height=64): def request(method, url, body=None, headers=None, timeout=10): req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) try: - with urllib.request.urlopen(req, timeout=timeout) as response: + with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response: payload = response.read(MAX_RESPONSE_BYTES) return response.status, payload except urllib.error.HTTPError as error: @@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout): def main(): parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents") - parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--base-url", default="https://localhost:8443") + parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt") parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed") parser.add_argument("--duration", type=float, default=30) parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only") @@ -179,6 +182,10 @@ def main(): parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready") args = parser.parse_args() + global TLS_CONTEXT + if args.base_url.startswith("https://"): + TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file) + if args.duration <= 0 or args.duration > 1800: parser.error("duration must be in (0, 1800]") if args.requests < 0: diff --git a/scripts/lab7-worker-compromise.sh b/scripts/lab7-worker-compromise.sh new file mode 100755 index 0000000..f9d9294 --- /dev/null +++ b/scripts/lab7-worker-compromise.sh @@ -0,0 +1,209 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +runtime_dir="$repo_dir/.runtime/evidence/lab7-worker" +state_file="$runtime_dir/queued-attachment.json" +rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json" +rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)" +rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)" +rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)" +summary_file="$repo_dir/evidence/lab7-worker-compromise.json" +started_at=$(date +%s) + +compose() { + docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@" +} + +wait_healthy() { + service=$1 + remaining=60 + while [ "$remaining" -gt 0 ]; do + container=$(compose ps -q "$service") + if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then + return 0 + fi + remaining=$((remaining - 1)) + sleep 1 + done + echo "$service did not become healthy" >&2 + return 1 +} + +rotate_worker_secret() { + secret_file=$1 + compose stop openbao-agent-worker >/dev/null + ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \ + "$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null + docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null + wait_healthy openbao-agent-worker +} + +restore() { + compose stop worker openbao-agent-worker >/dev/null 2>&1 || true + compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true + compose restart redis-broker redis-gateway >/dev/null 2>&1 || true + compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true + compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true + rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true + compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true +} + +mkdir -p "$runtime_dir" +chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir" +trap restore EXIT HUP INT TERM + +openbao_address=$(compose port openbao 8200) +BAO_ADDR="https://$openbao_address" +BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" +export BAO_ADDR BAO_CACERT + +compose stop worker >/dev/null +compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file" +chmod 0600 "$state_file" + +old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys +import urllib.parse + +with open(sys.argv[1], encoding="utf-8") as source: + secret = json.load(source) +print(urllib.parse.urlsplit(secret["database_url"]).password) +PY +) +old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys +import urllib.parse + +with open(sys.argv[1], encoding="utf-8") as source: + print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password) +PY +) +old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + print(json.load(source)["s3_access_key"]) +PY +) +old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + print(json.load(source)["s3_secret_key"]) +PY +) + +python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \ + "$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY' +import json +import os +import sys +import urllib.parse + + +def replace_password(url, password): + parsed = urllib.parse.urlsplit(url) + username = urllib.parse.quote(parsed.username or "", safe="") + netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) + +with open(sys.argv[1], encoding="utf-8") as source: + secret = json.load(source) +secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3]) +secret["database_url"] = replace_password(secret["database_url"], sys.argv[4]) +secret["s3_secret_key"] = sys.argv[5] +with open(sys.argv[2], "w", encoding="utf-8") as destination: + json.dump(secret, destination, separators=(",", ":")) + destination.write("\n") +os.chmod(sys.argv[2], 0o600) +PY + +compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null +compose restart redis-broker redis-gateway >/dev/null +compose exec -T postgres sh -c \ + "psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null +compose exec -T postgres sh -c \ + 'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null +compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null +compose --profile security run --rm --no-deps \ + -e REDISCLI_AUTH="$old_broker_password" \ + --entrypoint /bin/sh dependency-tests -c ' + output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \ + --cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \ + --user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true) + case "$output" in + *WRONGPASS*|*invalid\ username-password*) exit 0 ;; + *) echo "expected old broker credential rejection" >&2; exit 1 ;; + esac + ' + +if compose --profile security run --rm --no-deps \ + -e OLD_DATABASE_PASSWORD="$old_database_password" \ + --entrypoint /bin/sh dependency-tests -c ' + psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \ + --set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1 + '; then + echo "old worker database credential was not rejected" >&2 + exit 1 +fi + +if compose --profile security run --rm --no-deps \ + -e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \ + --entrypoint /bin/sh dependency-tests -c ' + export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt + export MC_CONFIG_DIR=/tmp/old-worker-mc + mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 && + mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1 + '; then + echo "old worker S3 credential was not rejected" >&2 + exit 1 +fi + +rotate_worker_secret "$rotated_secret" +compose up -d --no-deps --force-recreate worker >/dev/null +wait_healthy worker + +compose --profile experiments run --rm --no-deps -T lab5-experiment \ + /app/lab5experiment api-verify <"$state_file" >/dev/null + +finished_at=$(date +%s) +python3 - "$summary_file" "$started_at" "$finished_at" <<'PY' +import datetime +import json +import sys + +started_at = int(sys.argv[2]) +finished_at = int(sys.argv[3]) +summary = { + "schema_version": 1, + "scenario": "worker credential compromise and rotation", + "generated_at": datetime.datetime.fromtimestamp( + finished_at, datetime.timezone.utc + ).replace(microsecond=0).isoformat(), + "recovery_seconds": finished_at - started_at, + "old_credential_rejected": True, + "old_database_credential_rejected": True, + "old_s3_credential_rejected": True, + "openbao_secret_rotated": True, + "queued_attachment_preserved": True, + "result": "PASS", +} +with open(sys.argv[1], "w", encoding="utf-8") as destination: + json.dump(summary, destination, ensure_ascii=False, indent=2) + destination.write("\n") +PY + +trap - EXIT HUP INT TERM +restore +rm -f "$rotated_secret" +echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment" diff --git a/scripts/openbao-bootstrap.sh b/scripts/openbao-bootstrap.sh new file mode 100755 index 0000000..dc67ac5 --- /dev/null +++ b/scripts/openbao-bootstrap.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +api_secret_file=$1 +worker_secret_file=$2 + +require_command "$BAO_BIN" +require_command python3 +require_runtime_path "$api_secret_file" +require_runtime_path "$worker_secret_file" +require_private_file "$api_secret_file" +require_private_file "$worker_secret_file" +require_unsealed +load_operator_token + +python3 - "$api_secret_file" "$worker_secret_file" <<'PY' +import json +import sys + +for path in sys.argv[1:]: + with open(path, encoding="utf-8") as source: + value = json.load(source) + if not isinstance(value, dict) or not value: + raise SystemExit(f"secret input must be a non-empty JSON object: {path}") +PY + +mounts_file=$(mktemp "$openbao_runtime_dir/.mounts.XXXXXX") +auth_file=$(mktemp "$openbao_runtime_dir/.auth.XXXXXX") +chmod 0600 "$mounts_file" "$auth_file" + +cleanup() { + rm -f "$mounts_file" "$auth_file" +} + +trap cleanup EXIT HUP INT TERM + +"$BAO_BIN" secrets list -format=json >"$mounts_file" +if ! python3 - "$mounts_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + raise SystemExit(0 if "secret/" in json.load(source) else 1) +PY +then + "$BAO_BIN" secrets enable -path=secret -version=2 kv >/dev/null +fi + +"$BAO_BIN" auth list -format=json >"$auth_file" +if ! python3 - "$auth_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + raise SystemExit(0 if "approle/" in json.load(source) else 1) +PY +then + "$BAO_BIN" auth enable approle >/dev/null +fi + +"$BAO_BIN" policy write messenger-api "$repo_dir/infra/openbao/policies/messenger-api.hcl" >/dev/null +"$BAO_BIN" policy write messenger-worker "$repo_dir/infra/openbao/policies/messenger-worker.hcl" >/dev/null + +for service in api worker; do + "$BAO_BIN" delete "auth/approle/role/messenger-$service" >/dev/null 2>&1 || true + "$BAO_BIN" write "auth/approle/role/messenger-$service" \ + bind_secret_id=true \ + secret_id_ttl=1h \ + secret_id_num_uses=0 \ + token_policies="messenger-$service" \ + token_no_default_policy=true \ + token_ttl=15m \ + token_max_ttl=1h \ + token_num_uses=0 >/dev/null +done + +"$BAO_BIN" kv put -mount=secret messenger/api @"$api_secret_file" >/dev/null +"$BAO_BIN" kv put -mount=secret messenger/worker @"$worker_secret_file" >/dev/null + +"$script_dir/openbao-issue-approle.sh" api "$openbao_runtime_dir/approle/api" +"$script_dir/openbao-issue-approle.sh" worker "$openbao_runtime_dir/approle/worker" + +echo "OpenBao policies, AppRoles and KV v2 secrets configured" + diff --git a/scripts/openbao-common.sh b/scripts/openbao-common.sh new file mode 100755 index 0000000..9497ce6 --- /dev/null +++ b/scripts/openbao-common.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +set -eu + +umask 077 + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +openbao_runtime_dir="$repo_dir/.runtime/openbao" +openbao_operator_file="$openbao_runtime_dir/operator/init.json" + +BAO_BIN=${BAO_BIN:-$repo_dir/scripts/bao-docker.sh} +BAO_ADDR=${BAO_ADDR:-https://localhost:8200} +BAO_CACERT=${BAO_CACERT:-$repo_dir/.runtime/security/openbao/ca.crt} + +export BAO_ADDR BAO_CACERT + +fail() { + echo "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +require_file() { + [ -f "$1" ] || fail "required file not found: $1" +} + +require_private_file() { + require_file "$1" + permissions=$(stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1") + case "$permissions" in + 600|400) ;; + *) fail "file must have permissions 0600 or 0400: $1" ;; + esac +} + +require_runtime_path() { + case "$1" in + "$openbao_runtime_dir"/*) ;; + *) fail "runtime credentials must stay under $openbao_runtime_dir" ;; + esac +} + +json_field() { + file=$1 + expression=$2 + python3 - "$file" "$expression" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + value = json.load(source) + +for component in sys.argv[2].split("."): + value = value[component] + +if isinstance(value, bool): + print(str(value).lower()) +else: + print(value) +PY +} + +load_operator_token() { + require_private_file "$openbao_operator_file" + BAO_TOKEN=$(json_field "$openbao_operator_file" root_token) + export BAO_TOKEN +} + +openbao_status() { + status_file=$1 + "$BAO_BIN" status -format=json >"$status_file" 2>/dev/null || true + [ -s "$status_file" ] || fail "OpenBao is not reachable at $BAO_ADDR" +} + +require_unsealed() { + mkdir -p "$openbao_runtime_dir" + chmod 0700 "$openbao_runtime_dir" + status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") + chmod 0600 "$status_file" + openbao_status "$status_file" + initialized=$(json_field "$status_file" initialized) + sealed=$(json_field "$status_file" sealed) + rm -f "$status_file" + [ "$initialized" = "true" ] || fail "OpenBao is not initialized" + [ "$sealed" = "false" ] || fail "OpenBao is sealed" +} + diff --git a/scripts/openbao-init.sh b/scripts/openbao-init.sh new file mode 100755 index 0000000..9e36a33 --- /dev/null +++ b/scripts/openbao-init.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command "$BAO_BIN" +require_command python3 +require_file "$BAO_CACERT" + +mkdir -p "$openbao_runtime_dir/operator" +chmod 0700 "$openbao_runtime_dir" "$openbao_runtime_dir/operator" + +status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") +init_file=$(mktemp "$openbao_runtime_dir/operator/.init.XXXXXX") +chmod 0600 "$status_file" "$init_file" + +cleanup() { + rm -f "$status_file" "$init_file" +} + +trap cleanup EXIT HUP INT TERM + +openbao_status "$status_file" + +if [ "$(json_field "$status_file" initialized)" = "true" ]; then + echo "OpenBao is already initialized" + exit 0 +fi + +if [ -e "$openbao_operator_file" ]; then + fail "operator material already exists for an uninitialized OpenBao instance" +fi + +"$BAO_BIN" operator init \ + -key-shares=3 \ + -key-threshold=2 \ + -format=json >"$init_file" + +chmod 0600 "$init_file" +mv "$init_file" "$openbao_operator_file" + +echo "OpenBao initialized; operator material saved to .runtime/openbao/operator/init.json" +echo "Keep this file offline and run scripts/openbao-unseal.sh" + diff --git a/scripts/openbao-issue-approle.sh b/scripts/openbao-issue-approle.sh new file mode 100755 index 0000000..4506ab9 --- /dev/null +++ b/scripts/openbao-issue-approle.sh @@ -0,0 +1,87 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +service=$1 +output_dir=$2 + +case "$service" in + api|worker) ;; + *) fail "service must be api or worker" ;; +esac + +require_runtime_path "$output_dir" +require_command "$BAO_BIN" +require_command python3 +require_unsealed + +if [ -z "${BAO_TOKEN:-}" ]; then + load_operator_token +fi + +role="messenger-$service" +parent_dir=$(dirname "$output_dir") +staging_dir="$parent_dir/.${service}.$$" +response_file="$staging_dir/response.json" + +cleanup() { + rm -rf "$staging_dir" +} + +trap cleanup EXIT HUP INT TERM + +mkdir -p "$parent_dir" "$staging_dir" +chmod 0700 "$parent_dir" "$staging_dir" + +"$BAO_BIN" read -field=role_id "auth/approle/role/$role/role-id" >"$staging_dir/role-id" +"$BAO_BIN" write -f -format=json "auth/approle/role/$role/secret-id" >"$response_file" + +python3 - "$response_file" "$staging_dir" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + data = json.load(source)["data"] + +for name, field in (("secret-id", "secret_id"), ("secret-id-accessor", "secret_id_accessor")): + path = os.path.join(sys.argv[2], name) + with open(path, "w", encoding="utf-8") as destination: + destination.write(data[field] + "\n") + os.chmod(path, 0o600) +PY + +rm -f "$response_file" +chmod 0600 "$staging_dir/role-id" + +old_accessor= +if [ -f "$output_dir/secret-id-accessor" ]; then + old_accessor=$(cat "$output_dir/secret-id-accessor") +fi + +mkdir -p "$output_dir" +chmod 0700 "$output_dir" +for file in role-id secret-id secret-id-accessor; do + mv "$staging_dir/$file" "$output_dir/$file" +done + +if [ -n "$old_accessor" ]; then + "$BAO_BIN" write "auth/approle/role/$role/secret-id-accessor/destroy" \ + secret_id_accessor="$old_accessor" >/dev/null 2>&1 || true +fi + +if [ "$output_dir" = "$openbao_runtime_dir/approle/$service" ]; then + docker compose --project-directory "$repo_dir" --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" \ + run --rm --no-deps -e SERVICE="$service" openbao-identity-install +fi + +echo "issued short-lived AppRole credentials for $service in ${output_dir#"$repo_dir/"}" + diff --git a/scripts/openbao-rotate.sh b/scripts/openbao-rotate.sh new file mode 100755 index 0000000..7460197 --- /dev/null +++ b/scripts/openbao-rotate.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +service=$1 +secret_file=$2 + +case "$service" in + api|worker) ;; + *) fail "service must be api or worker" ;; +esac + +require_command "$BAO_BIN" +require_command python3 +require_runtime_path "$secret_file" +require_private_file "$secret_file" +require_unsealed +load_operator_token + +python3 - "$secret_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + value = json.load(source) +if not isinstance(value, dict) or not value: + raise SystemExit("replacement secret must be a non-empty JSON object") +PY + +"$BAO_BIN" kv put -mount=secret "messenger/$service" @"$secret_file" >/dev/null +"$script_dir/openbao-issue-approle.sh" "$service" "$openbao_runtime_dir/approle/$service" + +echo "$service secret rotated; OpenBao Agent will render the new KV version" + diff --git a/scripts/openbao-seed.sh b/scripts/openbao-seed.sh new file mode 100755 index 0000000..c20f553 --- /dev/null +++ b/scripts/openbao-seed.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +set -eu + +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +seed_dir="$repo_dir/.runtime/openbao/seed" +mkdir -p "$seed_dir" +chmod 0700 "$repo_dir/.runtime/openbao" "$seed_dir" + +python3 - "$1" "$seed_dir/api.json" "$seed_dir/worker.json" <<'PY' +import json +import os +import sys + +values = {} +with open(sys.argv[1], encoding="utf-8") as source: + for raw_line in source: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + name, separator, value = line.partition("=") + if not separator: + raise SystemExit(f"invalid env line: {raw_line.rstrip()}") + values[name.strip()] = os.environ.get(name.strip(), value) + +required = ( + "DATABASE_URL", "DATABASE_WORKER_URL", "CACHE_URL", "BROKER_WORKER_URL", + "S3_API_ACCESS_KEY", "S3_API_SECRET_KEY", "S3_WORKER_ACCESS_KEY", "S3_WORKER_SECRET_KEY", +) +missing = [name for name in required if not values.get(name)] +if missing: + raise SystemExit("missing environment values: " + ", ".join(missing)) + +documents = ( + (sys.argv[2], { + "database_url": values["DATABASE_URL"], + "cache_url": values["CACHE_URL"], + "s3_access_key": values["S3_API_ACCESS_KEY"], + "s3_secret_key": values["S3_API_SECRET_KEY"], + }), + (sys.argv[3], { + "database_url": values["DATABASE_WORKER_URL"], + "broker_url": values["BROKER_WORKER_URL"], + "s3_access_key": values["S3_WORKER_ACCESS_KEY"], + "s3_secret_key": values["S3_WORKER_SECRET_KEY"], + }), +) + +for path, value in documents: + with open(path, "w", encoding="utf-8") as destination: + json.dump(value, destination, separators=(",", ":")) + destination.write("\n") + os.chmod(path, 0o600) +PY + +echo "OpenBao seed files created under .runtime/openbao/seed" diff --git a/scripts/openbao-test-policy.sh b/scripts/openbao-test-policy.sh new file mode 100755 index 0000000..7134e5c --- /dev/null +++ b/scripts/openbao-test-policy.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command curl +require_command python3 +require_command "$BAO_BIN" +require_file "$BAO_CACERT" +require_unsealed +load_operator_token + +test_dir="$openbao_runtime_dir/policy-test.$$" +mkdir -p "$test_dir" +chmod 0700 "$test_dir" + +cleanup() { + for service in api worker; do + headers="$test_dir/$service/token-header" + if [ -f "$headers" ]; then + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --request POST \ + "$BAO_ADDR/v1/auth/token/revoke-self" >/dev/null 2>&1 || true + fi + done + rm -rf "$test_dir" +} + +trap cleanup EXIT HUP INT TERM + +login() { + service=$1 + identity_dir="$test_dir/$service/identity" + session_dir="$test_dir/$service" + response_file="$session_dir/login.json" + payload_file="$session_dir/login-payload.json" + + mkdir -p "$session_dir" + chmod 0700 "$session_dir" + "$script_dir/openbao-issue-approle.sh" "$service" "$identity_dir" >/dev/null + + python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as role_source: + role_id = role_source.read().strip() +with open(sys.argv[2], encoding="utf-8") as secret_source: + secret_id = secret_source.read().strip() +with open(sys.argv[3], "w", encoding="utf-8") as destination: + json.dump({"role_id": role_id, "secret_id": secret_id}, destination) +os.chmod(sys.argv[3], 0o600) +PY + + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$payload_file" \ + --output "$response_file" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") + [ "$status" = "200" ] || fail "$service AppRole login returned HTTP $status" + chmod 0600 "$response_file" + + python3 - "$response_file" "$service" "$session_dir/token-header" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + auth = json.load(source)["auth"] + +expected_policy = f"messenger-{sys.argv[2]}" +if not auth["renewable"]: + raise SystemExit("AppRole token is not renewable") +if not 0 < auth["lease_duration"] <= 3600: + raise SystemExit("AppRole token TTL is outside the configured limit") +if auth["policies"] != [expected_policy]: + raise SystemExit(f"unexpected token policies: {auth['policies']}") + +with open(sys.argv[3], "w", encoding="utf-8") as destination: + destination.write(f"X-Vault-Token: {auth['client_token']}\n") +os.chmod(sys.argv[3], 0o600) +PY + + rm -f "$payload_file" "$response_file" "$identity_dir/secret-id" +} + +expect_status() { + service=$1 + method=$2 + path=$3 + expected=$4 + payload=${5:-} + headers="$test_dir/$service/token-header" + output="$test_dir/$service/response.json" + + if [ -n "$payload" ]; then + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --header "Content-Type: application/json" \ + --request "$method" \ + --data-binary "$payload" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path") + else + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --request "$method" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path") + fi + + [ "$status" = "$expected" ] || fail "$service $method $path: expected HTTP $expected, got $status" + echo "PASS $service $method $path -> $status" +} + +login api +login worker + +expect_status api GET secret/data/messenger/api 200 +expect_status api GET secret/data/messenger/worker 403 +expect_status api POST secret/data/messenger/api/probe 403 '{"data":{"probe":"denied"}}' +expect_status api GET sys/policies/acl 403 +expect_status api POST auth/token/renew-self 200 '{}' + +expect_status worker GET secret/data/messenger/worker 200 +expect_status worker GET secret/data/messenger/api 403 +expect_status worker POST secret/data/messenger/worker/probe 403 '{"data":{"probe":"denied"}}' +expect_status worker GET sys/policies/acl 403 +expect_status worker POST auth/token/renew-self 200 '{}' + +echo "OpenBao AppRole allow/deny and renewal checks passed" + diff --git a/scripts/openbao-unseal.sh b/scripts/openbao-unseal.sh new file mode 100755 index 0000000..ae8586d --- /dev/null +++ b/scripts/openbao-unseal.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command "$BAO_BIN" +require_command python3 +require_file "$BAO_CACERT" +require_private_file "$openbao_operator_file" + +mkdir -p "$openbao_runtime_dir" +keys_file=$(mktemp "$openbao_runtime_dir/.unseal.XXXXXX") +status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") +chmod 0600 "$keys_file" "$status_file" + +cleanup() { + rm -f "$keys_file" "$status_file" +} + +trap cleanup EXIT HUP INT TERM + +openbao_status "$status_file" + +if [ "$(json_field "$status_file" sealed)" = "false" ]; then + echo "OpenBao is already unsealed" + exit 0 +fi + +python3 - "$openbao_operator_file" >"$keys_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + material = json.load(source) + +threshold = material["unseal_threshold"] +for key in material["unseal_keys_b64"][:threshold]: + print(key) +PY + +while IFS= read -r key; do + "$BAO_BIN" operator unseal "$key" /dev/null +done <"$keys_file" + +openbao_status "$status_file" +[ "$(json_field "$status_file" sealed)" = "false" ] || fail "OpenBao remains sealed" + +echo "OpenBao unsealed" + diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 695e3f8..d29cd0b 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then fi manifest=$1 -deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"} +deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"} current_manifest=$deploy_dir/current-images.env active_containers=$deploy_dir/active-api-containers previous_containers=$deploy_dir/previous-api-containers @@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf upstream_backup=$deploy_dir/upstream.previous.conf candidate_file=$deploy_dir/candidate-api-containers replicas=${API_REPLICAS:-2} -project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5} +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7} +public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443} +public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt} switched=false nginx_created=false upstream_changed=false @@ -52,7 +54,7 @@ write_upstream() { : > "$output" for container in $containers; do name=$(container_name "$container") - printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" done } @@ -81,7 +83,8 @@ wait_for_containers() { } smoke_check() { - compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready + curl --fail --silent --show-error --noproxy '*' \ + --cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null } remove_containers() { @@ -123,6 +126,7 @@ trap 'exit 143' TERM load_manifest "$manifest" new_api_image=$API_IMAGE +test -r "$public_ca_file" if [ -f "$previous_containers" ]; then remove_containers "$(cat "$previous_containers")" @@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup" compose pull api nginx swagger prometheus grafana s3 s3-bootstrap compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 +compose run --rm security-bootstrap compose run --rm --no-deps s3-bootstrap compose run --rm --no-deps migrate diff --git a/scripts/scan-security.sh b/scripts/scan-security.sh new file mode 100755 index 0000000..ffd40ad --- /dev/null +++ b/scripts/scan-security.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +runtime_dir="$repo_dir/evidence/runtime/security-scan" +cache_dir="$repo_dir/.runtime/trivy-cache" +raw_file="$runtime_dir/trivy-fs.json" +summary_file="$repo_dir/evidence/lab7-security-scan.json" +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +image_list=${SCAN_IMAGES:-} +if [ -z "$image_list" ]; then + containers=$(docker compose --project-directory "$repo_dir" --env-file "$env_file" ps -q) + [ -n "$containers" ] || { echo "start the stack or set SCAN_IMAGES" >&2; exit 1; } + image_list=$(docker inspect --format '{{.Config.Image}}' $containers | sort -u) +fi + +mkdir -p "$runtime_dir" "$cache_dir" + +docker run --rm \ + -v "$repo_dir:/src:ro" \ + -v "$runtime_dir:/out" \ + -v "$cache_dir:/root/.cache/trivy" \ + aquasec/trivy:0.76.2 fs \ + --scanners vuln,misconfig,secret \ + --format json \ + --output /out/trivy-fs.json \ + --skip-dirs /src/.git \ + --skip-dirs /src/.runtime \ + --skip-dirs /src/.venv \ + --skip-dirs /src/evidence/runtime \ + --skip-files /src/Dockerfile.example \ + /src + +image_reports= +index=0 +for image in $image_list; do + index=$((index + 1)) + report="$runtime_dir/trivy-image-$index.json" + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$runtime_dir:/out" \ + -v "$cache_dir:/root/.cache/trivy" \ + aquasec/trivy:0.76.2 image \ + --scanners vuln,secret \ + --format json \ + --output "/out/trivy-image-$index.json" \ + "$image" + image_reports="$image_reports $image=$report" +done + +python3 - "$raw_file" "$summary_file" $image_reports <<'PY' +import datetime +import json +import sys +from collections import Counter + +with open(sys.argv[1], encoding="utf-8") as source: + report = json.load(source) + +vulnerabilities = Counter() +misconfigurations = Counter() +secrets = 0 +for result in report.get("Results") or []: + for vulnerability in result.get("Vulnerabilities") or []: + vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1 + for finding in result.get("Misconfigurations") or []: + misconfigurations[finding.get("Severity", "UNKNOWN")] += 1 + secrets += len(result.get("Secrets") or []) + +image_vulnerabilities = Counter() +image_secrets = 0 +scanned_images = [] +image_raw_reports = [] +findings = {} + + +def record_findings(report, scope): + for result in report.get("Results") or []: + for finding in result.get("Vulnerabilities") or []: + key = (finding["VulnerabilityID"], finding["PkgName"], finding.get("InstalledVersion", "")) + entry = findings.setdefault(key, { + "id": finding["VulnerabilityID"], + "package": finding["PkgName"], + "installed_version": finding.get("InstalledVersion", ""), + "fixed_version": finding.get("FixedVersion", ""), + "severity": finding.get("Severity", "UNKNOWN"), + "title": finding.get("Title", ""), + "scopes": [], + }) + if scope not in entry["scopes"]: + entry["scopes"].append(scope) + + +record_findings(report, "repository") +for item in sys.argv[3:]: + image, path = item.split("=", 1) + scanned_images.append(image) + with open(path, encoding="utf-8") as source: + image_report = json.load(source) + image_raw_reports.append("evidence/runtime/security-scan/" + path.rsplit("/", 1)[-1]) + record_findings(image_report, image) + for result in image_report.get("Results") or []: + for vulnerability in result.get("Vulnerabilities") or []: + image_vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1 + image_secrets += len(result.get("Secrets") or []) + +summary = { + "schema_version": 1, + "scanner": "Trivy 0.76.2", + "generated_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(), + "scope": "repository filesystem: vulnerabilities, misconfigurations, secrets", + "vulnerabilities": dict(sorted(vulnerabilities.items())), + "misconfigurations": dict(sorted(misconfigurations.items())), + "secret_findings": secrets, + "scanned_images": scanned_images, + "image_vulnerabilities": dict(sorted(image_vulnerabilities.items())), + "image_secret_findings": image_secrets, + "findings": sorted((finding for finding in findings.values() + if "repository" in finding["scopes"] or finding["severity"] == "CRITICAL"), + key=lambda finding: (finding["package"], finding["id"])), + "findings_scope": "all repository findings and critical image findings; full findings in raw reports", + "raw_report": "evidence/runtime/security-scan/trivy-fs.json", + "image_raw_reports": image_raw_reports, +} +with open(sys.argv[2], "w", encoding="utf-8") as destination: + json.dump(summary, destination, ensure_ascii=False, indent=2) + destination.write("\n") +PY + +cat "$summary_file" diff --git a/scripts/test-lab6.sh b/scripts/test-lab6.sh index c2ffb45..4e26b5e 100755 --- a/scripts/test-lab6.sh +++ b/scripts/test-lab6.sh @@ -11,8 +11,9 @@ case "$profile" in ;; esac -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} loki_url=${LOKI_URL:-http://127.0.0.1:3100} @@ -35,6 +36,7 @@ compose config --quiet compose up -d --build --wait --wait-timeout 180 python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode mixed \ --duration "${BASELINE_LOAD_SECONDS:-15}" \ --requests "${BASELINE_REQUESTS:-120}" \ @@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \ --require >/dev/null if [ "$profile" = "full" ]; then - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh fi arguments= diff --git a/scripts/test-leaf-rotation.sh b/scripts/test-leaf-rotation.sh new file mode 100755 index 0000000..c68e779 --- /dev/null +++ b/scripts/test-leaf-rotation.sh @@ -0,0 +1,160 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +security_dir=$repo_dir/.runtime/security +rotation_dir=$security_dir/rotation-test +old_dir=$rotation_dir/old-load-balancer +new_dir=$rotation_dir/new-load-balancer +expired_dir=$rotation_dir/expired-load-balancer +override_file=$repo_dir/.runtime/leaf-rotation.override.yaml +new_identity=spiffe://mtusi/lab7/load-balancer-v2 +rotation_active=0 +base_url=${BASE_URL:-https://localhost:8443} + +compose() { + docker compose --env-file "$env_file" -f "$repo_dir/compose.yaml" "$@" +} + +compose_rotated() { + docker compose --env-file "$env_file" \ + -f "$repo_dir/compose.yaml" -f "$override_file" "$@" +} + +install_security_material() { + compose run --rm --no-deps security-bootstrap >/dev/null +} + +wait_for_https() { + attempts=0 + while [ "$attempts" -lt 30 ]; do + if curl --silent --show-error --fail \ + --cacert "$security_dir/public-ca/ca.crt" \ + "$base_url/health/ready" >/dev/null 2>&1; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 1 + done + return 1 +} + +restore() { + status=$? + trap - EXIT HUP INT TERM + + if [ "$rotation_active" -eq 1 ] && [ -f "$old_dir/tls.crt" ]; then + cp "$old_dir/tls.crt" "$security_dir/load-balancer/tls.crt" + cp "$old_dir/tls.key" "$security_dir/load-balancer/tls.key" + chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key" + install_security_material || status=1 + compose up -d --no-deps --no-build --force-recreate api nginx >/dev/null || status=1 + wait_for_https || status=1 + fi + + rm -rf "$rotation_dir" + rm -f "$override_file" + exit "$status" +} + +trap restore EXIT HUP INT TERM + +[ -f "$env_file" ] || { + echo "environment file not found: $env_file" >&2 + exit 1 +} +[ -f "$security_dir/internal-ca/ca.crt" ] || { + echo "run infra/pki/bootstrap.sh first" >&2 + exit 1 +} + +umask 077 +rm -rf "$rotation_dir" +mkdir -p "$old_dir" "$new_dir" "$expired_dir" +cp "$security_dir/load-balancer/tls.crt" "$old_dir/tls.crt" +cp "$security_dir/load-balancer/tls.key" "$old_dir/tls.key" + +cat >"$rotation_dir/extensions.cnf" </dev/null + rm "$directory/tls.csr" +} + +issue_leaf "$new_dir" 30 +issue_leaf "$expired_dir" 0 + +openssl verify -purpose sslclient \ + -CAfile "$security_dir/internal-ca/ca.crt" "$new_dir/tls.crt" >/dev/null +if openssl verify -purpose sslclient -attime "$(( $(date +%s) + 2 ))" \ + -CAfile "$security_dir/internal-ca/ca.crt" "$expired_dir/tls.crt" >/dev/null 2>&1; then + echo "zero-lifetime certificate unexpectedly passed expiry verification" >&2 + exit 1 +fi + +cat >"$override_file" </dev/null +wait_for_https + +compose_rotated --profile security run --rm --no-deps --entrypoint /bin/sh security-tests -c ' + set -eu + curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/load-balancer/tls.crt \ + --key /src/.runtime/security/load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null + if curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/rotation-test/old-load-balancer/tls.crt \ + --key /src/.runtime/security/rotation-test/old-load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null 2>&1; then + echo "retired load-balancer identity unexpectedly reached API" >&2 + exit 1 + fi + if curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/rotation-test/expired-load-balancer/tls.crt \ + --key /src/.runtime/security/rotation-test/expired-load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null 2>&1; then + echo "expired load-balancer leaf unexpectedly reached API" >&2 + exit 1 + fi +' + +echo "leaf rotation passed: new identity accepted, retired identity rejected, expired leaf rejected" diff --git a/scripts/test-login-rate-limit.py b/scripts/test-login-rate-limit.py new file mode 100644 index 0000000..3db9269 --- /dev/null +++ b/scripts/test-login-rate-limit.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import secrets +import ssl +import urllib.error +import urllib.request + + +def request(context, method, url, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request_headers = dict(headers or {}) + if data is not None: + request_headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, method=method, headers=request_headers) + try: + with urllib.request.urlopen(request, context=context, timeout=10) as response: + return response.status, response.headers + except urllib.error.HTTPError as error: + return error.code, error.headers + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="https://localhost:8443") + parser.add_argument("--ca-file", required=True) + args = parser.parse_args() + + context = ssl.create_default_context(cafile=args.ca_file) + username = "limit_" + secrets.token_hex(6) + password = "Rate-limit-" + secrets.token_urlsafe(18) + status, _ = request(context, "POST", args.base_url + "/api/v1/users", { + "username": username, + "display_name": "Rate limit check", + "password": password, + }) + if status != 201: + raise SystemExit(f"registration returned {status}") + + authorization = "Basic " + base64.b64encode((username + ":wrong-password").encode()).decode() + instances = set() + statuses = [] + retry_after = None + for _ in range(6): + status, headers = request(context, "POST", args.base_url + "/api/v1/auth/sessions", headers={ + "Authorization": authorization, + }) + statuses.append(status) + instances.add(headers.get("X-Instance-Id")) + if status == 429: + retry_after = headers.get("Retry-After") + + if statuses != [401, 401, 401, 401, 401, 429]: + raise SystemExit(f"unexpected statuses: {statuses}") + if not retry_after or int(retry_after) < 1: + raise SystemExit("429 response has no positive Retry-After") + instances.discard(None) + if len(instances) < 2: + raise SystemExit(f"requests reached {len(instances)} API instance") + print("PASS: shared login rate limit returned 429 with Retry-After across two API replicas") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-openbao-lifecycle.sh b/scripts/test-openbao-lifecycle.sh new file mode 100755 index 0000000..0ffd61f --- /dev/null +++ b/scripts/test-openbao-lifecycle.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command curl +require_command docker +require_command python3 +require_command "$BAO_BIN" +require_file "$BAO_CACERT" +require_unsealed +load_operator_token + +test_dir=$(mktemp -d "$openbao_runtime_dir/.lifecycle.XXXXXX") +chmod 0700 "$test_dir" +sealed=0 + +compose() { + docker compose --project-directory "$repo_dir" \ + --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" "$@" +} + +agent_lookup() { + compose exec -T openbao-agent-worker sh -c ' + BAO_ADDR=https://openbao:8200 \ + BAO_CACERT=/run/security/openbao/ca.crt \ + BAO_TOKEN=$(cat /run/openbao-agent/token) \ + bao token lookup >/dev/null 2>&1 + ' +} + +recover() { + status=$? + trap - EXIT HUP INT TERM + if [ "$sealed" -eq 1 ]; then + "$script_dir/openbao-unseal.sh" >/dev/null || status=1 + compose restart openbao-agent-worker >/dev/null || status=1 + attempts=0 + while [ "$attempts" -lt 30 ] && ! agent_lookup; do + attempts=$((attempts + 1)) + sleep 1 + done + agent_lookup || status=1 + fi + rm -rf "$test_dir" + exit "$status" +} + +trap recover EXIT HUP INT TERM + +http_status() { + header_file=$1 + method=$2 + path=$3 + payload=${4:-} + output=$test_dir/response.json + + if [ -n "$payload" ]; then + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$header_file" \ + --header "Content-Type: application/json" \ + --request "$method" \ + --data-binary "$payload" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path" + else + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$header_file" \ + --request "$method" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path" + fi +} + +write_token_header() { + response_file=$1 + header_file=$2 + python3 - "$response_file" "$header_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + document = json.load(source) +token = document.get("auth", document)["client_token"] +with open(sys.argv[2], "w", encoding="utf-8") as destination: + destination.write(f"X-Vault-Token: {token}\n") +os.chmod(sys.argv[2], 0o600) +PY +} + +issue_login_material() { + name=$1 + identity_dir=$test_dir/$name-identity + payload_file=$test_dir/$name-login.json + + "$script_dir/openbao-issue-approle.sh" worker "$identity_dir" >/dev/null + python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + role_id = source.read().strip() +with open(sys.argv[2], encoding="utf-8") as source: + secret_id = source.read().strip() +with open(sys.argv[3], "w", encoding="utf-8") as destination: + json.dump({"role_id": role_id, "secret_id": secret_id}, destination) +os.chmod(sys.argv[3], 0o600) +PY + printf '%s\n' "$payload_file" +} + +login_payload=$(issue_login_material renewable) +login_response=$test_dir/renewable-login-response.json +login_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$login_payload" \ + --output "$login_response" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") +[ "$login_status" = 200 ] || fail "AppRole login returned HTTP $login_status" +chmod 0600 "$login_response" +renewable_header=$test_dir/renewable-token-header +write_token_header "$login_response" "$renewable_header" + +[ "$(http_status "$renewable_header" GET secret/data/messenger/worker)" = 200 ] || fail "worker token cannot read own secret" +[ "$(http_status "$renewable_header" POST auth/token/renew-self '{}')" = 200 ] || fail "worker token renewal failed" +[ "$(http_status "$renewable_header" POST auth/token/revoke-self)" = 204 ] || fail "worker token self-revocation failed" +[ "$(http_status "$renewable_header" GET auth/token/lookup-self)" = 403 ] || fail "revoked token remained valid" + +short_response=$test_dir/short-token-response.json +"$BAO_BIN" token create \ + -policy=messenger-worker \ + -no-default-policy \ + -ttl=2s \ + -explicit-max-ttl=2s \ + -format=json >"$short_response" +chmod 0600 "$short_response" +short_header=$test_dir/short-token-header +write_token_header "$short_response" "$short_header" +[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 200 ] || fail "short-lived token cannot read own secret" +sleep 3 +[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 403 ] || fail "expired token remained valid" + +sealed_login_payload=$(issue_login_material sealed) +"$BAO_BIN" operator seal >/dev/null +sealed=1 + +if agent_lookup; then + fail "Agent token lookup succeeded while OpenBao was sealed" +fi +sealed_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$sealed_login_payload" \ + --output "$test_dir/sealed-login-response.json" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") +[ "$sealed_status" = 503 ] || fail "sealed AppRole login: expected HTTP 503, got $sealed_status" + +"$script_dir/openbao-unseal.sh" >/dev/null +compose restart openbao-agent-worker >/dev/null +attempts=0 +while [ "$attempts" -lt 30 ] && ! agent_lookup; do + attempts=$((attempts + 1)) + sleep 1 +done +agent_lookup || fail "Agent did not authenticate after unseal" +sealed=0 + +echo "OpenBao lifecycle passed: renew, revoke, expiry, seal denial and Agent re-authentication" diff --git a/scripts/test-tls-matrix.sh b/scripts/test-tls-matrix.sh new file mode 100755 index 0000000..a88a8bb --- /dev/null +++ b/scripts/test-tls-matrix.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security} +public_port=${PUBLIC_TLS_PORT:-8443} +public_host=${PUBLIC_TLS_HOST:-localhost} +api_address=${API_TLS_ADDRESS:-api:8443} +metrics_address=${METRICS_TLS_ADDRESS:-api:9091} +empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$" + +cleanup() { + rm -f "$empty_ca" +} + +trap cleanup EXIT HUP INT TERM + +require_file() { + if [ ! -r "$1" ]; then + echo "missing security material: $1" >&2 + exit 1 + fi +} + +expect_failure() { + name=$1 + shift + + if "$@" >/dev/null 2>&1; then + echo "FAIL: $name unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: $name" +} + +expect_tls_success() { + name=$1 + certificate=$2 + key=$3 + + if curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$certificate" --key "$key" \ + "https://$api_address/health/ready" >/dev/null 2>&1; then + echo "PASS: $name" + return + fi + echo "FAIL: $name" >&2 + exit 1 +} + +expect_tls_failure() { + name=$1 + shift + + if curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + "$@" "https://$api_address/health/ready" >/dev/null 2>&1; then + echo "FAIL: $name unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: $name" +} + +expect_public_hostname_failure() { + if openssl s_client \ + -connect "$public_host:$public_port" \ + -servername wrong.local \ + -verify_hostname wrong.local \ + -verify_return_error \ + -CAfile "$security_dir/public-ca/ca.crt" /dev/null 2>&1; then + echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: public TLS with wrong hostname" +} + +for file in \ + "$security_dir/public-ca/ca.crt" \ + "$security_dir/internal-ca/ca.crt" \ + "$security_dir/foreign-ca/ca.crt" \ + "$security_dir/load-balancer/tls.crt" \ + "$security_dir/load-balancer/tls.key" \ + "$security_dir/prometheus/tls.crt" \ + "$security_dir/prometheus/tls.key" \ + "$security_dir/forbidden-client/tls.crt" \ + "$security_dir/forbidden-client/tls.key" \ + "$security_dir/foreign-client/tls.crt" \ + "$security_dir/foreign-client/tls.key"; do + require_file "$file" +done + +: >"$empty_ca" +chmod 0600 "$empty_ca" + +curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/public-ca/ca.crt" \ + "https://$public_host:$public_port/health/ready" >/dev/null +echo "PASS: public TLS with trusted CA and hostname" + +expect_failure "public TLS with empty CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$empty_ca" \ + "https://$public_host:$public_port/health/ready" + +expect_failure "public TLS with foreign CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/foreign-ca/ca.crt" \ + "https://$public_host:$public_port/health/ready" + +expect_public_hostname_failure + +expect_tls_failure "API mTLS without client certificate" +expect_tls_failure "API mTLS with forbidden trusted identity" \ + --cert "$security_dir/forbidden-client/tls.crt" \ + --key "$security_dir/forbidden-client/tls.key" +expect_tls_failure "API mTLS with client certificate from foreign CA" \ + --cert "$security_dir/foreign-client/tls.crt" \ + --key "$security_dir/foreign-client/tls.key" +expect_tls_success "API mTLS with load-balancer identity" \ + "$security_dir/load-balancer/tls.crt" \ + "$security_dir/load-balancer/tls.key" + +expect_tls_failure "business API with Prometheus identity" \ + --cert "$security_dir/prometheus/tls.crt" \ + --key "$security_dir/prometheus/tls.key" + +expect_failure "metrics mTLS without client certificate" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + "https://$metrics_address/metrics" + +expect_failure "metrics mTLS with load-balancer identity" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/load-balancer/tls.crt" \ + --key "$security_dir/load-balancer/tls.key" \ + "https://$metrics_address/metrics" + +expect_failure "metrics mTLS with client certificate from foreign CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/foreign-client/tls.crt" \ + --key "$security_dir/foreign-client/tls.key" \ + "https://$metrics_address/metrics" + +curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/prometheus/tls.crt" \ + --key "$security_dir/prometheus/tls.key" \ + "https://$metrics_address/metrics" >/dev/null +echo "PASS: metrics mTLS with Prometheus identity" -- 2.54.0 From a8edaf058dc20e3e47fe2afa63d88d1ebe402c08 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 6 Nov 2026 21:18:51 +0300 Subject: [PATCH 5/5] chore: finalize lab7 security verification --- .gitea/workflows/ci.yml | 54 +++- .gitea/workflows/deploy.yml | 29 +- .gitea/workflows/incidents.yml | 26 +- .gitignore | 1 - REPORT.md | 189 +++++++++-- cmd/loginburst/main.go | 175 +++++++++++ cmd/messenger/main.go | 11 +- compose.yaml | 2 + config/messenger.yaml | 1 + contracts/trust-matrix.md | 58 ++-- evidence/lab7-argon2-budget.json | 28 ++ evidence/lab7-security-checks.json | 19 ++ evidence/lab7-security-scan.json | 331 ++++++++++++++++++++ evidence/lab7-worker-compromise.json | 12 + infra/openbao/README.md | 6 +- infra/openbao/agent/api-secrets.ctmpl | 1 - infra/openbao/agent/api.hcl | 1 - infra/openbao/agent/worker-secrets.ctmpl | 1 - infra/openbao/agent/worker.hcl | 1 - infra/openbao/config.hcl | 1 - infra/openbao/policies/messenger-api.hcl | 1 - infra/openbao/policies/messenger-worker.hcl | 1 - internal/bootstrap/pipeline.go | 4 +- internal/bootstrap/server.go | 11 +- internal/config/config.go | 102 +++--- internal/config/config_test.go | 2 + internal/config/loader.go | 2 +- internal/messenger/httpapi/attachments.go | 17 - internal/messenger/httpapi/auth.go | 10 +- internal/messenger/httpapi/errors.go | 10 + internal/messenger/httpapi/handlers.go | 2 +- internal/messenger/httpapi/server.go | 4 + internal/messenger/httpapi/server_test.go | 17 + internal/password/password.go | 37 ++- internal/password/password_test.go | 30 +- internal/postgres/attachments.go | 15 - internal/security/mtlsproxy/proxy.go | 6 +- internal/security/tlsconfig/server_test.go | 2 +- scripts/bao-docker.sh | 2 + scripts/bootstrap-local.sh | 11 +- scripts/lab7-worker-compromise.sh | 6 +- scripts/openbao-bootstrap.sh | 1 - scripts/openbao-common.sh | 5 +- scripts/openbao-init.sh | 1 - scripts/openbao-issue-approle.sh | 2 +- scripts/openbao-rotate.sh | 1 - scripts/openbao-test-policy.sh | 1 - scripts/openbao-unseal.sh | 1 - scripts/scan-security.sh | 3 + scripts/stop-one.sh | 3 +- scripts/sync-docker-workspace.sh | 51 +++ scripts/test-leaf-rotation.sh | 1 + security/THREAT_MODEL.md | 45 ++- 53 files changed, 1130 insertions(+), 224 deletions(-) create mode 100644 cmd/loginburst/main.go create mode 100644 evidence/lab7-argon2-budget.json create mode 100644 evidence/lab7-security-checks.json create mode 100644 evidence/lab7-security-scan.json create mode 100644 evidence/lab7-worker-compromise.json create mode 100755 scripts/sync-docker-workspace.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 36db92a..77aa40d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -24,6 +24,9 @@ jobs: LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 OPENBAO_BIND_ADDRESS: 127.0.0.1:0 + BAO_ADDR: https://openbao:8200 + DOCKER_WORKSPACE_MIRROR: "1" + CI_WORKSPACE: /tmp/mtusi2026lab7-${{ github.run_id }} steps: - name: Checkout exact commit run: | @@ -34,7 +37,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: go.mod + go-version: "1.26.10" cache: false - name: Check contract @@ -66,8 +69,16 @@ jobs: - name: Prepare local stack run: | + mkdir -p "$CI_WORKSPACE" + git archive HEAD | tar -x -C "$CI_WORKSPACE" + cd "$CI_WORKSPACE" cp lab7.env.example .env docker compose config --quiet + ./infra/pki/bootstrap.sh + ./scripts/openbao-seed.sh .env + ./scripts/sync-docker-workspace.sh + docker compose up -d security-bootstrap openbao-permissions openbao + docker network connect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" @@ -78,15 +89,15 @@ jobs: - name: Integration checks run: | + cd "$CI_WORKSPACE" test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-tls-matrix ENV_FILE=.env ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh docker compose --profile security run --rm dependency-tests - OPENBAO_ADDRESS=$(docker compose port openbao 8200) - ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh - ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + ENV_FILE=.env BAO_ADDR=https://openbao:8200 BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR=https://openbao:8200 BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt @@ -97,25 +108,36 @@ jobs: make test-worker-compromise ENV_FILE=.env make scan-security ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 - - - name: Upload security evidence - if: always() - uses: actions/upload-artifact@v4 - with: - name: lab7-security-${{ github.sha }} - path: | - evidence/lab7-*.json - evidence/runtime/security-scan - if-no-files-found: warn - retention-days: 14 + mkdir -p evidence/runtime + api_restart_counts=$(docker inspect --format '{{.RestartCount}}' $(docker compose ps -q api)) + go run ./cmd/loginburst -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt > evidence/runtime/login-burst.json + test "$(docker inspect --format '{{.RestartCount}}' $(docker compose ps -q api))" = "$api_restart_counts" - name: Container logs if: failure() - run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + run: if [ -f "$CI_WORKSPACE/.env" ]; then cd "$CI_WORKSPACE"; docker compose logs --no-color --tail=200; fi + + - name: Upload security evidence + if: always() + uses: actions/upload-artifact@v3 + env: + ACTIONS_RUNTIME_URL: https://gitea.privcloud.tech/api/actions_pipeline/ + with: + name: lab7-security-${{ github.sha }} + path: | + ${{ env.CI_WORKSPACE }}/evidence/lab7-*.json + ${{ env.CI_WORKSPACE }}/evidence/runtime/security-scan + ${{ env.CI_WORKSPACE }}/evidence/runtime/login-burst.json + if-no-files-found: warn + retention-days: 14 - name: Stop local stack if: always() run: | + test -d "$CI_WORKSPACE" || exit 0 + cd "$CI_WORKSPACE" + trap './scripts/sync-docker-workspace.sh --clean' EXIT + docker network disconnect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 7fdf0aa..609de2a 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -31,6 +31,9 @@ jobs: LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 OPENBAO_BIND_ADDRESS: 127.0.0.1:0 + BAO_ADDR: https://openbao:8200 + DOCKER_WORKSPACE_MIRROR: "1" + CI_WORKSPACE: /tmp/mtusi2026lab7-${{ github.run_id }} API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-api:${{ github.sha }} NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-nginx:${{ github.sha }} SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-swagger:${{ github.sha }} @@ -48,7 +51,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: go.mod + go-version: "1.26.10" cache: false - name: Check contract @@ -80,8 +83,16 @@ jobs: - name: Prepare local stack run: | + mkdir -p "$CI_WORKSPACE" + git archive HEAD | tar -x -C "$CI_WORKSPACE" + cd "$CI_WORKSPACE" cp lab7.env.example .env docker compose config --quiet + ./infra/pki/bootstrap.sh + ./scripts/openbao-seed.sh .env + ./scripts/sync-docker-workspace.sh + docker compose up -d security-bootstrap openbao-permissions openbao + docker network connect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" @@ -92,15 +103,15 @@ jobs: - name: Integration checks run: | + cd "$CI_WORKSPACE" test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-tls-matrix ENV_FILE=.env ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh docker compose --profile security run --rm dependency-tests - OPENBAO_ADDRESS=$(docker compose port openbao 8200) - ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh - ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + ENV_FILE=.env BAO_ADDR=https://openbao:8200 BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR=https://openbao:8200 BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt @@ -109,6 +120,10 @@ jobs: make test-lab5-acceptance make test-worker-compromise ENV_FILE=.env ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 + mkdir -p evidence/runtime + api_restart_counts=$(docker inspect --format '{{.RestartCount}}' $(docker compose ps -q api)) + go run ./cmd/loginburst -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt > evidence/runtime/login-burst.json + test "$(docker inspect --format '{{.RestartCount}}' $(docker compose ps -q api))" = "$api_restart_counts" - name: Publish verified images id: publish @@ -139,11 +154,15 @@ jobs: - name: Container logs if: failure() - run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + run: if [ -f "$CI_WORKSPACE/.env" ]; then cd "$CI_WORKSPACE"; docker compose logs --no-color --tail=200; fi - name: Stop local stack if: always() run: | + test -d "$CI_WORKSPACE" || exit 0 + cd "$CI_WORKSPACE" + trap './scripts/sync-docker-workspace.sh --clean' EXIT + docker network disconnect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true diff --git a/.gitea/workflows/incidents.yml b/.gitea/workflows/incidents.yml index 57c4b52..ab49dae 100644 --- a/.gitea/workflows/incidents.yml +++ b/.gitea/workflows/incidents.yml @@ -20,6 +20,9 @@ jobs: LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 OPENBAO_BIND_ADDRESS: 127.0.0.1:0 + BAO_ADDR: https://openbao:8200 + DOCKER_WORKSPACE_MIRROR: "1" + CI_WORKSPACE: /tmp/mtusi2026lab7-${{ github.run_id }} steps: - name: Checkout exact commit run: | @@ -30,13 +33,21 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: go.mod + go-version: "1.26.10" cache: false - name: Prepare stack run: | + mkdir -p "$CI_WORKSPACE" + git archive HEAD | tar -x -C "$CI_WORKSPACE" + cd "$CI_WORKSPACE" cp lab7.env.example .env docker compose config --quiet + ./infra/pki/bootstrap.sh + ./scripts/openbao-seed.sh .env + ./scripts/sync-docker-workspace.sh + docker compose up -d security-bootstrap openbao-permissions openbao + docker network connect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" @@ -47,6 +58,7 @@ jobs: - name: Reproduce SLO incidents run: | + cd "$CI_WORKSPACE" ENV_FILE=.env \ BASE_URL=https://nginx:8443 \ CA_FILE=.runtime/security/public-ca/ca.crt \ @@ -58,20 +70,26 @@ jobs: - name: Upload sanitized evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 + env: + ACTIONS_RUNTIME_URL: https://gitea.privcloud.tech/api/actions_pipeline/ with: name: lab6-incidents-${{ github.sha }} - path: evidence/runtime/ci + path: ${{ env.CI_WORKSPACE }}/evidence/runtime/ci if-no-files-found: warn retention-days: 14 - name: Container logs if: failure() - run: docker compose logs --no-color --tail=200 + run: if [ -f "$CI_WORKSPACE/.env" ]; then cd "$CI_WORKSPACE"; docker compose logs --no-color --tail=200; fi - name: Stop local stack if: always() run: | + test -d "$CI_WORKSPACE" || exit 0 + cd "$CI_WORKSPACE" + trap './scripts/sync-docker-workspace.sh --clean' EXIT + docker network disconnect "${COMPOSE_PROJECT_NAME}_secrets" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true diff --git a/.gitignore b/.gitignore index d7b71d6..66ededd 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,3 @@ coverage/ # Inspect git diff --cached: ignore rules are not a secret scanner. evidence/runtime/ -.runtime/ diff --git a/REPORT.md b/REPORT.md index 9801db4..01ec188 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,186 @@ -# Отчёт — лабораторная 7 +# Отчёт — лабораторная работа № 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Автор:** Пластинин Денис, БВТ2303 + +**Целевая оценка:** 5 + +**Выбранный трек:** 1 — компрометация worker + +## Цель работы + +Цель работы — провести границы доверия в ранее реализованном мессенджере, включить проверяемое TLS/mTLS взаимодействие, разделить права сервисов и вынести runtime secrets в OpenBao. За основу взята лабораторная работа № 6: сохранены API, обработка вложений через Redis Streams, PostgreSQL, MinIO и стек наблюдаемости. + +Внешний клиент подключается только к Nginx по HTTPS. Nginx обращается к двум API-репликам по mTLS, причём API принимает только сертификат с разрешённым URI SAN балансировщика. Внутренние зависимости используют TLS, отдельные service identities и минимальные ACL. Наличие сертификата, подписанного доверенным CA, само по себе не даёт доступ: identity дополнительно сопоставляется с разрешённой ролью. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker с Compose v2, `make`, Python 3 и `curl`. Все используемые образы закреплены по версиям в `compose.yaml`. Локальные ключи и operator material создаются в игнорируемой директории `.runtime`. -## Архитектура и решения +```sh +cp lab7.env.example .env +./scripts/bootstrap-local.sh .env +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Скрипт создаёт локальную PKI, запускает OpenBao с Raft storage, при первом запуске выполняет `init`/`unseal`, создаёт policies и AppRole, затем поднимает стенд с двумя API-репликами. Повторный запуск использует сохранённое состояние OpenBao и выполняет только необходимый bootstrap. + +Основные интерфейсы: + +- API и Swagger через HTTPS: `https://localhost:8443` и `http://localhost:8081`; +- Grafana: `https://localhost:3000`; +- Prometheus: `http://localhost:9090`; +- OpenBao operator endpoint: `https://localhost:8200`. + +Публичный CA создаётся в `.runtime/security/public-ca/ca.crt`. Проверка API: + +```sh +make test BASE_URL=https://localhost:8443 CA_FILE="$PWD/.runtime/security/public-ca/ca.crt" +make test-security BASE_URL=https://localhost:8443 CA_FILE="$PWD/.runtime/security/public-ca/ca.crt" +``` + +Безопасная остановка сохраняет volumes: + +```sh +docker compose --env-file .env stop +``` + +Намеренное удаление учебного стенда и его данных: + +```sh +docker compose --env-file .env down --volumes --remove-orphans +rm -rf .runtime +``` + +Root token и unseal key находятся только в `.runtime/openbao/operator/init.json` с правами `0600`. Их хранит оператор; приложения и CI runtime их не получают. В production этот файл должен находиться во внешнем защищённом хранилище, а unseal shares должны быть разделены между операторами. + +## Архитектура + +```mermaid +flowchart LR + C[Client] -->|HTTPS + Bearer| N[Nginx] + N -->|mTLS| A1[API 1] + N -->|mTLS| A2[API 2] + A1 & A2 -->|TLS + api role| P[(PostgreSQL)] + A1 & A2 -->|mTLS + ACL| V[(Valkey)] + A1 & A2 -->|HTTPS + staging policy| S[(MinIO)] + D[Dispatcher] -->|mTLS producer| R[(Redis Streams)] + R -->|mTLS consumer| W[Worker] + W -->|HTTPS + attachment policy| S + A1 & A2 & D & W -->|mTLS| O[OTel gateway] + BA[OpenBao Agents] -->|TLS + AppRole| B[(OpenBao Raft)] + BA -->|secret file 0600| A1 & A2 & W +``` + +PostgreSQL остаётся источником истины для пользователей, чатов, сообщений, metadata вложений, outbox и состояния заданий. MinIO хранит содержимое вложений. Redis Streams переносит задания с гарантией at-least-once, а Valkey хранит session token hashes и общие счётчики login rate limit. Дедупликация и переходы состояний выполняются транзакционно в PostgreSQL. + +OpenBao работает в non-dev режиме на persistent Raft storage. API и worker имеют разные AppRole и KV policies. Agent аутентифицируется короткоживущим SecretID, обновляет renewable token и рендерит JSON secret в защищённый Docker volume; приложение получает путь через `MESSENGER_SECRETS_FILE`. Token Agent хранится в tmpfs, root token не используется приложением. SecretID действует один час и позволяет повторную аутентификацию после рестарта; затем оператор выдаёт новый. + +PostgreSQL использует `verify-full` и отдельные роли API, dispatcher, worker, reconciler, exporter и migrator. Valkey и Redis Streams защищены native TLS, ACL и Envoy gateway, который проверяет точный URI SAN клиента. MinIO использует HTTPS и отдельные scoped users. OTel ingest и metrics endpoints требуют mTLS с allowlist identities. Полная схема связей приведена в `contracts/trust-matrix.md`. + +Контейнеры приложения запускаются non-root, с `read_only`, `no-new-privileges`, `cap_drop: ALL`, отдельным `tmpfs` и лимитами CPU/RAM. Stateful компоненты сохраняют данные в named volumes; их исключения из `read_only` ограничены каталогами данных. + +## Пароли, сессии и защита login + +Пароли хешируются Argon2id с индивидуальной солью и сохранёнными параметрами. При успешном входе `NeedsRehash` сравнивает параметры хеша с текущей политикой; устаревший хеш обновляется compare-and-swap запросом, поэтому параллельный login не перезаписывает более новый пароль. + +Параметры `m=65536 KiB`, `t=3`, `p=2` измерены в Linux-контейнере с лимитом 2 CPU и 512 MiB. Пять операций дали около 60,0 мс на hash и 64,8 мс на verify при примерно 64 MiB памяти на операцию. Частотный лимит ограничивает login, а общий semaphore внутри Hasher допускает не более двух `Hash`/`Verify` на API-реплику, включая регистрацию и rehash. Поэтому одновременно занятая Argon2 память ограничена примерно 128 MiB на реплику; лишние операции сразу получают `429 Retry-After: 1`. Лимит задаётся `server.password.max_concurrent_operations`. Raw summary сохранён в `evidence/lab7-argon2-budget.json`. + +`cmd/loginburst` создаёт одновременные волны по 20 регистраций и login-запросов через Nginx, проверяет ответы `429` с положительным `Retry-After`, отсутствие `5xx` и readiness после нагрузки. CI дополнительно проверяет, что API containers не перезапускались. `X-Instance-Id` — случайный alias на время жизни процесса; hostname и IP в него не входят. + +Session token является случайным opaque bearer token. В Valkey сохраняется только его SHA-256 hash с TTL; logout удаляет запись. Login limiter общий для API-реплик: Lua script атомарно увеличивает счётчики по нормализованному login и адресу источника, выставляет TTL и возвращает время до повторной попытки. При превышении лимита API отвечает `429` и положительным `Retry-After`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт и OpenAPI | `make check`, `make validate` | структура и OpenAPI 3.1 проходят проверку | CI `verify` | +| Unit и integration | `go test ./...`, PostgreSQL/Valkey integration tests | тесты проходят по TLS connections | CI и test output | +| Публичный API через HTTPS | `make test ... CA_FILE=...` | полный smoke проходит через Nginx | public smoke output | +| Недоверенный внешний TLS | `make test-security` | правильный CA проходит, пустой/чужой CA отклонён | security test output | +| LB→API mTLS | `make test-tls-matrix` | 13 положительных и отрицательных cases проходят | TLS matrix output | +| DB/S3/broker/OTel least privilege | `infra/security/test-dependency-matrix.sh` | разрешённые операции проходят, cross-role операции отклонены | dependency matrix output | +| OpenBao scoped policy | `scripts/openbao-test-policy.sh` | own read `200`, cross/write/policy `403`, token renew `200` | policy test output | +| OpenBao expiry/revoke/seal | `make test-openbao-lifecycle` | revoked/expired token `403`, sealed login `503`, Agent восстановил аутентификацию после unseal | lifecycle test output | +| Ротация leaf и отзыв identity | `make test-leaf-rotation` | новая LB identity проходит, прежняя same-CA identity и expired leaf отклонены на новых connections | rotation test output | +| Общий login limiter | `make test-login-rate-limit ...` | запросы через обе replicas завершаются `429` с `Retry-After` | rate-limit test output | +| Rehash Argon2id | `go test ./internal/password ./internal/postgres ./internal/messenger/httpapi` | проверены `NeedsRehash` и конкурентное CAS update | unit/integration tests | +| Argon2/login DoS budget | `go test ./internal/password -run '^$' -bench 'BenchmarkArgon2id(Hash\|Verify)$' -benchtime=5x -benchmem` | hash 60,0 мс, verify 64,8 мс, около 64 MiB/op при 2 CPU | `evidence/lab7-argon2-budget.json` | +| Dependency/image/security scan | `make scan-security` | secrets не найдены; исправляемые high/critical Go CVE обновлены | `evidence/lab7-security-scan.json` | +| Компрометация worker | `make test-worker-compromise` | старый broker credential отклонён, после rotation queued attachment обработан без потери | `evidence/lab7-worker-compromise.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперимент: компрометация worker -## Эксперименты и отказы +### Гипотеза -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Если worker credential будет скомпрометирован, злоумышленник получит только consumer operations своего stream, ограниченные таблицы PostgreSQL и два MinIO prefix. После смены broker password и worker secret старая identity перестанет работать, а уже поставленное задание сохранится и завершится после восстановления worker. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +### Методика + +Стенд запускался в контейнерах с лимитом worker `2 CPU / 1 GiB`. Скрипт `scripts/lab7-worker-compromise.sh` останавливает worker, создаёт реальное задание загрузки через API, меняет Redis ACL password, не обновляя secret работающего worker, и проверяет отказ старой identity. Затем он записывает новый secret в OpenBao, пересоздаёт OpenBao Agent и worker и проверяет созданное вложение через API. В `trap` всегда восстанавливается исходный учебный secret. + +Отдельная dependency matrix моделирует возможности скомпрометированного worker: ему разрешены чтение/изменение attachment job state, consumer/ack операции stream и работа с `staging/`/`attachments/`; запрещены таблица `users`, unrelated Redis streams и произвольный S3 prefix. + +### Изменение blast radius относительно lab6 + +| Ресурс | До разделения прав в lab6 | После least privilege в lab7 | +| --- | --- | --- | +| PostgreSQL | API, dispatcher, worker и reconciler получали общий `DATABASE_URL`; worker мог использовать права основной роли на пользователей, чаты и сообщения | `messenger_worker` читает/обновляет только attachments/jobs/variants, создаёт audit/outbox; чтение `users` запрещено | +| MinIO | один `S3_ACCESS_KEY` имел `Get/Put/Delete` на весь bucket | worker видит только `staging/*` и `attachments/*`; `private/*` и управление users/policies запрещены | +| Redis Streams | dispatcher и worker использовали общий `BROKER_URL` | dispatcher может только `XADD` в `attachment-jobs`; worker может consumer/ack/recovery команды только для job/DLQ streams | +| TLS identity | внутренние зависимости принимали сетевое соединение без service URI проверки | gateway сопоставляет worker certificate с `spiffe://mtusi/lab7/worker`; same-CA чужая identity отклоняется | +| Secrets | все static credentials находились в общем `.env` процесса | worker получает отдельный JSON через scoped OpenBao AppRole; API policy не читает worker path | + +Таким образом, до разделения кража worker environment давала общий доступ к данным приложения и всему bucket. После разделения она ограничена обработкой вложений. Ротация в эксперименте меняет broker, PostgreSQL и MinIO credentials внутри worker secret; старые credentials не открывают новое соединение даже при наличии прежнего leaf certificate. + +### Результат + +| Воздействие | Ожидание | Наблюдение | Потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Старый broker password после ротации | authentication error | worker получил `WRONGPASS/invalid username-password` | нет | +| Cross-role DB/S3/stream operations | ACL denial | все отрицательные cases dependency matrix отклонены | нет | +| Обновление worker secret в OpenBao | Agent рендерит новый secret, worker становится healthy | от начала сценария до проверки вложения — 21 секунда; возврат исходных паролей измеряется отдельно | нет | +| Проверка задания, созданного до отзыва | вложение обработано после восстановления | API verification прошла | queued attachment сохранён | -## Ограничения и следующие шаги +Эксперимент доказывает ограничение blast radius и восстановление задания, но не моделирует кражу root-доступа к Docker host. Сценарий разрывает broker connections перезапуском gateway и завершает PostgreSQL sessions роли worker через `pg_terminate_backend`; затем проверяет новые подключения со старыми паролями и отказ старому S3 key. Для production потребуются внешний KMS/HSM для CA, response-wrapped SecretID, автоматизированный revoke и несколько независимых повторов с измерением времени восстановления. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Scan triage + +`scripts/scan-security.sh` запускает Trivy filesystem scan и image scan всех образов работающего Compose-стенда. Raw JSON сохраняется в игнорируемом `evidence/runtime/security-scan` и CI artifact, компактная сводка — в `evidence/lab7-security-scan.json`. Найденные у приложения CVE устранены обновлением OpenTelemetry, gRPC, `x/net`, `klauspost/compress`, Go toolchain и Nginx. Весь стенд не объявляется свободным от CVE: сканирование сторонних образов находит отдельные уязвимые компоненты. Secrets в repository/images не найдены. + +| Finding / класс | Применимость и решение | +| --- | --- | +| `CVE-2026-81871`, `CVE-2026-81872`, `CVE-2026-81870` в OTLP | Затрагивают используемые приложением exporters/log SDK. Обновлены до OTLP `1.45.0` / log `0.21.0`; отключение TLS verification не используется | +| `CVE-2026-33322`, `CVE-2026-33419` в MinIO | Требуют OIDC/LDAP auth, которые в стенде не настроены; используется локальная scoped access-key auth. Образ не опубликован наружу. Обновление MinIO остаётся задачей перед production | +| `CVE-2026-77405`, `CVE-2026-77408`, `CVE-2026-77411` в AMQP library сторонних images | AMQP receivers/exporters и MinIO AMQP notifications не настроены; брокер стенда — Redis Streams. Пакет присутствует в бинарнике, но этот путь не используется конфигурацией | +| `CVE-2026-31789` в OpenSSL | Advisory относится к 32-bit обработке X.509; измеренный стенд `linux/arm64`. Не экстраполируем это исключение на остальные CVE OpenSSL | +| `CVE-2026-111497` (Brotli), `CVE-2026-48930` (Node.js) в UI images | Grafana/Swagger доступны только на loopback; приложение Go не использует эти пакеты. Достижимость уязвимых UI-путей не доказана; фиксированные upstream images нужны перед внешней публикацией | +| `CVE-2025-68121` и остальные stdlib/OS findings сторонних images | Собственный API пересобран patched Go. Для сторонних бинарников пересборка не выполнялась; это остаточный риск, который не снимается одной сетевой изоляцией. Ограничиваем доступ и ресурсы, сохраняем raw findings для обновления images | +| `UNKNOWN`, например `CVE-2026-39824` в Windows API | Severity не определена источником. Windows-only путь не выполняется в Linux; остальные UNKNOWN требуют отдельной проверки, а не считаются автоматически безопасными | + +Данный scan triage подтверждает применимость конкретных ограничений учебного стенда. Он не заменяет обновление сторонних образов и не является разрешением на production deployment. + +Замечания к отсутствию `HEALTHCHECK` внутри отдельных toolbox/bootstrap images не дают обхода runtime health: Compose задаёт healthchecks долговечным сервисам, а bootstrap containers одноразовые и должны завершиться с кодом 0. Это остаётся осознанным ограничением, а не исключением проверки приложения. + +## Ограничения + +- PKI локальная и не содержит CRL/OCSP. Отзыв leaf выполняется удалением доверия/сменой allowlist и пересозданием соединений. +- PostgreSQL, Valkey, Redis, MinIO и OpenBao запущены по одному экземпляру; работа проверяет безопасность границ, а не HA этих хранилищ. +- Operator init/unseal ручной. Это допустимо для учебного стенда, но production требует auto-unseal через KMS/HSM и раздельного хранения shares. +- Static S3/DB/broker credentials выдаются через OpenBao KV и ротируются оператором. Dynamic database credentials были бы следующим улучшением. +- Rate limiter хранится в Valkey и при его отказе блокирует login; уже выданные сессии также зависят от Valkey. + +## Источники + +- [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) +- [OWASP File Upload Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/File_Upload_Cheat_Sheet.html) +- [OpenBao AppRole](https://openbao.org/docs/auth/approle/) +- [OpenBao policies](https://openbao.org/docs/concepts/policies/) +- [OpenBao production hardening](https://openbao.org/docs/internals/security/) +- [PostgreSQL SSL support](https://www.postgresql.org/docs/current/libpq-ssl.html) +- [Redis TLS](https://redis.io/docs/latest/operate/oss_and_stack/management/security/encryption/) +- [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) +- [MinIO policy based access control](https://min.io/docs/minio/linux/administration/identity-access-management/policy-based-access-control.html) +- [OpenTelemetry Collector security](https://opentelemetry.io/docs/security/config-best-practices/) +- [Docker Engine security](https://docs.docker.com/engine/security/) +- [Trivy filesystem scanning](https://trivy.dev/latest/docs/target/filesystem/) + +OpenAI Codex использовался для подготовки и ревью реализации; результаты проверены тестами, CI-сценариями и воспроизводимыми отрицательными экспериментами. diff --git a/cmd/loginburst/main.go b/cmd/loginburst/main.go new file mode 100644 index 0000000..520a576 --- /dev/null +++ b/cmd/loginburst/main.go @@ -0,0 +1,175 @@ +package main + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type result struct { + status int + retryAfter string + instance string + duration time.Duration + err error +} + +type phase struct { + Name string `json:"name"` + Requests int `json:"requests"` + Statuses map[int]int `json:"statuses"` + DurationMS int64 `json:"duration_ms"` + P95MS int64 `json:"p95_ms"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + var ( + baseURL = flag.String("base-url", "https://localhost:8443", "API endpoint") + caFile = flag.String("ca-file", ".runtime/security/public-ca/ca.crt", "trusted public CA") + count = flag.Int("requests", 20, "concurrent requests per phase") + ) + flag.Parse() + if *count < 8 || *count > 100 { + return errors.New("requests must be between 8 and 100") + } + ca, err := os.ReadFile(*caFile) + if err != nil { + return err + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca) { + return errors.New("invalid public CA") + } + transport := &http.Transport{ + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots}, + DisableKeepAlives: true, MaxConnsPerHost: *count, + } + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + username := fmt.Sprintf("burst_%d", time.Now().UnixNano()) + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": "Login burst", "password": "T3st-burst-password-123", + }) + if err != nil { + return err + } + register, err := burst(client, *count, "registration", func(index int) (*http.Request, error) { + request, err := http.NewRequest(http.MethodPost, *baseURL+"/api/v1/users", bytes.NewReader(body)) + if err != nil { + return nil, err + } + request.Header.Set("Content-Type", "application/json") + return request, nil + }) + if err != nil { + return err + } + login, err := burst(client, *count, "login", func(index int) (*http.Request, error) { + request, err := http.NewRequest(http.MethodPost, *baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return nil, err + } + request.SetBasicAuth(fmt.Sprintf("%s_%d", username, index), "wrong-password") + return request, nil + }) + if err != nil { + return err + } + response, err := client.Get(*baseURL + "/health/ready") + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("readiness after burst: HTTP %d", response.StatusCode) + } + return json.NewEncoder(os.Stdout).Encode(struct { + Phases []phase `json:"phases"` + Ready bool `json:"ready_after_burst"` + }{[]phase{register, login}, true}) +} + +func burst(client *http.Client, count int, name string, request func(int) (*http.Request, error)) (phase, error) { + var ( + start = make(chan struct{}) + results = make(chan result, count) + group sync.WaitGroup + instances = make(map[string]struct{}) + statuses = make(map[int]int) + durations = make([]time.Duration, 0, count) + started = time.Now() + ) + for index := range count { + group.Add(1) + go func() { + defer group.Done() + <-start + began := time.Now() + value, err := request(index) + if err != nil { + results <- result{err: err} + return + } + response, err := client.Do(value) + if err != nil { + results <- result{err: err} + return + } + _, copyErr := io.Copy(io.Discard, response.Body) + _ = response.Body.Close() + results <- result{response.StatusCode, response.Header.Get("Retry-After"), response.Header.Get("X-Instance-Id"), time.Since(began), copyErr} + }() + } + close(start) + group.Wait() + close(results) + for value := range results { + if value.err != nil { + return phase{}, fmt.Errorf("%s burst: %w", name, value.err) + } + if value.status == http.StatusTooManyRequests { + retryAfter, err := strconv.Atoi(value.retryAfter) + if err != nil || retryAfter < 1 { + return phase{}, errors.New("overload response has no positive Retry-After") + } + } + allowed := value.status == http.StatusTooManyRequests + if name == "registration" { + allowed = allowed || value.status == http.StatusCreated || value.status == http.StatusConflict + } else { + allowed = allowed || value.status == http.StatusUnauthorized + } + if !allowed { + return phase{}, fmt.Errorf("%s burst: unexpected HTTP %d", name, value.status) + } + statuses[value.status]++ + durations = append(durations, value.duration) + if strings.TrimSpace(value.instance) != "" { + instances[value.instance] = struct{}{} + } + } + if statuses[http.StatusTooManyRequests] == 0 || len(instances) < 2 { + return phase{}, fmt.Errorf("%s burst: expected overload across two replicas, statuses=%v instances=%d", name, statuses, len(instances)) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + return phase{name, count, statuses, time.Since(started).Milliseconds(), durations[(count*95+99)/100-1].Milliseconds()}, nil +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 39bd0bd..b309dc7 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -2,7 +2,6 @@ package main import ( "errors" - "log/slog" "os" "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/bootstrap" @@ -11,24 +10,24 @@ import ( func main() { appLogger := logger.New() - if err := run(appLogger, os.Args[1:]); err != nil { + if err := run(os.Args[1:]); err != nil { appLogger.Error("application stopped", "error", err) os.Exit(1) } } -func run(logger *slog.Logger, arguments []string) error { +func run(arguments []string) error { if len(arguments) == 0 { - return bootstrap.RunServer(logger) + return bootstrap.RunServer() } if len(arguments) == 1 && arguments[0] == "migrate" { return bootstrap.RunMigrations() } if len(arguments) == 1 && arguments[0] == "dispatch" { - return bootstrap.RunDispatcher(logger) + return bootstrap.RunDispatcher() } if len(arguments) == 1 && arguments[0] == "worker" { - return bootstrap.RunWorker(logger) + return bootstrap.RunWorker() } if arguments[0] == "reconcile-attachments" { return bootstrap.RunReconciliation(arguments[1:], os.Stdout) diff --git a/compose.yaml b/compose.yaml index fa8ba0c..c893395 100644 --- a/compose.yaml +++ b/compose.yaml @@ -343,6 +343,8 @@ services: openbao-cli: profiles: ["tools"] image: openbao/openbao:2.7.1 + user: "0:0" + entrypoint: ["bao"] environment: BAO_ADDR: https://openbao:8200 BAO_CACERT: /run/security/openbao/ca.crt diff --git a/config/messenger.yaml b/config/messenger.yaml index 1bc0f3a..80ee8ac 100644 --- a/config/messenger.yaml +++ b/config/messenger.yaml @@ -29,6 +29,7 @@ server: parallelism: 2 salt_length: 16 key_length: 32 + max_concurrent_operations: 2 database: min_connections: 1 diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..9e82391 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,27 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +| Связь | Порт и протокол | Server trust и client identity | Минимальные права / lifecycle | Доказательство | +| --- | --- | --- | --- | --- | +| Клиент → Nginx | `8443`, TLS 1.3 | Public CA, SAN `localhost`; Bearer session | Сессия хранится как hash с TTL; общий login limiter | `make test-security`; `make test`; `make test-login-rate-limit` | +| Nginx → API replicas | `8443`, mTLS | Internal CA; server SAN `api`; client URI `spiffe://mtusi/lab7/load-balancer` | Leaf выпускается bootstrap и меняется без rebuild | `make test-tls-matrix`: allow LB, deny no-cert/foreign/forbidden | +| Prometheus → API metrics | `9091`, mTLS | Internal CA; URI `spiffe://mtusi/lab7/prometheus` | Только `/metrics`; business route отсутствует | `make test-tls-matrix` | +| API/worker/dispatcher/reconciler → PostgreSQL | `5432`, TLS 1.3 `verify-full` + client cert | SAN `postgres`; отдельные API, worker, dispatcher, reconciler certs | Разные DB roles/grants; migrator отдельный one-shot | `infra/security/test-dependency-matrix.sh` | +| API → Valkey | gateway `16379`, mTLS; backend TLS | SAN `redis-gateway`; API URI SAN; Valkey ACL user `messenger_api` | Только session/rate-limit prefixes; password приходит из OpenBao | dependency matrix: разрешён session key, запрещён arbitrary key | +| Dispatcher → Redis Streams | gateway `26379`, mTLS; backend TLS | Dispatcher URI SAN | Producer ACL без consume | dependency matrix: `XADD` разрешён, `XREAD` запрещён | +| Worker → Redis Streams | gateway `36379`, mTLS; backend TLS | Worker URI SAN | Consumer/ack ACL, unrelated stream запрещён; credential ротируется | dependency matrix и `make test-worker-compromise` | +| API/worker/reconciler → MinIO | `9000`, HTTPS | Internal CA, SAN `s3` | API: staging; worker: staging+attachments; reconciler: cleanup; разные keys | dependency matrix проверяет разрешённые и запрещённые prefixes | +| API/worker/dispatcher/reconciler → OTel gateway | `4317/4318`, mTLS | Internal CA; exact allowlist service URI SAN | Только telemetry ingest | dependency matrix отклоняет same-CA Prometheus identity | +| OTel Collector → Loki/Tempo/Prometheus | локальный TLS gateway → backend network | Collector identity и Internal CA | Backend ports не опубликованы; plaintext hop только внутри выделенной `telemetry_storage` network | `infra/otel/collector.yaml`, Prometheus targets, TLS matrix | +| Grafana → Prometheus/Loki/Tempo | отдельные HTTPS gateways | Grafana client certificate, Internal CA, exact URI SAN | Read-only datasource access; UI опубликован только на `127.0.0.1:3000` по HTTPS | provisioned datasource health API и gateway config | +| OpenBao Agent API/worker → OpenBao | `8200`, TLS 1.3 | OpenBao SAN; AppRole RoleID+SecretID с TTL 1 час | Раздельные KV policies; renewable token; rendered file `0600` в защищённом volume, Agent token в tmpfs | `scripts/openbao-test-policy.sh`, `scripts/openbao-rotate.sh` | +| Оператор → OpenBao | `127.0.0.1:8200`, TLS 1.3 | CA; root token только в operator runtime file | Init/unseal/bootstrap/rotation; Raft volume persistent | `scripts/bootstrap-local.sh`, `infra/openbao/README.md` | +| Оператор → Grafana | `127.0.0.1:3000`, HTTPS | Internal CA, login/password | Административный UI не слушает внешний интерфейс | `docker compose port grafana 3000` и HTTPS health | +| Replication/cluster channels | N/A | PostgreSQL, Valkey, Redis, MinIO и OpenBao запущены по одному экземпляру | Репликация не заявлена; отказ зависимости обнаруживается readiness | Документированное ограничение стенда | -| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | -| --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +## Каналы bootstrap, health и scrape -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. - -## Обязательные отрицательные проверки - -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. - -## Bootstrap и эксплуатация - -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. - -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +- `127.0.0.1:8082` существует только внутри каждого app container и используется Docker healthcheck; business router на нём отсутствует. +- Публичный `/health/ready` проходит через ingress TLS, но не раскрывает credentials или содержимое зависимостей. +- Metrics listener требует Prometheus URI SAN; сертификат LB и сертификаты приложений отклоняются. +- Bootstrap jobs подключены только к нужным внутренним сетям, не публикуют порты и завершаются после установки ACL/roles/secrets. +- Private keys, root token, unseal shares, RoleID/SecretID и rendered secrets хранятся в `.runtime`/Docker volumes и отсутствуют в Git. diff --git a/evidence/lab7-argon2-budget.json b/evidence/lab7-argon2-budget.json new file mode 100644 index 0000000..2f9022f --- /dev/null +++ b/evidence/lab7-argon2-budget.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "environment": { + "os": "linux", + "architecture": "arm64", + "cpu_limit": 2, + "memory_limit_mib": 512, + "go_version": "1.26.5" + }, + "parameters": { + "memory_kib": 65536, + "iterations": 3, + "parallelism": 2, + "salt_length": 16, + "key_length": 32 + }, + "method": "go test benchmark, 5 iterations per operation", + "hash": { + "nanoseconds_per_operation": 60029195, + "bytes_per_operation": 67114704, + "allocations_per_operation": 59 + }, + "verify": { + "nanoseconds_per_operation": 64816424, + "bytes_per_operation": 67113590, + "allocations_per_operation": 53 + } +} diff --git a/evidence/lab7-security-checks.json b/evidence/lab7-security-checks.json new file mode 100644 index 0000000..f5b43ff --- /dev/null +++ b/evidence/lab7-security-checks.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "generated_at": "2026-11-06T18:18:28.005396+00:00", + "scope": "local runtime checks on the lab7 Compose stack; remote CI is separate", + "public_https_smoke": "PASS", + "public_and_api_mtls_matrix_cases": 13, + "dependency_mtls_and_acl_matrix": "PASS", + "openbao_policy_allow_deny_and_renewal": "PASS", + "openbao_expiry_revoke_seal_and_reauth": "PASS", + "leaf_rotation_old_identity_and_expired_leaf_denied": "PASS", + "shared_login_rate_limit_across_two_replicas": "PASS", + "postgres_and_valkey_integration": "PASS", + "go_tests_and_vet": "PASS", + "grafana_datasource_health": { + "prometheus": "OK", + "loki": "OK", + "tempo": "OK" + } +} diff --git a/evidence/lab7-security-scan.json b/evidence/lab7-security-scan.json new file mode 100644 index 0000000..80ab1f1 --- /dev/null +++ b/evidence/lab7-security-scan.json @@ -0,0 +1,331 @@ +{ + "schema_version": 1, + "scanner": "Trivy 0.76.2", + "generated_at": "2026-11-06T18:17:33+00:00", + "scope": "repository filesystem: vulnerabilities, misconfigurations, secrets", + "vulnerabilities": { + "UNKNOWN": 3 + }, + "misconfigurations": { + "LOW": 2 + }, + "secret_findings": 0, + "scanned_images": [ + "envoyproxy/envoy:v1.35.2", + "grafana/loki:3.7.9", + "grafana/tempo:2.10.10", + "mtusi-grafana:lab7", + "mtusi-messenger:lab7", + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z", + "mtusi-nginx:lab7", + "mtusi-prometheus:lab7", + "mtusi-swagger:lab7", + "nginx/nginx-prometheus-exporter:1.5.1", + "oliver006/redis_exporter:v1.89.0-alpine", + "openbao/openbao:2.7.1", + "otel/opentelemetry-collector-contrib:0.160.0", + "postgres:17.6-alpine3.22", + "quay.io/prometheuscommunity/postgres-exporter:v0.20.1", + "redis:8.2.2-alpine", + "valkey/valkey:8.1.4-alpine" + ], + "image_vulnerabilities": { + "CRITICAL": 26, + "HIGH": 479, + "LOW": 508, + "MEDIUM": 795, + "UNKNOWN": 92 + }, + "image_secret_findings": 0, + "findings": [ + { + "id": "CVE-2026-111497", + "package": "brotli-libs", + "installed_version": "1.2.0-r1", + "fixed_version": "1.2.1-r0", + "severity": "CRITICAL", + "title": "brotli: Out-of-Bounds Write in Google Brotli Decoder", + "scopes": [ + "mtusi-grafana:lab7" + ] + }, + { + "id": "CVE-2026-111497", + "package": "brotli-libs", + "installed_version": "1.2.0-r0", + "fixed_version": "1.2.1-r0", + "severity": "CRITICAL", + "title": "brotli: Out-of-Bounds Write in Google Brotli Decoder", + "scopes": [ + "mtusi-swagger:lab7" + ] + }, + { + "id": "CVE-2026-33322", + "package": "github.com/minio/minio", + "installed_version": "v0.0.0-20251015172955-9e49d5e7a648", + "fixed_version": "", + "severity": "CRITICAL", + "title": "MinIO has JWT Algorithm Confusion in OIDC Authentication", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-33419", + "package": "github.com/minio/minio", + "installed_version": "v0.0.0-20251015172955-9e49d5e7a648", + "fixed_version": "", + "severity": "CRITICAL", + "title": "MinIO LDAP login brute-force via user enumeration and missing rate limit", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-77405", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.10.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Missing Explicit TLS Minimum Version Configuration In URI Parser", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-77405", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.12.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Missing Explicit TLS Minimum Version Configuration In URI Parser", + "scopes": [ + "otel/opentelemetry-collector-contrib:0.160.0" + ] + }, + { + "id": "CVE-2026-77408", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.10.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Silent Data Truncation and State Corruption via Shortstr Integer Overflow", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-77408", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.12.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Silent Data Truncation and State Corruption via Shortstr Integer Overflow", + "scopes": [ + "otel/opentelemetry-collector-contrib:0.160.0" + ] + }, + { + "id": "CVE-2026-77411", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.10.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Protocol Desynchronization and Frame Injection via Integer Overflow in readLongstr", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-77411", + "package": "github.com/rabbitmq/amqp091-go", + "installed_version": "v1.12.0", + "fixed_version": "1.13.0", + "severity": "CRITICAL", + "title": "github.com/rabbitmq/amqp091-go: RabbitMQ amqp091-go: Protocol Desynchronization and Frame Injection via Integer Overflow in readLongstr", + "scopes": [ + "otel/opentelemetry-collector-contrib:0.160.0" + ] + }, + { + "id": "GO-2026-5932", + "package": "golang.org/x/crypto", + "installed_version": "v0.57.0", + "fixed_version": "0.58.1-0.20261009181718-2f1f834330b2", + "severity": "UNKNOWN", + "title": "The golang.org/x/crypto/openpgp package is unmaintained, unsafe by design, and has known security issues", + "scopes": [ + "repository", + "grafana/tempo:2.10.10", + "mtusi-grafana:lab7", + "mtusi-messenger:lab7" + ] + }, + { + "id": "CVE-2026-78661", + "package": "golang.org/x/net", + "installed_version": "v0.60.0", + "fixed_version": "0.62.0", + "severity": "UNKNOWN", + "title": "When coalescing adjacent text nodes in addText and fosterParent, appen ...", + "scopes": [ + "repository", + "grafana/tempo:2.10.10", + "mtusi-grafana:lab7", + "mtusi-messenger:lab7" + ] + }, + { + "id": "CVE-2026-97045", + "package": "golang.org/x/net", + "installed_version": "v0.60.0", + "fixed_version": "0.62.0", + "severity": "UNKNOWN", + "title": "In childTextNodesAreLiteral, walking up the tree previously stopped at ...", + "scopes": [ + "repository", + "grafana/tempo:2.10.10", + "mtusi-grafana:lab7", + "mtusi-messenger:lab7" + ] + }, + { + "id": "CVE-2026-33186", + "package": "google.golang.org/grpc", + "installed_version": "v1.72.0", + "fixed_version": "1.79.3", + "severity": "CRITICAL", + "title": "google.golang.org/grpc/grpc-go: google.golang.org/grpc/authz: gRPC-Go: Authorization bypass due to improper HTTP/2 path validation", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2026-31789", + "package": "libcrypto3", + "installed_version": "3.5.4-r0", + "fixed_version": "3.5.6-r0", + "severity": "CRITICAL", + "title": "openssl: OpenSSL: Heap buffer overflow on 32-bit systems from large X.509 certificate processing", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z", + "postgres:17.6-alpine3.22", + "redis:8.2.2-alpine", + "valkey/valkey:8.1.4-alpine" + ] + }, + { + "id": "CVE-2026-31789", + "package": "libcrypto3", + "installed_version": "3.5.5-r0", + "fixed_version": "3.5.6-r0", + "severity": "CRITICAL", + "title": "openssl: OpenSSL: Heap buffer overflow on 32-bit systems from large X.509 certificate processing", + "scopes": [ + "mtusi-swagger:lab7" + ] + }, + { + "id": "CVE-2026-31789", + "package": "libssl3", + "installed_version": "3.5.4-r0", + "fixed_version": "3.5.6-r0", + "severity": "CRITICAL", + "title": "openssl: OpenSSL: Heap buffer overflow on 32-bit systems from large X.509 certificate processing", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z", + "postgres:17.6-alpine3.22", + "redis:8.2.2-alpine", + "valkey/valkey:8.1.4-alpine" + ] + }, + { + "id": "CVE-2026-31789", + "package": "libssl3", + "installed_version": "3.5.5-r0", + "fixed_version": "3.5.6-r0", + "severity": "CRITICAL", + "title": "openssl: OpenSSL: Heap buffer overflow on 32-bit systems from large X.509 certificate processing", + "scopes": [ + "mtusi-swagger:lab7" + ] + }, + { + "id": "CVE-2026-48930", + "package": "nodejs", + "installed_version": "24.13.0-r1", + "fixed_version": "24.17.0-r0", + "severity": "CRITICAL", + "title": "nodejs: Node.js: Silent authority rebinding due to embedded-nul hostnames in TLS handling", + "scopes": [ + "mtusi-swagger:lab7" + ] + }, + { + "id": "CVE-2026-31789", + "package": "openssl", + "installed_version": "3.5.4-r0", + "fixed_version": "3.5.6-r0", + "severity": "CRITICAL", + "title": "openssl: OpenSSL: Heap buffer overflow on 32-bit systems from large X.509 certificate processing", + "scopes": [ + "valkey/valkey:8.1.4-alpine" + ] + }, + { + "id": "CVE-2025-68121", + "package": "stdlib", + "installed_version": "v1.24.8", + "fixed_version": "1.24.13, 1.25.7, 1.26.0-rc.3", + "severity": "CRITICAL", + "title": "crypto/tls: crypto/tls: Incorrect certificate validation during TLS session resumption", + "scopes": [ + "mtusi-minio:RELEASE.2025-10-15T17-29-55Z" + ] + }, + { + "id": "CVE-2025-68121", + "package": "stdlib", + "installed_version": "v1.25.3", + "fixed_version": "1.24.13, 1.25.7, 1.26.0-rc.3", + "severity": "CRITICAL", + "title": "crypto/tls: crypto/tls: Incorrect certificate validation during TLS session resumption", + "scopes": [ + "nginx/nginx-prometheus-exporter:1.5.1" + ] + }, + { + "id": "CVE-2025-68121", + "package": "stdlib", + "installed_version": "v1.24.6", + "fixed_version": "1.24.13, 1.25.7, 1.26.0-rc.3", + "severity": "CRITICAL", + "title": "crypto/tls: crypto/tls: Incorrect certificate validation during TLS session resumption", + "scopes": [ + "postgres:17.6-alpine3.22" + ] + } + ], + "findings_scope": "all repository findings and critical image findings; full findings in raw reports", + "raw_report": "evidence/runtime/security-scan/trivy-fs.json", + "image_raw_reports": [ + "evidence/runtime/security-scan/trivy-image-1.json", + "evidence/runtime/security-scan/trivy-image-2.json", + "evidence/runtime/security-scan/trivy-image-3.json", + "evidence/runtime/security-scan/trivy-image-4.json", + "evidence/runtime/security-scan/trivy-image-5.json", + "evidence/runtime/security-scan/trivy-image-6.json", + "evidence/runtime/security-scan/trivy-image-7.json", + "evidence/runtime/security-scan/trivy-image-8.json", + "evidence/runtime/security-scan/trivy-image-9.json", + "evidence/runtime/security-scan/trivy-image-10.json", + "evidence/runtime/security-scan/trivy-image-11.json", + "evidence/runtime/security-scan/trivy-image-12.json", + "evidence/runtime/security-scan/trivy-image-13.json", + "evidence/runtime/security-scan/trivy-image-14.json", + "evidence/runtime/security-scan/trivy-image-15.json", + "evidence/runtime/security-scan/trivy-image-16.json", + "evidence/runtime/security-scan/trivy-image-17.json" + ] +} diff --git a/evidence/lab7-worker-compromise.json b/evidence/lab7-worker-compromise.json new file mode 100644 index 0000000..83d3f96 --- /dev/null +++ b/evidence/lab7-worker-compromise.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "scenario": "worker credential compromise and rotation", + "generated_at": "2026-11-06T18:08:45+00:00", + "recovery_seconds": 21, + "old_credential_rejected": true, + "old_database_credential_rejected": true, + "old_s3_credential_rejected": true, + "openbao_secret_rotated": true, + "queued_attachment_preserved": true, + "result": "PASS" +} diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 2b75368..37037f6 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -22,8 +22,8 @@ scripts/openbao-init.sh scripts/openbao-unseal.sh chmod 0600 .runtime/openbao/seed/api.json .runtime/openbao/seed/worker.json scripts/openbao-bootstrap.sh \ - .runtime/openbao/seed/api.json \ - .runtime/openbao/seed/worker.json + "$PWD/.runtime/openbao/seed/api.json" \ + "$PWD/.runtime/openbao/seed/worker.json" scripts/openbao-test-policy.sh ``` @@ -37,7 +37,7 @@ Seed-файлы — непустые JSON-объекты с фактически ```sh chmod 0600 .runtime/openbao/seed/api-next.json -scripts/openbao-rotate.sh api .runtime/openbao/seed/api-next.json +scripts/openbao-rotate.sh api "$PWD/.runtime/openbao/seed/api-next.json" ``` Ротация создаёт новую версию KV v2 и новый короткоживущий `SecretID`; предыдущий accessor отзывается. Agent перечитывает статический secret не позднее пяти минут. Уже выданный token может жить до своего TTL, поэтому при компрометации дополнительно отзывают его accessor или отключают роль и перезапускают Agent с новой идентичностью. diff --git a/infra/openbao/agent/api-secrets.ctmpl b/infra/openbao/agent/api-secrets.ctmpl index 3e8892b..6e7ab45 100644 --- a/infra/openbao/agent/api-secrets.ctmpl +++ b/infra/openbao/agent/api-secrets.ctmpl @@ -1,4 +1,3 @@ {{- with secret "secret/data/messenger/api" -}} {{ .Data.data | toJSON }} {{- end -}} - diff --git a/infra/openbao/agent/api.hcl b/infra/openbao/agent/api.hcl index 964d993..5f7bf0b 100644 --- a/infra/openbao/agent/api.hcl +++ b/infra/openbao/agent/api.hcl @@ -41,4 +41,3 @@ template { perms = "0600" backup = false } - diff --git a/infra/openbao/agent/worker-secrets.ctmpl b/infra/openbao/agent/worker-secrets.ctmpl index 9e2f36e..211b79d 100644 --- a/infra/openbao/agent/worker-secrets.ctmpl +++ b/infra/openbao/agent/worker-secrets.ctmpl @@ -1,4 +1,3 @@ {{- with secret "secret/data/messenger/worker" -}} {{ .Data.data | toJSON }} {{- end -}} - diff --git a/infra/openbao/agent/worker.hcl b/infra/openbao/agent/worker.hcl index 66567de..762e83c 100644 --- a/infra/openbao/agent/worker.hcl +++ b/infra/openbao/agent/worker.hcl @@ -41,4 +41,3 @@ template { perms = "0600" backup = false } - diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl index 939484f..38e734a 100644 --- a/infra/openbao/config.hcl +++ b/infra/openbao/config.hcl @@ -17,4 +17,3 @@ listener "tcp" { } log_level = "info" - diff --git a/infra/openbao/policies/messenger-api.hcl b/infra/openbao/policies/messenger-api.hcl index aaebd73..4b4edb6 100644 --- a/infra/openbao/policies/messenger-api.hcl +++ b/infra/openbao/policies/messenger-api.hcl @@ -29,4 +29,3 @@ path "auth/token/revoke-self" { path "sys/policies/*" { capabilities = ["deny"] } - diff --git a/infra/openbao/policies/messenger-worker.hcl b/infra/openbao/policies/messenger-worker.hcl index 8357e16..4eba2a5 100644 --- a/infra/openbao/policies/messenger-worker.hcl +++ b/infra/openbao/policies/messenger-worker.hcl @@ -29,4 +29,3 @@ path "auth/token/revoke-self" { path "sys/policies/*" { capabilities = ["deny"] } - diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index e25b114..031c87d 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -28,7 +28,7 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry" ) -func RunDispatcher(logger *slog.Logger) error { +func RunDispatcher() error { settings, err := config.LoadDispatcher() if err != nil { return err @@ -87,7 +87,7 @@ func RunDispatcher(logger *slog.Logger) error { ) } -func RunWorker(logger *slog.Logger) error { +func RunWorker() error { settings, err := config.LoadWorker() if err != nil { return err diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index d464a5b..e21d9c0 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -5,13 +5,13 @@ import ( "crypto/tls" "errors" "fmt" - "log/slog" "net" "net/http" - "os" "os/signal" "syscall" + "github.com/google/uuid" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config" "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database" "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/loginlimit" @@ -30,7 +30,7 @@ type serverResult struct { err error } -func RunServer(logger *slog.Logger) error { +func RunServer() error { settings, err := config.LoadServer() if err != nil { return err @@ -49,10 +49,11 @@ func RunServer(logger *slog.Logger) error { if err != nil { return err } - instanceID, err := os.Hostname() + instanceUUID, err := uuid.NewRandom() if err != nil { return err } + instanceID := "api-" + instanceUUID.String() observability, logger, err := startTelemetry("messenger-api", instanceID) if err != nil { return err @@ -91,7 +92,7 @@ func RunServer(logger *slog.Logger) error { MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, KeyLength: settings.PasswordKeyLength, - }) + }, settings.PasswordMaxConcurrentOperations) if err != nil { return err } diff --git a/internal/config/config.go b/internal/config/config.go index b022fc8..707aa92 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,53 +12,54 @@ import ( ) type ServerConfig struct { - Address string - HealthAddress string - MetricsAddress string - TLSCAPath string - TLSCertificatePath string - TLSPrivateKeyPath string - TLSAllowedClientURI string - TLSAllowedMetricsClientURI string - DatabaseURL string - CacheURL string - CacheTLS ClientTLSConfig - SessionTTL time.Duration - IdempotencyTTL time.Duration - LoginRateLimitAccountAttempts int - LoginRateLimitSourceAttempts int - LoginRateLimitWindow time.Duration - DBMaxConns int32 - DBMinConns int32 - DBMaxConnectionAge time.Duration - DBMaxConnectionIdle time.Duration - DBHealthCheckPeriod time.Duration - DependencyTimeout time.Duration - RequestTimeout time.Duration - HTTPReadHeaderTimeout time.Duration - HTTPReadTimeout time.Duration - HTTPWriteTimeout time.Duration - HTTPIdleTimeout time.Duration - HTTPShutdownTimeout time.Duration - PasswordMemoryKiB uint32 - PasswordIterations uint32 - PasswordParallelism uint8 - PasswordSaltLength uint32 - PasswordKeyLength uint32 - SwaggerPort string - S3Endpoint string - S3Region string - S3Bucket string - S3AccessKey string - S3SecretKey string - S3UseTLS bool - S3CAPath string - S3Timeout time.Duration - AttachmentMaxBytes int64 - AttachmentMaxPixels int64 - AttachmentBodyMemory int64 - AttachmentDecodeMemory int64 - PipelineVersion int + Address string + HealthAddress string + MetricsAddress string + TLSCAPath string + TLSCertificatePath string + TLSPrivateKeyPath string + TLSAllowedClientURI string + TLSAllowedMetricsClientURI string + DatabaseURL string + CacheURL string + CacheTLS ClientTLSConfig + SessionTTL time.Duration + IdempotencyTTL time.Duration + LoginRateLimitAccountAttempts int + LoginRateLimitSourceAttempts int + LoginRateLimitWindow time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + PasswordMaxConcurrentOperations int + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3CAPath string + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentBodyMemory int64 + AttachmentDecodeMemory int64 + PipelineVersion int } type MigrationConfig struct { @@ -251,6 +252,10 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + passwordMaxConcurrentOperations, err := integer(values, "server.password.max_concurrent_operations", 1, 64) + if err != nil { + return ServerConfig{}, err + } swaggerPort, err := requiredPort(values, "server.swagger_port") if err != nil { return ServerConfig{}, err @@ -306,7 +311,8 @@ func LoadServer() (ServerConfig, error) { HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), - SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + PasswordMaxConcurrentOperations: passwordMaxConcurrentOperations, + SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, S3CAPath: storage.caPath, S3Timeout: storage.timeout, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7306499..6c97dec 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -59,6 +59,7 @@ func TestLoadServerLoadsTLSAndHealthSettings(t *testing.T) { require.Equal(t, 5, settings.LoginRateLimitAccountAttempts) require.Equal(t, 20, settings.LoginRateLimitSourceAttempts) require.Equal(t, time.Minute, settings.LoginRateLimitWindow) + require.Equal(t, 2, settings.PasswordMaxConcurrentOperations) } func TestLoadServerRejectsNonLoopbackHealthAddress(t *testing.T) { @@ -152,6 +153,7 @@ server: parallelism: 2 salt_length: 16 key_length: 32 + max_concurrent_operations: 2 database: min_connections: 0 max_connections: 4 diff --git a/internal/config/loader.go b/internal/config/loader.go index 5084cd1..00e6ea0 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -74,7 +74,7 @@ func loadSecretFile(values *viper.Viper, path string) error { if err != nil { return fmt.Errorf("open messenger secrets: %w", err) } - defer file.Close() + defer func() { _ = file.Close() }() decoder := json.NewDecoder(io.LimitReader(file, 64<<10)) var secrets map[string]string if err := decoder.Decode(&secrets); err != nil { diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index 84b6a79..09c9236 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -6,7 +6,6 @@ import ( "mime" "net/http" "net/url" - "regexp" "strconv" "strings" "time" @@ -153,22 +152,6 @@ func attachmentVariant(query url.Values) (string, bool) { } } -var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) - -func traceparent(value string) string { - if traceparentPattern.MatchString(value) { - return value - } - return "" -} - -func tracestate(value string) string { - if len(value) <= 512 { - return value - } - return "" -} - func attachmentIDFromRequest(r *http.Request) (string, *apiError) { parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") attachmentID, valid := canonicalID(parts[3]) diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go index 5f26df8..9197881 100644 --- a/internal/messenger/httpapi/auth.go +++ b/internal/messenger/httpapi/auth.go @@ -37,7 +37,7 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError credentials, err := s.userRepository.UserByUsername(r.Context(), username) if errors.Is(err, model.ErrNotFound) { if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { - return internalError() + return passwordOperationError(verifyErr) } return unauthorized() } @@ -46,7 +46,7 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError } verified, err := s.passwords.Verify(passwordValue, credentials.PasswordHash) if err != nil { - return internalError() + return passwordOperationError(err) } if !verified { return unauthorized() @@ -70,14 +70,14 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError func (s *Server) rehashPassword(ctx context.Context, credentials model.UserCredentials, passwordValue string) *apiError { needsRehash, err := s.passwords.NeedsRehash(credentials.PasswordHash) if err != nil { - return internalError() + return passwordOperationError(err) } if !needsRehash { return nil } passwordHash, err := s.passwords.Hash(passwordValue) if err != nil { - return internalError() + return passwordOperationError(err) } updated, err := s.userRepository.UpdatePasswordHash(ctx, credentials.ID, credentials.PasswordHash, passwordHash) if err != nil { @@ -95,7 +95,7 @@ func (s *Server) rehashPassword(ctx context.Context, credentials model.UserCrede } verified, err := s.passwords.Verify(passwordValue, current.PasswordHash) if err != nil { - return internalError() + return passwordOperationError(err) } if !verified { return unauthorized() diff --git a/internal/messenger/httpapi/errors.go b/internal/messenger/httpapi/errors.go index 0567d7f..95b6af4 100644 --- a/internal/messenger/httpapi/errors.go +++ b/internal/messenger/httpapi/errors.go @@ -2,7 +2,10 @@ package httpapi import ( "encoding/json" + "errors" "net/http" + + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/password" ) const ( @@ -40,6 +43,13 @@ func rateLimited() *apiError { return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "too many requests"} } +func passwordOperationError(err error) *apiError { + if errors.Is(err, password.ErrBusy) { + return rateLimited() + } + return internalError() +} + func notFound() *apiError { return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"} } diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go index e119c59..70e63c3 100644 --- a/internal/messenger/httpapi/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -64,7 +64,7 @@ func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError { } passwordHash, err := s.passwords.Hash(input.Password) if err != nil { - return internalError() + return passwordOperationError(err) } user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) if errors.Is(err, model.ErrConflict) { diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go index 5371e98..64867fd 100644 --- a/internal/messenger/httpapi/server.go +++ b/internal/messenger/httpapi/server.go @@ -115,6 +115,7 @@ func (s *Server) MetricsHandler() http.Handler { } func (s *Server) serveHealth(w http.ResponseWriter, r *http.Request, handler requestHandler) { + w.Header().Set("X-Instance-Id", s.instanceID) if err := handler(w, r); err != nil { writeError(w, "", err) } @@ -168,6 +169,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { if apiErr.status == http.StatusUnauthorized { wrapped.Header().Set("WWW-Authenticate", authenticationChallenge(r)) } + if apiErr.status == http.StatusTooManyRequests && wrapped.Header().Get("Retry-After") == "" { + wrapped.Header().Set("Retry-After", "1") + } writeError(wrapped, requestID, apiErr) } } diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index 29e51db..675c420 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -16,6 +16,7 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment" "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab7/internal/password" ) var errDependency = errors.New("dependency unavailable") @@ -254,6 +255,22 @@ func (s *MessengerSuite) TestSessionCreationReturnsRateLimitWithRetryAfter() { s.Contains(response.Body.String(), `"code":"rate_limited"`) } +func (s *MessengerSuite) TestPasswordOperationLimitReturnsRateLimit() { + registration := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{hashErr: password.ErrBusy}) + response := serveRequest(registration, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":"T3st-password-123"}`) + s.Equal(http.StatusTooManyRequests, response.Code) + s.Equal("1", response.Header().Get("Retry-After")) + + store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}} + login := s.newTestServer(store, sessionStub{}, passwordsStub{err: password.ErrBusy}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("student", "T3st-password-123") + response = httptest.NewRecorder() + login.Handler().ServeHTTP(response, request) + s.Equal(http.StatusTooManyRequests, response.Code) + s.Equal("1", response.Header().Get("Retry-After")) +} + func (s *MessengerSuite) TestSessionCreationRehashesPasswordWithCompareAndSwap() { var updated []string store := messengerStoreStub{ diff --git a/internal/password/password.go b/internal/password/password.go index 0d97def..f8d771f 100644 --- a/internal/password/password.go +++ b/internal/password/password.go @@ -15,6 +15,7 @@ import ( var ( ErrInvalidConfig = errors.New("invalid password configuration") ErrInvalidHash = errors.New("invalid password hash") + ErrBusy = errors.New("password operation limit reached") ) type Config struct { @@ -26,17 +27,24 @@ type Config struct { } type Hasher struct { - config Config - dummyHash string + config Config + dummyHash string + operations chan struct{} } -func New(config Config) (*Hasher, error) { +func New(config Config, maxConcurrentOperations int) (*Hasher, error) { if err := validateConfig(config); err != nil { return nil, err } + if maxConcurrentOperations < 1 || maxConcurrentOperations > 64 { + return nil, ErrInvalidConfig + } salt := make([]byte, config.SaltLength) digest := argon2.IDKey([]byte("invalid-user-password"), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) - return &Hasher{config: config, dummyHash: encode(config, salt, digest)}, nil + return &Hasher{ + config: config, dummyHash: encode(config, salt, digest), + operations: make(chan struct{}, maxConcurrentOperations), + }, nil } func validateConfig(config Config) error { @@ -51,6 +59,10 @@ func validateConfig(config Config) error { } func (h *Hasher) Hash(value string) (string, error) { + if !h.acquire() { + return "", ErrBusy + } + defer h.release() salt := make([]byte, h.config.SaltLength) if _, err := rand.Read(salt); err != nil { return "", fmt.Errorf("generate password salt: %w", err) @@ -60,6 +72,10 @@ func (h *Hasher) Hash(value string) (string, error) { } func (h *Hasher) Verify(value, encodedHash string) (bool, error) { + if !h.acquire() { + return false, ErrBusy + } + defer h.release() config, salt, digest, err := decode(encodedHash) if err != nil { return false, err @@ -68,6 +84,19 @@ func (h *Hasher) Verify(value, encodedHash string) (bool, error) { return subtle.ConstantTimeCompare(actual, digest) == 1, nil } +func (h *Hasher) acquire() bool { + select { + case h.operations <- struct{}{}: + return true + default: + return false + } +} + +func (h *Hasher) release() { + <-h.operations +} + func (h *Hasher) NeedsRehash(encodedHash string) (bool, error) { config, _, _, err := decode(encodedHash) if err != nil { diff --git a/internal/password/password_test.go b/internal/password/password_test.go index efe63f4..5ff2d96 100644 --- a/internal/password/password_test.go +++ b/internal/password/password_test.go @@ -16,7 +16,7 @@ func TestPasswordSuite(t *testing.T) { } func (s *PasswordSuite) SetupTest() { - hasher, err := New(testConfig()) + hasher, err := New(testConfig(), 2) s.Require().NoError(err) s.hasher = hasher } @@ -48,7 +48,7 @@ func (s *PasswordSuite) TestNeedsRehashDetectsChangedParameters() { legacyConfig := testConfig() legacyConfig.KeyLength = 16 - legacy, err := New(legacyConfig) + legacy, err := New(legacyConfig, 2) s.Require().NoError(err) legacyHash, err := legacy.Hash("T3st-password-123") s.Require().NoError(err) @@ -57,12 +57,34 @@ func (s *PasswordSuite) TestNeedsRehashDetectsChangedParameters() { s.True(needsRehash) } +func (s *PasswordSuite) TestBusyOperationsRecoverAfterCapacityIsReleased() { + s.hasher.operations <- struct{}{} + s.hasher.operations <- struct{}{} + defer func() { <-s.hasher.operations }() + _, err := s.hasher.Hash("T3st-password-123") + s.ErrorIs(err, ErrBusy) + _, err = s.hasher.Verify("T3st-password-123", s.hasher.DummyHash()) + s.ErrorIs(err, ErrBusy) + + <-s.hasher.operations + hash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + verified, err := s.hasher.Verify("T3st-password-123", hash) + s.Require().NoError(err) + s.True(verified) +} + +func (s *PasswordSuite) TestNewRejectsInvalidOperationLimit() { + _, err := New(testConfig(), 0) + s.ErrorIs(err, ErrInvalidConfig) +} + func testConfig() Config { return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32} } func BenchmarkArgon2idHash(b *testing.B) { - hasher, err := New(productionConfig()) + hasher, err := New(productionConfig(), 2) if err != nil { b.Fatal(err) } @@ -76,7 +98,7 @@ func BenchmarkArgon2idHash(b *testing.B) { } func BenchmarkArgon2idVerify(b *testing.B) { - hasher, err := New(productionConfig()) + hasher, err := New(productionConfig(), 2) if err != nil { b.Fatal(err) } diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 2a5d759..6b1beb6 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -263,21 +263,6 @@ func scanAttachmentWithReferenceAndObjectKey( return err } -func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { - var stagingKey *string - err := row.Scan( - &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, - &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, - &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, - &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, - &attachment.Referenced, - ) - if stagingKey != nil { - attachment.StagingKey = *stagingKey - } - return err -} - func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` var referenced bool diff --git a/internal/security/mtlsproxy/proxy.go b/internal/security/mtlsproxy/proxy.go index ff88d71..29e435f 100644 --- a/internal/security/mtlsproxy/proxy.go +++ b/internal/security/mtlsproxy/proxy.go @@ -40,7 +40,7 @@ func Run(ctx context.Context, config Config) error { if err != nil { return fmt.Errorf("listen: %w", err) } - defer listener.Close() + defer func() { _ = listener.Close() }() go func() { <-ctx.Done() _ = listener.Close() @@ -66,7 +66,7 @@ func Run(ctx context.Context, config Config) error { func serve(ctx context.Context, connection net.Conn, config Config, tlsConfiguration *tls.Config) { client := tls.Server(connection, tlsConfiguration) - defer client.Close() + defer func() { _ = client.Close() }() handshakeCtx, cancel := context.WithTimeout(ctx, config.DialTimeout) defer cancel() if err := client.HandshakeContext(handshakeCtx); err != nil { @@ -76,7 +76,7 @@ func serve(ctx context.Context, connection net.Conn, config Config, tlsConfigura if err != nil { return } - defer target.Close() + defer func() { _ = target.Close() }() result := make(chan struct{}, 2) copyConnection := func(destination, source net.Conn) { diff --git a/internal/security/tlsconfig/server_test.go b/internal/security/tlsconfig/server_test.go index 3183b4f..55c9a1b 100644 --- a/internal/security/tlsconfig/server_test.go +++ b/internal/security/tlsconfig/server_test.go @@ -73,7 +73,7 @@ func handshake(serverConfig, clientConfig *tls.Config) (error, error) { if err != nil { return err, err } - defer listener.Close() + defer func() { _ = listener.Close() }() serverResult := make(chan error, 1) go func() { connection, acceptErr := listener.Accept() diff --git a/scripts/bao-docker.sh b/scripts/bao-docker.sh index 303b988..ed51bc4 100755 --- a/scripts/bao-docker.sh +++ b/scripts/bao-docker.sh @@ -6,5 +6,7 @@ script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +"$script_dir/sync-docker-workspace.sh" + exec docker compose --project-directory "$repo_dir" --env-file "$env_file" run --rm --no-deps -T \ -v "$repo_dir:$repo_dir:ro" openbao-cli "$@" diff --git a/scripts/bootstrap-local.sh b/scripts/bootstrap-local.sh index 07c6233..fa35970 100755 --- a/scripts/bootstrap-local.sh +++ b/scripts/bootstrap-local.sh @@ -20,14 +20,17 @@ if [ ! -f "$repo_dir/.runtime/security/openbao/tls.crt" ]; then fi "$script_dir/openbao-seed.sh" "$env_file" +"$script_dir/sync-docker-workspace.sh" compose up -d security-bootstrap openbao-permissions openbao -openbao_address=$(compose port openbao 8200) -BAO_ADDR="https://$openbao_address" +if [ -z "${BAO_ADDR:-}" ]; then + openbao_address=$(compose port openbao 8200) + BAO_ADDR="https://$openbao_address" +fi BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" export BAO_ADDR BAO_CACERT ENV_FILE="$env_file" remaining=60 -until curl --silent --show-error --cacert "$BAO_CACERT" \ +until curl --silent --show-error --connect-timeout 2 --max-time 3 --cacert "$BAO_CACERT" \ --output /dev/null "$BAO_ADDR/v1/sys/health"; do remaining=$((remaining - 1)) [ "$remaining" -gt 0 ] || { @@ -40,7 +43,7 @@ done ENV_FILE="$env_file" "$script_dir/openbao-init.sh" ENV_FILE="$env_file" "$script_dir/openbao-unseal.sh" remaining=60 -until curl --fail --silent --show-error --cacert "$BAO_CACERT" \ +until curl --fail --silent --show-error --connect-timeout 2 --max-time 3 --cacert "$BAO_CACERT" \ --output /dev/null "$BAO_ADDR/v1/sys/health"; do remaining=$((remaining - 1)) [ "$remaining" -gt 0 ] || { diff --git a/scripts/lab7-worker-compromise.sh b/scripts/lab7-worker-compromise.sh index f9d9294..5239a8c 100755 --- a/scripts/lab7-worker-compromise.sh +++ b/scripts/lab7-worker-compromise.sh @@ -56,8 +56,10 @@ mkdir -p "$runtime_dir" chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir" trap restore EXIT HUP INT TERM -openbao_address=$(compose port openbao 8200) -BAO_ADDR="https://$openbao_address" +if [ -z "${BAO_ADDR:-}" ]; then + openbao_address=$(compose port openbao 8200) + BAO_ADDR="https://$openbao_address" +fi BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" export BAO_ADDR BAO_CACERT diff --git a/scripts/openbao-bootstrap.sh b/scripts/openbao-bootstrap.sh index dc67ac5..e27e458 100755 --- a/scripts/openbao-bootstrap.sh +++ b/scripts/openbao-bootstrap.sh @@ -90,4 +90,3 @@ done "$script_dir/openbao-issue-approle.sh" worker "$openbao_runtime_dir/approle/worker" echo "OpenBao policies, AppRoles and KV v2 secrets configured" - diff --git a/scripts/openbao-common.sh b/scripts/openbao-common.sh index 9497ce6..385486a 100755 --- a/scripts/openbao-common.sh +++ b/scripts/openbao-common.sh @@ -30,7 +30,9 @@ require_file() { require_private_file() { require_file "$1" - permissions=$(stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1") + if ! permissions=$(stat -c '%a' "$1" 2>/dev/null); then + permissions=$(stat -f '%Lp' "$1") + fi case "$permissions" in 600|400) ;; *) fail "file must have permissions 0600 or 0400: $1" ;; @@ -88,4 +90,3 @@ require_unsealed() { [ "$initialized" = "true" ] || fail "OpenBao is not initialized" [ "$sealed" = "false" ] || fail "OpenBao is sealed" } - diff --git a/scripts/openbao-init.sh b/scripts/openbao-init.sh index 9e36a33..7725646 100755 --- a/scripts/openbao-init.sh +++ b/scripts/openbao-init.sh @@ -43,4 +43,3 @@ mv "$init_file" "$openbao_operator_file" echo "OpenBao initialized; operator material saved to .runtime/openbao/operator/init.json" echo "Keep this file offline and run scripts/openbao-unseal.sh" - diff --git a/scripts/openbao-issue-approle.sh b/scripts/openbao-issue-approle.sh index 4506ab9..82a340b 100755 --- a/scripts/openbao-issue-approle.sh +++ b/scripts/openbao-issue-approle.sh @@ -79,9 +79,9 @@ if [ -n "$old_accessor" ]; then fi if [ "$output_dir" = "$openbao_runtime_dir/approle/$service" ]; then + "$script_dir/sync-docker-workspace.sh" docker compose --project-directory "$repo_dir" --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" \ run --rm --no-deps -e SERVICE="$service" openbao-identity-install fi echo "issued short-lived AppRole credentials for $service in ${output_dir#"$repo_dir/"}" - diff --git a/scripts/openbao-rotate.sh b/scripts/openbao-rotate.sh index 7460197..3ab0485 100755 --- a/scripts/openbao-rotate.sh +++ b/scripts/openbao-rotate.sh @@ -39,4 +39,3 @@ PY "$script_dir/openbao-issue-approle.sh" "$service" "$openbao_runtime_dir/approle/$service" echo "$service secret rotated; OpenBao Agent will render the new KV version" - diff --git a/scripts/openbao-test-policy.sh b/scripts/openbao-test-policy.sh index 7134e5c..f6301e6 100755 --- a/scripts/openbao-test-policy.sh +++ b/scripts/openbao-test-policy.sh @@ -137,4 +137,3 @@ expect_status worker GET sys/policies/acl 403 expect_status worker POST auth/token/renew-self 200 '{}' echo "OpenBao AppRole allow/deny and renewal checks passed" - diff --git a/scripts/openbao-unseal.sh b/scripts/openbao-unseal.sh index ae8586d..5bc6c8a 100755 --- a/scripts/openbao-unseal.sh +++ b/scripts/openbao-unseal.sh @@ -48,4 +48,3 @@ openbao_status "$status_file" [ "$(json_field "$status_file" sealed)" = "false" ] || fail "OpenBao remains sealed" echo "OpenBao unsealed" - diff --git a/scripts/scan-security.sh b/scripts/scan-security.sh index ffd40ad..f6c8fce 100755 --- a/scripts/scan-security.sh +++ b/scripts/scan-security.sh @@ -17,6 +17,7 @@ if [ -z "$image_list" ]; then fi mkdir -p "$runtime_dir" "$cache_dir" +"$script_dir/sync-docker-workspace.sh" docker run --rm \ -v "$repo_dir:/src:ro" \ @@ -50,6 +51,8 @@ for image in $image_list; do image_reports="$image_reports $image=$report" done +"$script_dir/sync-docker-workspace.sh" --pull-scan + python3 - "$raw_file" "$summary_file" $image_reports <<'PY' import datetime import json diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 5327251..a8fdc02 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -5,7 +5,8 @@ set -eu : "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" for container in $(docker compose ps -q api); do - instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + instance=$(docker exec "$container" wget -S -q -O /dev/null http://127.0.0.1:8082/health/live 2>&1 | + sed -n 's/^[[:space:]]*X-Instance-Id:[[:space:]]*//p' | tr -d '\r') if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then docker stop --time 6 "$container" >/dev/null exit 0 diff --git a/scripts/sync-docker-workspace.sh b/scripts/sync-docker-workspace.sh new file mode 100755 index 0000000..9a9ec75 --- /dev/null +++ b/scripts/sync-docker-workspace.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +set -eu + +[ "${DOCKER_WORKSPACE_MIRROR:-0}" = "1" ] || exit 0 + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +mode=${1:-sync} + +case "$mode" in + sync|--pull-scan) ;; + --clean) + case "$repo_dir" in + /tmp/mtusi2026lab7-*) ;; + *) echo "cleanup is restricted to temporary CI workspaces" >&2; exit 2 ;; + esac + run_id=${repo_dir#/tmp/mtusi2026lab7-} + case "$run_id" in + ''|*[!0-9]*) echo "invalid CI workspace name" >&2; exit 2 ;; + esac + docker run --rm --network none -v "$repo_dir:/workspace" alpine:3.24.2 \ + find /workspace -mindepth 1 -delete + exit 0 + ;; + *) echo "usage: $0 [--pull-scan|--clean]" >&2; exit 2 ;; +esac + +[ -f "$repo_dir/compose.yaml" ] || { + echo "repository checkout is incomplete" >&2 + exit 1 +} + +container=$(docker create --network none \ + -v "$repo_dir:/workspace" \ + alpine:3.24.2 /bin/true) + +cleanup() { + docker rm -f "$container" >/dev/null 2>&1 || true +} + +trap cleanup EXIT HUP INT TERM + +if [ "$mode" = "--pull-scan" ]; then + destination=$repo_dir/evidence/runtime/security-scan + mkdir -p "$destination" + docker cp "$container:/workspace/evidence/runtime/security-scan/." "$destination" + exit 0 +fi + +docker cp "$repo_dir/." "$container:/workspace" diff --git a/scripts/test-leaf-rotation.sh b/scripts/test-leaf-rotation.sh index c68e779..729e70a 100755 --- a/scripts/test-leaf-rotation.sh +++ b/scripts/test-leaf-rotation.sh @@ -25,6 +25,7 @@ compose_rotated() { } install_security_material() { + "$script_dir/sync-docker-workspace.sh" compose run --rm --no-deps security-bootstrap >/dev/null } diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..54eec22 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,50 @@ -# Модель угроз — заполнить +# Модель угроз ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +Защищаются пароли и сессии пользователей, сообщения и вложения, задания обработки, сервисные учётные данные, ключи PKI, секреты OpenBao и telemetry. В модели участвуют пользователь чата, посторонний пользователь, внешний клиент, скомпрометированный сервис и оператор стенда. + +Внешний клиент видит только HTTPS ingress. Сервисные сети, PostgreSQL, Valkey, Redis Streams, MinIO, OpenBao и telemetry не публикуются наружу. Оператор владеет CA private keys, root token и unseal material; приложения их не получают. Границы и identities перечислены в [матрице доверия](../contracts/trust-matrix.md). ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +```mermaid +flowchart LR + C[Клиент] -->|TLS + Bearer| N[Nginx] + N -->|mTLS: load-balancer| A[API x2] + A -->|TLS + отдельная роль| P[(PostgreSQL)] + A -->|mTLS + ACL| V[(Valkey)] + A -->|HTTPS + scoped key| S[(MinIO)] + D[Dispatcher] -->|mTLS + producer ACL| R[(Redis Streams)] + W[Worker] -->|mTLS + consumer ACL| R + W -->|HTTPS + scoped key| S + A -->|mTLS| O[OTel gateway] + W -->|mTLS| O + BA[OpenBao Agent] -->|TLS + AppRole| B[(OpenBao Raft)] + BA -->|0600 file| A +``` | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | - -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +| Клиент подменяет ingress или подключается к нему с чужим CA/hostname | Клиент общается с ожидаемым сервером | TLS 1.3, проверка публичного CA и SAN `localhost` | `make test-security` отклоняет пустой/чужой CA и неверный hostname | Локальный CA учебный; его private key хранит оператор | +| Процесс в сети обращается прямо к API, выдавая себя за LB | Бизнес API доступен только ingress | API требует client certificate и URI SAN `spiffe://mtusi/lab7/load-balancer` | `make test-tls-matrix`: нет cert, чужой CA и same-CA forbidden identity отклоняются | Нет полноценной service mesh revocation; leaf меняется оператором | +| Postgres role одного сервиса читает таблицы другого | Компрометация worker/dispatcher не раскрывает пользователей и сообщения | Раздельные роли, `pg_hba`, grants по таблицам, TLS `verify-full`, сопоставление role и client certificate | dependency matrix: worker не читает `users`, dispatcher не читает `attachments`, API с worker cert не входит | Миграционная роль привилегированна и запускается только одноразовым job | +| API или worker выходит за разрешённый S3 prefix | Временные и готовые объекты доступны только владельцу этапа | Разные MinIO users/policies и HTTPS | dependency matrix запрещает API писать `attachments/` и worker писать `private/` | MinIO root credentials нужны bootstrap job и не монтируются в приложения | +| Producer читает очередь или worker пишет произвольный stream | Разделены отправка и обработка заданий | Redis ACL, разные users и mTLS identities | dependency matrix запрещает dispatcher `XREAD`, worker — `XADD` в unrelated stream | Redis Streams даёт at-least-once; дедупликация остаётся в PostgreSQL | +| Чужой same-CA сервис отправляет telemetry | Только известные сервисы создают traces/logs/metrics | OTel Envoy проверяет URI SAN API/worker/dispatcher/reconciler | dependency matrix отклоняет Prometheus identity | Collector доступен через gateway; локальный hop gateway→collector изолирован сетью | +| Украденный password hash перебирается offline | Пароль остаётся дорогим для перебора | Argon2id с salt и параметрами; rehash после успешного login | unit tests проверяют формат, verify и `NeedsRehash` | Слабый пароль всё ещё можно подобрать; параметры требуют периодического пересмотра | +| Login DoS или подбор пароля идёт через разные API replicas | Лимит един для всего ingress | Атомарный Lua limiter в Valkey по login и source address, `429 Retry-After` | `make test-login-rate-limit` распределяет запросы между двумя replicas | При недоступности Valkey login закрывается ошибкой зависимости | +| Пользователь подделывает `X-User-Id` или читает чужое вложение | Identity берётся только из Bearer session; ACL чата обязательна | Hash session token в Valkey, membership checks в PostgreSQL | публичный smoke/security и унаследованные ACL tests | Отзыв сессии зависит от доступности Valkey | +| API получает worker secret или меняет политики OpenBao | Сервис видит только свой KV path | Отдельные AppRole, policies, renewable token и Agent template | `scripts/openbao-test-policy.sh`: own read 200, cross/write/policy 403, renew 200 | Initial SecretID записывает оператор в файл `0600`; в production нужен response wrapping | +| Компрометированный worker продолжает работать после отзыва | Отозванная identity больше не обрабатывает задания | Ротация Redis credential и worker KV secret, перезапуск Agent/worker | `make test-worker-compromise`: старый credential отвергнут, новая задача сохранена и обработана после ротации | Уже открытое соединение живёт до разрыва; сценарий принудительно перезапускает broker gateway | +| Секрет или private key попадает в Git/image/log | Credentials не покидают runtime storage | `.runtime` игнорируется, secrets как files, read-only mounts, scan secret detector | `make scan-security`, `secret_findings=0` | Локальный Docker volume и host filesystem защищает оператор | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +`infra/pki/bootstrap.sh` создаёт три CA и leaf certificates в `.runtime/security` с правами `0700/0600`; `security-bootstrap` переносит только нужные материалы в отдельные volumes. OpenBao работает в non-dev режиме на persistent Raft storage. Первый `init` сохраняет operator material в игнорируемом каталоге с правами `0600`; оператор выполняет `unseal`, bootstrap policies и выдачу AppRole. Root token не передаётся приложениям. + +OpenBao Agent аутентифицируется через отдельный AppRole, обновляет renewable token и рендерит JSON secret в защищённый Docker volume. Короткоживущий SecretID допускает повторный вход в пределах часа; затем оператор выдаёт новый. Ротация выполняется `scripts/openbao-rotate.sh`, после чего Agent перечитывает KV и приложение перезапускается без пересборки образа. При sealed OpenBao уже запущенный процесс продолжает работать с последним отрендеренным secret до его перезапуска; новый Agent не получит token, и расширенный healthcheck не признает его healthy по одному старому файлу. + +При компрометации worker оператор меняет broker password, обновляет `secret/messenger/worker`, отзывает старый AppRole SecretID/токен и пересоздаёт worker. Задание остаётся в Redis Stream/PostgreSQL и после восстановления обрабатывается повторно безопасно. + +Trivy проверяет зависимости, Docker/Compose misconfiguration и secrets. Исправляемые CRITICAL/HIGH зависимости обновляются до fixed versions. Замечания к одноразовым bootstrap/test containers оцениваются отдельно: они не публикуют портов, не входят в runtime приложения и запускаются только оператором; где возможно, для них также используется non-root user. -- 2.54.0