From 709346783ec73084130f7f17826f8b2080d6387a Mon Sep 17 00:00:00 2001 From: dizi Date: Thu, 22 Oct 2026 16:20:54 +0300 Subject: [PATCH 01/11] chore: import lab4 baseline --- .dockerignore | 1 + .gitea/workflows/ci.yml | 93 +++ .gitea/workflows/deploy.yml | 192 +++++ Dockerfile | 44 ++ cmd/dependencycheck/main.go | 330 +++++++++ cmd/lab4experiment/main.go | 676 ++++++++++++++++++ cmd/messenger/main.go | 31 + compose.yaml | 328 +++++++++ deploy/compose.images.yaml | 24 + deploy/compose.runtime.yaml | 4 + go.mod | 55 ++ go.sum | 117 +++ infra/minio/bootstrap.sh | 36 + infra/minio/client.Dockerfile | 16 + infra/minio/server.Dockerfile | 18 + internal/bootstrap/migration.go | 25 + internal/bootstrap/reconciliation.go | 85 +++ internal/bootstrap/reconciliation_test.go | 13 + internal/bootstrap/server.go | 113 +++ internal/config/config.go | 400 +++++++++++ internal/config/config_test.go | 29 + internal/database/database.go | 93 +++ internal/database/migrations.go | 150 ++++ internal/logger/logger.go | 10 + internal/messenger/attachment/service.go | 253 +++++++ internal/messenger/attachment/service_test.go | 218 ++++++ internal/messenger/httpapi/attachments.go | 164 +++++ internal/messenger/httpapi/auth.go | 111 +++ internal/messenger/httpapi/errors.go | 61 ++ internal/messenger/httpapi/handlers.go | 302 ++++++++ .../messenger/httpapi/messenger_interface.go | 49 ++ internal/messenger/httpapi/metrics.go | 114 +++ internal/messenger/httpapi/model.go | 30 + internal/messenger/httpapi/request.go | 127 ++++ internal/messenger/httpapi/routes.go | 47 ++ internal/messenger/httpapi/server.go | 133 ++++ internal/messenger/httpapi/server_test.go | 333 +++++++++ internal/model/errors.go | 18 + internal/model/model.go | 153 ++++ internal/model/model_test.go | 24 + internal/objectstore/store.go | 176 +++++ internal/password/password.go | 133 ++++ internal/password/password_test.go | 44 ++ internal/postgres/attachments.go | 254 +++++++ internal/postgres/errors.go | 11 + internal/postgres/idempotency.go | 364 ++++++++++ internal/postgres/store.go | 442 ++++++++++++ internal/postgres/store_integration_test.go | 401 +++++++++++ internal/reconcile/reconcile.go | 350 +++++++++ internal/reconcile/reconcile_test.go | 180 +++++ internal/session/session.go | 163 +++++ internal/session/session_integration_test.go | 72 ++ lab5.env.example | 100 +++ migrations/0001_initial.sql | 50 ++ migrations/0002_message_idempotency.sql | 19 + migrations/0003_attachments.sql | 47 ++ migrations/0004_attachment_lifecycle.sql | 16 + migrations/0005_allow_pending_deletion.sql | 9 + migrations/embed.go | 8 + monitoring/grafana/dashboards/messenger.json | 143 ++++ .../provisioning/dashboards/messenger.yml | 10 + .../provisioning/datasources/prometheus.yml | 10 + monitoring/prometheus.yml | 28 + nginx/nginx.conf | 57 ++ nginx/upstream.conf | 1 + scripts/restart.sh | 9 + scripts/rolling-update.sh | 223 ++++++ scripts/stop-one.sh | 16 + 68 files changed, 8356 insertions(+) create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitea/workflows/deploy.yml create mode 100644 Dockerfile create mode 100644 cmd/dependencycheck/main.go create mode 100644 cmd/lab4experiment/main.go create mode 100644 cmd/messenger/main.go create mode 100644 compose.yaml create mode 100644 deploy/compose.images.yaml create mode 100644 deploy/compose.runtime.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/minio/bootstrap.sh create mode 100644 infra/minio/client.Dockerfile create mode 100644 infra/minio/server.Dockerfile create mode 100644 internal/bootstrap/migration.go create mode 100644 internal/bootstrap/reconciliation.go create mode 100644 internal/bootstrap/reconciliation_test.go create mode 100644 internal/bootstrap/server.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/database/database.go create mode 100644 internal/database/migrations.go create mode 100644 internal/logger/logger.go create mode 100644 internal/messenger/attachment/service.go create mode 100644 internal/messenger/attachment/service_test.go create mode 100644 internal/messenger/httpapi/attachments.go create mode 100644 internal/messenger/httpapi/auth.go create mode 100644 internal/messenger/httpapi/errors.go create mode 100644 internal/messenger/httpapi/handlers.go create mode 100644 internal/messenger/httpapi/messenger_interface.go create mode 100644 internal/messenger/httpapi/metrics.go create mode 100644 internal/messenger/httpapi/model.go create mode 100644 internal/messenger/httpapi/request.go create mode 100644 internal/messenger/httpapi/routes.go create mode 100644 internal/messenger/httpapi/server.go create mode 100644 internal/messenger/httpapi/server_test.go create mode 100644 internal/model/errors.go create mode 100644 internal/model/model.go create mode 100644 internal/model/model_test.go create mode 100644 internal/objectstore/store.go create mode 100644 internal/password/password.go create mode 100644 internal/password/password_test.go create mode 100644 internal/postgres/attachments.go create mode 100644 internal/postgres/errors.go create mode 100644 internal/postgres/idempotency.go create mode 100644 internal/postgres/store.go create mode 100644 internal/postgres/store_integration_test.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 internal/reconcile/reconcile_test.go create mode 100644 internal/session/session.go create mode 100644 internal/session/session_integration_test.go create mode 100644 lab5.env.example create mode 100644 migrations/0001_initial.sql create mode 100644 migrations/0002_message_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_attachment_lifecycle.sql create mode 100644 migrations/0005_allow_pending_deletion.sql create mode 100644 migrations/embed.go create mode 100644 monitoring/grafana/dashboards/messenger.json create mode 100644 monitoring/grafana/provisioning/dashboards/messenger.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus.yml create mode 100644 nginx/nginx.conf create mode 100644 nginx/upstream.conf create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh diff --git a/.dockerignore b/.dockerignore index bc16e556..ddd2c738 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +/messenger .DS_Store diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 00000000..8e0d5d56 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,93 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab5-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab5.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + docker compose up -d --scale api=2 --wait --wait-timeout 90 + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 00000000..36d9be7e --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,192 @@ +name: Deploy production + +on: + workflow_dispatch: + +concurrency: + group: production + cancel-in-progress: false + +jobs: + verify: + name: Verify selected commit + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + api_image: ${{ steps.publish.outputs.api_image }} + nginx_image: ${{ steps.publish.outputs.nginx_image }} + swagger_image: ${{ steps.publish.outputs.swagger_image }} + prometheus_image: ${{ steps.publish.outputs.prometheus_image }} + grafana_image: ${{ steps.publish.outputs.grafana_image }} + s3_image: ${{ steps.publish.outputs.s3_image }} + s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} + env: + COMPOSE_PROJECT_NAME: mtusi2026lab5-deploy-${{ github.run_id }} + COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio-client:${{ github.sha }} + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab5.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + + - name: Publish verified images + id: publish + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin + + publish() { + output=$1 + image=$2 + push_output=$(docker push "$image") + printf '%s\n' "$push_output" + digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1) + test -n "$digest" + repository=${image%:*} + printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT" + } + + publish api_image "$API_IMAGE" + publish nginx_image "$NGINX_IMAGE" + publish swagger_image "$SWAGGER_IMAGE" + publish prometheus_image "$PROMETHEUS_IMAGE" + publish grafana_image "$GRAFANA_IMAGE" + publish s3_image "$S3_IMAGE" + publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE" + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi + + deploy: + name: Deploy selected commit + needs: verify + runs-on: ubuntu-latest + environment: production + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + DEPLOY_SHA: ${{ github.sha }} + steps: + - name: Configure SSH + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + + - name: Prepare immutable image manifest + run: | + cat > deploy-images.env < 0 { + return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err) + } + return writeJSON(output, result) +} + +func sampleMemory( + ctx context.Context, + prometheusURL string, + expectedReplicas int, + done <-chan struct{}, + heapPeak *int64, + rssPeak *int64, + result chan<- error, +) { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + result <- ctx.Err() + return + case <-done: + result <- nil + return + case <-ticker.C: + replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) + if err != nil { + result <- err + return + } + if replicas != int64(expectedReplicas) { + result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas) + return + } + heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) + if err != nil { + result <- err + return + } + rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes")) + if err != nil { + result <- err + return + } + *heapPeak = max(*heapPeak, heap) + *rssPeak = max(*rssPeak, rss) + } + } +} + +type consistencyResult struct { + PageSize int `json:"page_size"` + SeededConsistent int `json:"seeded_consistent"` + FreshApply reconciliationRun `json:"fresh_apply"` + DryRun reconciliationRun `json:"dry_run"` + Apply reconciliationRun `json:"apply"` + RepeatedApply reconciliationRun `json:"repeated_apply"` + FreshPreserved bool `json:"fresh_preserved"` + OrphanDeleted bool `json:"orphan_deleted"` + UnusedDeleted bool `json:"unused_metadata_deleted"` + InUsePreserved bool `json:"in_use_metadata_preserved"` + AbandonedDeleted bool `json:"abandoned_attachment_deleted"` + TotalDurationMillis int64 `json:"total_duration_ms"` +} + +type reconciliationRun struct { + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func runConsistency(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("consistency", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + pageSize := flags.Int("page-size", 10, "reconciliation page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("invalid consistency experiment flags") + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + settings, err := config.LoadReconciliation() + if err != nil { + return err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + metadata := postgres.New(pool) + clean, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour, + }) + if err != nil { + return err + } + initial, err := clean.Run(ctx) + if err != nil { + return err + } + if hasAnomalies(initial) { + return errors.New("consistency experiment requires a stack without existing attachment anomalies") + } + + started := time.Now() + environment, err := prepare(ctx, *baseURL) + if err != nil { + return err + } + fixture, err := jpegFixture(320, 200) + if err != nil { + return err + } + seededConsistent := *pageSize + 1 + for index := range seededConsistent { + filename := fmt.Sprintf("consistent-%03d.jpg", index) + if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil { + return err + } + } + unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) + if err != nil { + return err + } + inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) + if err != nil { + return err + } + if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil { + return err + } + abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture))) + if err != nil { + return err + } + if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil { + return fmt.Errorf("prepare abandoned pending attachment: %w", err) + } + keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) + if err != nil { + return err + } + if err := objects.Delete(ctx, keys[unused.ID]); err != nil { + return err + } + if err := objects.Delete(ctx, keys[inUse.ID]); err != nil { + return err + } + orphanKey := "attachments/experiment-" + uuid.NewString() + "/original" + if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil { + return err + } + + fresh, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, + UnreferencedRetention: time.Hour, Apply: true, + }) + if err != nil { + return err + } + freshRun, err := measuredReconciliation(ctx, fresh) + if err != nil { + return err + } + freshPreserved := objectExists(ctx, objects, orphanKey) && + attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) && + attachmentExists(ctx, pool, abandoned.ID) + if _, err := pool.Exec(ctx, ` + UPDATE attachments + SET created_at = clock_timestamp() - INTERVAL '2 hours' + WHERE id = $1`, abandoned.ID); err != nil { + return fmt.Errorf("age abandoned attachment: %w", err) + } + + dryRun, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, + }) + if err != nil { + return err + } + dryRunResult, err := measuredReconciliation(ctx, dryRun) + if err != nil { + return err + } + apply, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true, + }) + if err != nil { + return err + } + applyResult, err := measuredReconciliation(ctx, apply) + if err != nil { + return err + } + repeatedResult, err := measuredReconciliation(ctx, apply) + if err != nil { + return err + } + + result := consistencyResult{ + PageSize: *pageSize, SeededConsistent: seededConsistent, + FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, + FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), + UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), + AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), + TotalDurationMillis: time.Since(started).Milliseconds(), + } + if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || + repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 { + return fmt.Errorf("unexpected reconciliation result: %+v", result) + } + return writeJSON(output, result) +} + +func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) { + started := time.Now() + report, err := reconciler.Run(ctx) + return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err +} + +func hasAnomalies(report reconcile.Report) bool { + return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || + report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || + report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || + report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0 +} + +func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { + rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids) + if err != nil { + return nil, fmt.Errorf("query attachment keys: %w", err) + } + defer rows.Close() + keys := make(map[string]string, len(ids)) + for rows.Next() { + var id string + var key string + if err := rows.Scan(&id, &key); err != nil { + return nil, fmt.Errorf("scan attachment key: %w", err) + } + keys[id] = key + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate attachment keys: %w", err) + } + if len(keys) != len(ids) { + return nil, errors.New("not all attachment keys were found") + } + return keys, nil +} + +func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool { + var exists bool + return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists +} + +func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool { + _, err := objects.Stat(ctx, key) + return err == nil +} + +func prepare(ctx context.Context, baseURL string) (setup, error) { + client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}} + username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "") + var user struct { + ID string `json:"id"` + } + if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{ + "username": username, "display_name": "Lab 4 experiment", "password": experimentPassword, + }, &user, ""); err != nil { + return setup{}, err + } + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword)) + var session struct { + Token string `json:"token"` + } + if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil { + return setup{}, err + } + client.token = session.Token + var chat struct { + ID string `json:"id"` + } + if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{ + "title": "Lab 4 experiment", "member_ids": []string{user.ID}, + }, &chat, "Bearer "+client.token); err != nil { + return setup{}, err + } + return setup{client: client, chatID: chat.ID}, nil +} + +func (c *apiClient) upload( + ctx context.Context, + chatID string, + filename string, + body io.Reader, + contentLength int64, +) (attachment, error) { + path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body) + if err != nil { + return attachment{}, err + } + request.ContentLength = contentLength + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "image/jpeg") + response, err := c.client.Do(request) + if err != nil { + return attachment{}, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message) + } + var result attachment + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { + return attachment{}, err + } + return result, nil +} + +func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error { + return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{ + "text": "", "attachment_ids": []string{attachmentID}, + }, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString()) +} + +func (c *apiClient) jsonRequest( + ctx context.Context, + method string, + path string, + payload any, + result any, + authorization string, + extraHeader ...string, +) error { + var body io.Reader + if payload != nil { + encoded, err := json.Marshal(payload) + if err != nil { + return err + } + body = bytes.NewReader(encoded) + } + request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return err + } + if payload != nil { + request.Header.Set("Content-Type", "application/json") + } + if authorization != "" { + request.Header.Set("Authorization", authorization) + } + for index := 0; index+1 < len(extraHeader); index += 2 { + request.Header.Set(extraHeader[index], extraHeader[index+1]) + } + response, err := c.client.Do(request) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode < 200 || response.StatusCode >= 300 { + message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message) + } + if result == nil { + return nil + } + return json.NewDecoder(response.Body).Decode(result) +} + +func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) + if err == nil && count == int64(expected) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err()) + case <-ticker.C: + } + } +} + +func activeReplicasQuery() string { + return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))` +} + +func freshMetricQuery(metric string) string { + return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric) +} + +func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil) + if err != nil { + return 0, err + } + values := request.URL.Query() + values.Set("query", query) + request.URL.RawQuery = values.Encode() + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return 0, fmt.Errorf("prometheus status %d", response.StatusCode) + } + var payload struct { + Status string `json:"status"` + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + return 0, err + } + if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 { + return 0, errors.New("prometheus query returned no scalar value") + } + var raw string + if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil { + return 0, err + } + value, err := strconv.ParseFloat(raw, 64) + return int64(value), err +} + +func jpegFixture(width, height int) ([]byte, error) { + pixels := image.NewRGBA(image.Rect(0, 0, width, height)) + state := uint64(0x9e3779b97f4a7c15) + for index := range pixels.Pix { + state ^= state << 13 + state ^= state >> 7 + state ^= state << 17 + pixels.Pix[index] = byte(state >> 24) + } + var encoded bytes.Buffer + if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil { + return nil, err + } + return encoded.Bytes(), nil +} + +type rateReader struct { + reader *bytes.Reader + rate int64 + started time.Time + read int64 +} + +func newRateReader(data []byte, rate int64) *rateReader { + return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()} +} + +func (r *rateReader) Read(buffer []byte) (int, error) { + for { + allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read + if allowed > 0 { + if int64(len(buffer)) > allowed { + buffer = buffer[:allowed] + } + count, err := r.reader.Read(buffer) + r.read += int64(count) + return count, err + } + time.Sleep(10 * time.Millisecond) + } +} + +func writeJSON(output io.Writer, value any) error { + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go new file mode 100644 index 00000000..c6ab63ec --- /dev/null +++ b/cmd/messenger/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "errors" + "log/slog" + "os" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger" +) + +func main() { + appLogger := logger.New() + if err := run(appLogger, os.Args[1:]); err != nil { + appLogger.Error("application stopped", "error", err) + os.Exit(1) + } +} + +func run(logger *slog.Logger, arguments []string) error { + if len(arguments) == 0 { + return bootstrap.RunServer(logger) + } + if len(arguments) == 1 && arguments[0] == "migrate" { + return bootstrap.RunMigrations() + } + if arguments[0] == "reconcile-attachments" { + return bootstrap.RunReconciliation(arguments[1:], os.Stdout) + } + return errors.New("usage: messenger [migrate|reconcile-attachments]") +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f1c370aa --- /dev/null +++ b/compose.yaml @@ -0,0 +1,328 @@ +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${POSTGRES_CPUS} + mem_limit: ${POSTGRES_MEMORY} + restart: unless-stopped + networks: [data] + + valkey: + image: valkey/valkey:8.1.4-alpine + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"] + environment: + VALKEY_PASSWORD: ${VALKEY_PASSWORD} + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${VALKEY_CPUS} + mem_limit: ${VALKEY_MEMORY} + restart: unless-stopped + networks: [data] + + s3: + build: + context: . + dockerfile: infra/minio/server.Dockerfile + args: + MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z + image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_BROWSER: "off" + MINIO_PROMETHEUS_AUTH_TYPE: public + volumes: + - s3_data:/data + expose: + - "9000" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${S3_CPUS} + mem_limit: ${S3_MEMORY} + restart: unless-stopped + networks: [data] + + s3-bootstrap: + build: + context: . + dockerfile: infra/minio/client.Dockerfile + args: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + depends_on: + s3: + condition: service_healthy + healthcheck: + disable: true + cpus: ${S3_BOOTSTRAP_CPUS} + mem_limit: ${S3_BOOTSTRAP_MEMORY} + restart: "no" + networks: [data] + + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/messenger", "migrate"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: ${MIGRATE_CPUS} + mem_limit: ${MIGRATE_MEMORY} + restart: "no" + networks: [data] + + api: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab5} + environment: + APP_HOST: ${APP_HOST} + APP_PORT: ${APP_PORT} + DATABASE_URL: ${DATABASE_URL} + CACHE_URL: ${CACHE_URL} + SESSION_STORE: ${SESSION_STORE} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} + HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} + HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} + HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} + HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} + HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} + PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} + PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} + PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} + PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} + PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} + SWAGGER_PORT: ${SWAGGER_PORT} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_USE_TLS: ${S3_USE_TLS} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} + ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} + ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} + ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES} + ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + interval: 2s + timeout: 1s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS} + mem_limit: ${API_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [application, data] + + nginx: + build: + context: . + target: loadbalancer + image: ${NGINX_IMAGE:-mtusi-nginx:lab5} + ports: + - "${API_BIND_ADDRESS}:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"] + interval: 2s + timeout: 1s + retries: 10 + cpus: ${NGINX_CPUS} + mem_limit: ${NGINX_MEMORY} + restart: unless-stopped + networks: [edge, application, monitoring] + + swagger: + build: + context: . + target: swagger + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5} + ports: + - "${SWAGGER_BIND_ADDRESS}:8080" + depends_on: + nginx: + condition: service_healthy + cpus: ${SWAGGER_CPUS} + mem_limit: ${SWAGGER_MEMORY} + restart: unless-stopped + networks: [edge] + + tests: + profiles: ["tests"] + build: + context: . + target: test + environment: + TEST_DATABASE_URL: ${DATABASE_URL} + TEST_CACHE_URL: ${CACHE_URL} + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [data] + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 + environment: + DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable + DATA_SOURCE_USER: ${POSTGRES_USER} + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_COLLECTION_TIMEOUT: 5s + depends_on: + postgres: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + valkey-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://valkey:6379 + REDIS_USER: default + REDIS_PASSWORD: ${VALKEY_PASSWORD} + depends_on: + valkey: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] + depends_on: + nginx: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [monitoring] + + prometheus: + build: + context: . + target: prometheus + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5} + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - prometheus_data:/prometheus + ports: + - "${PROMETHEUS_BIND_ADDRESS}:9090" + healthcheck: + test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"] + interval: 5s + timeout: 3s + retries: 20 + depends_on: + - api + - nginx-exporter + - postgres-exporter + - valkey-exporter + - s3 + cpus: ${PROMETHEUS_CPUS} + mem_limit: ${PROMETHEUS_MEMORY} + restart: unless-stopped + networks: [edge, application, data, monitoring] + + grafana: + build: + context: . + target: grafana + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5} + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana_data:/var/lib/grafana + ports: + - "${GRAFANA_BIND_ADDRESS}:3000" + depends_on: + - prometheus + cpus: ${GRAFANA_CPUS} + mem_limit: ${GRAFANA_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + +volumes: + postgres_data: + valkey_data: + prometheus_data: + grafana_data: + s3_data: + +networks: + edge: + application: + internal: true + data: + internal: true + monitoring: + internal: true diff --git a/deploy/compose.images.yaml b/deploy/compose.images.yaml new file mode 100644 index 00000000..4251b210 --- /dev/null +++ b/deploy/compose.images.yaml @@ -0,0 +1,24 @@ +services: + api: + image: ${API_IMAGE:?API_IMAGE is required} + + migrate: + image: ${API_IMAGE:?API_IMAGE is required} + + nginx: + image: ${NGINX_IMAGE:?NGINX_IMAGE is required} + + swagger: + image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required} + + prometheus: + image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required} + + grafana: + image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required} + + s3: + image: ${S3_IMAGE:?S3_IMAGE is required} + + s3-bootstrap: + image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required} diff --git a/deploy/compose.runtime.yaml b/deploy/compose.runtime.yaml new file mode 100644 index 00000000..b9dcd470 --- /dev/null +++ b/deploy/compose.runtime.yaml @@ -0,0 +1,4 @@ +services: + nginx: + volumes: + - ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..10f19e48 --- /dev/null +++ b/go.mod @@ -0,0 +1,55 @@ +module gitea.privcloud.tech/dizi/mtusi2026lab5 + +go 1.26.0 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.0.99 + github.com/prometheus/client_golang v1.23.2 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + github.com/valkey-io/valkey-go v1.0.78 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..5aa456c6 --- /dev/null +++ b/go.sum @@ -0,0 +1,117 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= +github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..68a28884 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -eu + +alias_name=bootstrap +policy_name=messenger-attachments +policy_file=/tmp/messenger-attachments-policy.json + +mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +mc mb --ignore-existing "$alias_name/$S3_BUCKET" +mc anonymous set none "$alias_name/$S3_BUCKET" + +cat >"$policy_file" </dev/null diff --git a/infra/minio/client.Dockerfile b/infra/minio/client.Dockerfile new file mode 100644 index 00000000..fe8ee0a4 --- /dev/null +++ b/infra/minio/client.Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MC_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc +WORKDIR /src/mc +RUN go mod download +RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc . + +FROM alpine:3.22.2 + +COPY --from=build /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh + +ENTRYPOINT ["mc"] diff --git a/infra/minio/server.Dockerfile b/infra/minio/server.Dockerfile new file mode 100644 index 00000000..44b75c4d --- /dev/null +++ b/infra/minio/server.Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MINIO_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio +WORKDIR /src/minio +RUN go mod download +RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio . + +FROM alpine:3.22.2 + +RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data +COPY --from=build /out/minio /usr/local/bin/minio + +USER minio +VOLUME ["/data"] +ENTRYPOINT ["minio"] diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go new file mode 100644 index 00000000..4cf77120 --- /dev/null +++ b/internal/bootstrap/migration.go @@ -0,0 +1,25 @@ +package bootstrap + +import ( + "context" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" +) + +func RunMigrations() error { + settings, err := config.LoadMigration() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + return database.Migrate(context.Background(), pool, settings.MigrationTimeout) +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go new file mode 100644 index 00000000..359ddbd0 --- /dev/null +++ b/internal/bootstrap/reconciliation.go @@ -0,0 +1,85 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "flag" + "io" + "os/signal" + "syscall" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" +) + +type reconciliationOutput struct { + Mode string `json:"mode"` + GracePeriod string `json:"grace_period"` + UnreferencedRetention string `json:"unreferenced_retention"` + PageSize int `json:"page_size"` + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func RunReconciliation(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) + flags.SetOutput(io.Discard) + apply := flags.Bool("apply", false, "apply safe repairs") + gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") + retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments") + pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]") + } + if *apply && *gracePeriod <= 0 { + return errors.New("grace-period must be positive in apply mode") + } + settings, err := config.LoadReconciliation() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + UnreferencedRetention: *retention, Apply: *apply, + }) + if err != nil { + return err + } + started := time.Now() + report, err := reconciler.Run(ctx) + if err != nil { + return err + } + mode := "dry-run" + if *apply { + mode = "apply" + } + return json.NewEncoder(output).Encode(reconciliationOutput{ + Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(), + PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report, + }) +} diff --git a/internal/bootstrap/reconciliation_test.go b/internal/bootstrap/reconciliation_test.go new file mode 100644 index 00000000..9a9d3b00 --- /dev/null +++ b/internal/bootstrap/reconciliation_test.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) { + err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard) + require.EqualError(t, err, "grace-period must be positive in apply mode") +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go new file mode 100644 index 00000000..7073922c --- /dev/null +++ b/internal/bootstrap/server.go @@ -0,0 +1,113 @@ +package bootstrap + +import ( + "context" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" +) + +func RunServer(logger *slog.Logger) error { + settings, err := config.LoadServer() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), databaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + sessions, err := session.New(settings.CacheURL) + if err != nil { + return err + } + defer func() { _ = sessions.Close() }() + passwords, err := password.New(password.Config{ + MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, + Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, + KeyLength: settings.PasswordKeyLength, + }) + if err != nil { + return err + } + objects, err := objectstore.New(objectStoreConfig(settings)) + if err != nil { + return err + } + repository := postgres.New(pool) + attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, + CleanupTimeout: settings.RequestTimeout, + }) + if err != nil { + return err + } + app := httpapi.NewServer(logger, httpapi.Dependencies{ + Users: repository, Chats: repository, Messages: repository, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + }, httpapi.Config{ + SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, + RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, + }) + server := &http.Server{ + Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, + ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout, + IdleTimeout: settings.HTTPIdleTimeout, + } + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + logger.Info("server started", "address", server.Addr) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + select { + case err := <-serveResult: + return err + case <-ctx.Done(): + logger.Info("shutdown requested") + app.SetReady(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + _ = server.Close() + return err + } + return nil + } +} + +func databaseConfig(settings config.ServerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func objectStoreConfig(settings config.ServerConfig) objectstore.Config { + return objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..dcf5b9a1 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,400 @@ +package config + +import ( + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/spf13/viper" +) + +type ServerConfig struct { + Address string + DatabaseURL string + CacheURL string + SessionTTL time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentBodyMemory int64 + AttachmentDecodeMemory int64 +} + +type MigrationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + MigrationTimeout time.Duration +} + +type ReconciliationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration +} + +type databaseConfig struct { + url string + maxConns int + minConns int + maxConnectionAge time.Duration + maxConnectionIdle time.Duration + healthCheckPeriod time.Duration + dependencyTimeout time.Duration +} + +type objectStorageConfig struct { + endpoint string + region string + bucket string + accessKey string + secretKey string + useTLS bool + timeout time.Duration +} + +func LoadServer() (ServerConfig, error) { + values := viper.New() + values.AutomaticEnv() + + host, err := requiredString(values, "APP_HOST") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "APP_PORT") + if err != nil { + return ServerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return ServerConfig{}, err + } + cacheURL, err := requiredString(values, "CACHE_URL") + if err != nil { + return ServerConfig{}, err + } + store, err := requiredString(values, "SESSION_STORE") + if err != nil { + return ServerConfig{}, err + } + if store != "valkey" { + return ServerConfig{}, errors.New("SESSION_STORE must be valkey") + } + + sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30) + if err != nil { + return ServerConfig{}, err + } + idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) + if err != nil { + return ServerConfig{}, err + } + requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return ServerConfig{}, err + } + readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return ServerConfig{}, err + } + readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return ServerConfig{}, err + } + writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return ServerConfig{}, err + } + idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600) + if err != nil { + return ServerConfig{}, err + } + shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return ServerConfig{}, err + } + passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024) + if err != nil { + return ServerConfig{}, err + } + passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20) + if err != nil { + return ServerConfig{}, err + } + passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255) + if err != nil { + return ServerConfig{}, err + } + passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024) + if err != nil { + return ServerConfig{}, err + } + passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024) + if err != nil { + return ServerConfig{}, err + } + swaggerPort, err := requiredPort(values, "SWAGGER_PORT") + if err != nil { + return ServerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + if err != nil { + return ServerConfig{}, err + } + attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { + return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes) + } + attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + minimumDecodeMemory := int64(attachmentMaxPixels) * 8 + if int64(attachmentDecodeMemory) < minimumDecodeMemory { + return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + } + + return ServerConfig{ + Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, + SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, + DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + RequestTimeout: requestTimeout, + HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout, + HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout, + HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), + PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), + PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), + SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + AttachmentBodyMemory: int64(attachmentBodyMemory), + AttachmentDecodeMemory: int64(attachmentDecodeMemory), + }, nil +} + +func LoadMigration() (MigrationConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return MigrationConfig{}, err + } + migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + if err != nil { + return MigrationConfig{}, err + } + return MigrationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + MigrationTimeout: migrationTimeout, + }, nil +} + +func LoadReconciliation() (ReconciliationConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + return ReconciliationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + }, nil +} + +func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { + endpoint, err := requiredString(values, "S3_ENDPOINT") + if err != nil { + return objectStorageConfig{}, err + } + region, err := requiredString(values, "S3_REGION") + if err != nil { + return objectStorageConfig{}, err + } + bucket, err := requiredString(values, "S3_BUCKET") + if err != nil { + return objectStorageConfig{}, err + } + accessKey, err := requiredString(values, "S3_ACCESS_KEY") + if err != nil { + return objectStorageConfig{}, err + } + secretKey, err := requiredString(values, "S3_SECRET_KEY") + if err != nil { + return objectStorageConfig{}, err + } + useTLS, err := boolean(values, "S3_USE_TLS") + if err != nil { + return objectStorageConfig{}, err + } + timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return objectStorageConfig{}, err + } + return objectStorageConfig{ + endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey, + secretKey: secretKey, useTLS: useTLS, timeout: timeout, + }, nil +} + +func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { + databaseURL, err := requiredString(values, "DATABASE_URL") + if err != nil { + return databaseConfig{}, err + } + maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) + if err != nil { + return databaseConfig{}, err + } + minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) + if err != nil { + return databaseConfig{}, err + } + maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) + if err != nil { + return databaseConfig{}, err + } + maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) + if err != nil { + return databaseConfig{}, err + } + healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) + if err != nil { + return databaseConfig{}, err + } + dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) + if err != nil { + return databaseConfig{}, err + } + return databaseConfig{ + url: databaseURL, maxConns: maxConns, minConns: minConns, + maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle, + healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout, + }, nil +} + +func requiredString(values *viper.Viper, name string) (string, error) { + value := strings.TrimSpace(values.GetString(name)) + if value == "" { + return "", fmt.Errorf("%s is required", name) + } + return value, nil +} + +func requiredPort(values *viper.Viper, name string) (string, error) { + value, err := requiredString(values, name) + if err != nil { + return "", err + } + if _, err := integerValue(value, 1, 65535); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return value, nil +} + +func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) { + value, err := integer(values, name, minimum, maximum) + if err != nil { + return 0, err + } + return time.Duration(value) * time.Second, nil +} + +func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { + value, err := requiredString(values, name) + if err != nil { + return 0, err + } + parsed, err := integerValue(value, minimum, maximum) + if err != nil { + return 0, fmt.Errorf("%s: %w", name, err) + } + return parsed, nil +} + +func integerValue(value string, minimum, maximum int) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("must be between %d and %d", minimum, maximum) + } + return parsed, nil +} + +func boolean(values *viper.Viper, name string) (bool, error) { + value, err := requiredString(values, name) + if err != nil { + return false, err + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s: must be true or false", name) + } + return parsed, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 00000000..863746ce --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,29 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4", + "DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60", + "DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + "S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5", + } + for name, value := range values { + t.Setenv(name, value) + } + t.Setenv("APP_HOST", "") + t.Setenv("CACHE_URL", "") + t.Setenv("SWAGGER_PORT", "") + + settings, err := LoadReconciliation() + require.NoError(t, err) + require.Equal(t, "attachments", settings.S3Bucket) +} diff --git a/internal/database/database.go b/internal/database/database.go new file mode 100644 index 00000000..ff1f8bd4 --- /dev/null +++ b/internal/database/database.go @@ -0,0 +1,93 @@ +package database + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/migrations" +) + +var ( + errURLRequired = errors.New("database URL is required") + errMinConnections = errors.New("minimum database connections must not be negative") + errMaxConnections = errors.New("maximum database connections must be positive") + errConnectionRange = errors.New("minimum database connections exceed maximum") + errMaxConnectionAge = errors.New("maximum database connection age must be positive") + errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive") + errHealthCheckPeriod = errors.New("database health check period must be positive") + errConnectionTimeout = errors.New("database connection timeout must be positive") +) + +type Config struct { + URL string + MinConnections int32 + MaxConnections int32 + MaxConnectionAge time.Duration + MaxConnectionIdle time.Duration + HealthCheckPeriod time.Duration + ConnectTimeout time.Duration +} + +func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + poolConfig, err := pgxpool.ParseConfig(config.URL) + if err != nil { + return nil, fmt.Errorf("parse database configuration: %w", err) + } + poolConfig.MinConns = config.MinConnections + poolConfig.MaxConns = config.MaxConnections + poolConfig.MaxConnLifetime = config.MaxConnectionAge + poolConfig.MaxConnIdleTime = config.MaxConnectionIdle + poolConfig.HealthCheckPeriod = config.HealthCheckPeriod + + connectCtx, cancel := context.WithTimeout(ctx, config.ConnectTimeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(connectCtx, poolConfig) + if err != nil { + return nil, fmt.Errorf("create database pool: %w", err) + } + if err := pool.Ping(connectCtx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + + return pool, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error { + if timeout <= 0 { + return errors.New("database migration timeout must be positive") + } + migrationCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return applyMigrations(migrationCtx, pool, migrations.FS) +} + +func validateConfig(config Config) error { + switch { + case config.URL == "": + return errURLRequired + case config.MinConnections < 0: + return errMinConnections + case config.MaxConnections < 1: + return errMaxConnections + case config.MinConnections > config.MaxConnections: + return errConnectionRange + case config.MaxConnectionAge <= 0: + return errMaxConnectionAge + case config.MaxConnectionIdle <= 0: + return errMaxConnectionIdle + case config.HealthCheckPeriod <= 0: + return errHealthCheckPeriod + case config.ConnectTimeout <= 0: + return errConnectionTimeout + default: + return nil + } +} diff --git a/internal/database/migrations.go b/internal/database/migrations.go new file mode 100644 index 00000000..095c93d1 --- /dev/null +++ b/internal/database/migrations.go @@ -0,0 +1,150 @@ +package database + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + migrationLockID int64 = 0x4d54555349324c42 + migrationUnlockTimeout = 5 * time.Second +) + +var migrationNamePattern = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.sql$`) + +type migration struct { + version int64 + name string + checksum string + sql string +} + +func applyMigrations(ctx context.Context, pool *pgxpool.Pool, migrationFS fs.FS) (returnErr error) { + migrations, err := readMigrations(migrationFS) + if err != nil { + return err + } + connection, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection for migrations: %w", err) + } + defer connection.Release() + if _, err := connection.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil { + return fmt.Errorf("lock migrations: %w", err) + } + defer func() { + unlockCtx, cancel := context.WithTimeout(context.Background(), migrationUnlockTimeout) + defer cancel() + if _, err := connection.Exec(unlockCtx, "SELECT pg_advisory_unlock($1)", migrationLockID); err != nil { + _ = connection.Conn().Close(unlockCtx) + if returnErr == nil { + returnErr = fmt.Errorf("unlock migrations: %w", err) + } + } + }() + + if _, err := connection.Exec(ctx, ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +)`); err != nil { + return fmt.Errorf("create migration history: %w", err) + } + for _, migration := range migrations { + if err := applyMigration(ctx, connection, migration); err != nil { + return err + } + } + return nil +} + +func readMigrations(migrationFS fs.FS) ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, ".") + if err != nil { + return nil, fmt.Errorf("read embedded migrations: %w", err) + } + migrations := make([]migration, 0, len(entries)) + versions := make(map[int64]string) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".sql" { + continue + } + matches := migrationNamePattern.FindStringSubmatch(entry.Name()) + if matches == nil { + return nil, fmt.Errorf("invalid migration filename %q", entry.Name()) + } + version, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil || version < 1 { + return nil, fmt.Errorf("invalid migration version in %q", entry.Name()) + } + if previous, exists := versions[version]; exists { + return nil, fmt.Errorf("migration version %d is used by %q and %q", version, previous, entry.Name()) + } + data, err := fs.ReadFile(migrationFS, entry.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", entry.Name(), err) + } + hash := sha256.Sum256(data) + versions[version] = entry.Name() + migrations = append(migrations, migration{ + version: version, + name: entry.Name(), + checksum: hex.EncodeToString(hash[:]), + sql: string(data), + }) + } + if len(migrations) == 0 { + return nil, errors.New("no database migrations found") + } + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].version < migrations[j].version + }) + return migrations, nil +} + +func applyMigration(ctx context.Context, connection *pgxpool.Conn, migration migration) error { + var checksum string + err := connection.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version = $1", migration.version).Scan(&checksum) + switch { + case err == nil && checksum != migration.checksum: + return fmt.Errorf("migration %q checksum does not match applied version", migration.name) + case err == nil: + return nil + case !errors.Is(err, pgx.ErrNoRows): + return fmt.Errorf("read migration %q state: %w", migration.name, err) + } + + transaction, err := connection.Begin(ctx) + if err != nil { + return fmt.Errorf("begin migration %q: %w", migration.name, err) + } + defer func() { + _ = transaction.Rollback(context.WithoutCancel(ctx)) + }() + if _, err := transaction.Exec(ctx, migration.sql, pgx.QueryExecModeSimpleProtocol); err != nil { + return fmt.Errorf("apply migration %q: %w", migration.name, err) + } + if _, err := transaction.Exec(ctx, ` +INSERT INTO schema_migrations (version, name, checksum) +VALUES ($1, $2, $3)`, migration.version, migration.name, migration.checksum); err != nil { + return fmt.Errorf("record migration %q: %w", migration.name, err) + } + if err := transaction.Commit(ctx); err != nil { + return fmt.Errorf("commit migration %q: %w", migration.name, err) + } + return nil +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 00000000..ab813660 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,10 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New() *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stdout, nil)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go new file mode 100644 index 00000000..f5a1c963 --- /dev/null +++ b/internal/messenger/attachment/service.go @@ -0,0 +1,253 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "mime" + "net/http" + "time" + + "github.com/google/uuid" + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type Repository interface { + ChatForUser(context.Context, string, string) (model.Chat, error) + CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) + MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) + DeletePendingAttachment(context.Context, string, string) error + AttachmentByStorage(context.Context, string, string) (model.Attachment, error) + AttachmentForUser(context.Context, string, string) (model.Attachment, error) +} + +type ObjectStore interface { + Put(context.Context, string, string, []byte) error + Get(context.Context, string) (io.ReadCloser, error) + Delete(context.Context, string) error +} + +type Config struct { + MaxBytes int64 + MaxPixels int64 + BodyMemoryBytes int64 + DecodeMemoryBytes int64 + CleanupTimeout time.Duration +} + +type Upload struct { + ChatID string + OwnerID string + Filename string + ContentType string + ContentLength int64 + Body io.Reader +} + +type Content struct { + Attachment model.Attachment + Body io.ReadCloser +} + +type Service struct { + repository Repository + objects ObjectStore + config Config + bodyMemory *semaphore.Weighted + decodeMemory *semaphore.Weighted + logger *slog.Logger +} + +func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 { + return nil, errors.New("invalid attachment service configuration") + } + if config.BodyMemoryBytes < config.MaxBytes { + return nil, errors.New("attachment body memory must fit one maximum upload") + } + if config.DecodeMemoryBytes < config.MaxPixels*8 { + return nil, errors.New("attachment decode memory must fit one maximum image") + } + return &Service{ + repository: repository, objects: objects, config: config, + bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes), + decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger, + }, nil +} + +func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) { + if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil { + return model.Attachment{}, mapDependencyError(err) + } + data, contentType, release, err := s.readImage(input) + if err != nil { + return model.Attachment{}, err + } + defer release() + + attachmentID := uuid.NewString() + objectKey := "attachments/" + uuid.NewString() + "/original" + digest := sha256.Sum256(data) + pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: objectKey, Status: model.AttachmentPending, + }) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil { + s.deletePending(pending.ID, objectKey) + return model.Attachment{}, model.ErrUnavailable + } + attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey) + if err == nil { + return attachment, nil + } + return s.resolveReadyOutcome(pending.ID, objectKey) +} + +func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) + defer cancel() + stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) + if err == nil && stored.Status == model.AttachmentReady { + return stored, nil + } + if err == nil && stored.Status == model.AttachmentPending { + if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { + s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr) + return model.Attachment{}, model.ErrUnavailable + } + if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil { + s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr) + } + return model.Attachment{}, model.ErrUnavailable + } + if errors.Is(err, model.ErrNotFound) { + if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { + s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr) + } + return model.Attachment{}, model.ErrUnavailable + } + s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err) + return model.Attachment{}, model.ErrUnavailable +} + +func (s *Service) deletePending(attachmentID, objectKey string) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) + defer cancel() + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil { + s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + } +} + +func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + return attachment, nil +} + +func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) { + attachment, err := s.Metadata(ctx, attachmentID, userID) + if err != nil { + return Content{}, err + } + body, err := s.objects.Get(ctx, attachment.ObjectKey) + if err != nil { + return Content{}, model.ErrUnavailable + } + return Content{Attachment: attachment, Body: body}, nil +} + +func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { + mediaType, parameters, err := mime.ParseMediaType(input.ContentType) + if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { + return nil, "", nil, model.ErrUnsupportedMedia + } + if input.ContentLength > s.config.MaxBytes { + return nil, "", nil, model.ErrPayloadTooLarge + } + reservation := s.config.MaxBytes + if !s.bodyMemory.TryAcquire(reservation) { + return nil, "", nil, model.ErrCapacity + } + releaseBody := func() { s.bodyMemory.Release(reservation) } + data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1)) + if err != nil { + releaseBody() + return nil, "", nil, model.ErrInvalidInput + } + if int64(len(data)) > s.config.MaxBytes { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + if len(data) == 0 { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + detected := http.DetectContentType(data) + if detected != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || imageContentType(format) != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) + decodeBytes := pixels * 8 + if !s.decodeMemory.TryAcquire(decodeBytes) { + releaseBody() + return nil, "", nil, model.ErrCapacity + } + if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { + s.decodeMemory.Release(decodeBytes) + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + release := func() { + s.decodeMemory.Release(decodeBytes) + releaseBody() + } + return data, mediaType, release, nil +} + +func imageContentType(format string) string { + switch format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + default: + return "" + } +} + +func mapDependencyError(err error) error { + switch { + case errors.Is(err, model.ErrNotFound): + return model.ErrNotFound + case errors.Is(err, model.ErrConflict): + return model.ErrConflict + default: + return model.ErrUnavailable + } +} diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go new file mode 100644 index 00000000..f06a895d --- /dev/null +++ b/internal/messenger/attachment/service_test.go @@ -0,0 +1,218 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func TestUploadLifecycle(t *testing.T) { + data := pngFixture(t, 2, 2) + events := make([]string, 0, 3) + repository := &repositoryStub{events: &events} + objects := &objectStub{events: &events} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentReady, result.Status) + require.Equal(t, []string{"pending", "put", "ready"}, events) +} + +func TestUploadValidation(t *testing.T) { + data := pngFixture(t, 2, 2) + tests := []struct { + name string + contentType string + data []byte + maxBytes int64 + maxPixels int64 + want error + }{ + {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge}, + {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge}, + {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels) + input := upload(test.data, test.contentType) + _, err := service.Upload(context.Background(), input) + require.ErrorIs(t, err, test.want) + }) + } +} + +func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) { + data := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + require.True(t, service.bodyMemory.TryAcquire(1024)) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.bodyMemory.Release(1024) + + require.True(t, service.decodeMemory.TryAcquire(32)) + _, err = service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.decodeMemory.Release(32) +} + +func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) { + oversized := pngFixture(t, 3, 2) + valid := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + + _, err := service.Upload(context.Background(), upload(oversized, "image/png")) + require.ErrorIs(t, err, model.ErrPayloadTooLarge) + _, err = service.Upload(context.Background(), upload(valid, "image/png")) + require.NoError(t, err) +} + +func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { + _, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, + Config{ + MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, + }, + ) + require.Error(t, err) +} + +func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{} + objects := &objectStub{putErr: errors.New("s3 unavailable")} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.True(t, repository.pendingDeleted) +} + +func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true} + objects := &objectStub{} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentReady, result.Status) + require.False(t, objects.deleted) +} + +func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { + t.Helper() + service, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, + Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, + }, + ) + require.NoError(t, err) + return service +} + +func upload(data []byte, contentType string) Upload { + return Upload{ + ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType, + ContentLength: int64(len(data)), Body: bytes.NewReader(data), + } +} + +type repositoryStub struct { + events *[]string + markErr error + lookupReady bool + pendingDeleted bool + pending model.Attachment +} + +func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} + +func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) { + attachment.Status = model.AttachmentPending + s.pending = attachment + s.record("pending") + return attachment, nil +} + +func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) { + if s.markErr != nil { + return model.Attachment{}, s.markErr + } + s.pending.Status = model.AttachmentReady + s.record("ready") + return s.pending, nil +} + +func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error { + s.pendingDeleted = true + return nil +} + +func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { + if s.lookupReady { + s.pending.Status = model.AttachmentReady + return s.pending, nil + } + return model.Attachment{}, model.ErrNotFound +} + +func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { + return s.pending, nil +} + +func (s *repositoryStub) record(event string) { + if s.events != nil { + *s.events = append(*s.events, event) + } +} + +type objectStub struct { + events *[]string + putErr error + deleted bool +} + +func (s *objectStub) Put(context.Context, string, string, []byte) error { + if s.events != nil { + *s.events = append(*s.events, "put") + } + return s.putErr +} + +func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (s *objectStub) Delete(context.Context, string) error { + s.deleted = true + return nil +} + +func pngFixture(t *testing.T, width, height int) []byte { + t.Helper() + var data bytes.Buffer + value := image.NewRGBA(image.Rect(0, 0, width, height)) + value.Set(0, 0, color.White) + require.NoError(t, png.Encode(&data, value)) + return data.Bytes() +} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go new file mode 100644 index 00000000..bab14f6d --- /dev/null +++ b/internal/messenger/httpapi/attachments.go @@ -0,0 +1,164 @@ +package httpapi + +import ( + "errors" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { + started := time.Now() + result := "client_error" + defer func() { + s.metrics.attachmentUploads.WithLabelValues(result).Inc() + s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) + }() + + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return invalidRequest() + } + filename, valid := attachmentFilename(query) + if !valid { + return invalidRequest() + } + uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ + ChatID: chatID, OwnerID: user.ID, Filename: filename, + ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + }) + if err != nil { + if errors.Is(err, model.ErrCapacity) { + result = "capacity_rejected" + w.Header().Set("Retry-After", "1") + } else if errors.Is(err, model.ErrUnavailable) { + result = "dependency_error" + } + return attachmentError(err) + } + result = "success" + s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) + writeJSON(w, http.StatusCreated, uploaded) + return nil +} + +func attachmentFilename(query url.Values) (string, bool) { + if len(query) != 1 || len(query["filename"]) != 1 { + return "", false + } + filename := query["filename"][0] + length := utf8.RuneCountInString(filename) + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { + return "", false + } + for _, char := range filename { + if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { + return "", false + } + } + return filename, true +} + +func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + writeJSON(w, http.StatusOK, value) + return nil +} + +func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || !originalVariant(query) { + return invalidRequest() + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + defer func() { _ = content.Body.Close() }() + w.Header().Set("Content-Type", content.Attachment.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, content.Body); err != nil { + s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err) + } + return nil +} + +func originalVariant(query url.Values) bool { + if len(query) == 0 { + return true + } + return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" +} + +func attachmentIDFromRequest(r *http.Request) (string, *apiError) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + attachmentID, valid := canonicalID(parts[3]) + if !valid { + return "", invalidRequest() + } + return attachmentID, nil +} + +func attachmentError(err error) *apiError { + switch { + case errors.Is(err, model.ErrNotFound): + return notFound() + case errors.Is(err, model.ErrInvalidInput): + return invalidRequest() + case errors.Is(err, model.ErrUnsupportedMedia): + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} + case errors.Is(err, model.ErrPayloadTooLarge): + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + case errors.Is(err, model.ErrCapacity): + return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + default: + return unavailable() + } +} diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go new file mode 100644 index 00000000..5baa066b --- /dev/null +++ b/internal/messenger/httpapi/auth.go @@ -0,0 +1,111 @@ +package httpapi + +import ( + "errors" + "io" + "net/http" + "strings" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1)) + if err != nil || len(body) != 0 { + return invalidRequest() + } + username, passwordValue, ok := singleBasicAuth(r) + if !ok { + return unauthorized() + } + credentials, err := s.userRepository.UserByUsername(r.Context(), username) + if errors.Is(err, model.ErrNotFound) { + if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { + return internalError() + } + return unauthorized() + } + if err != nil { + return unavailable() + } + verified, err := s.passwords.Verify(passwordValue, credentials.PasswordHash) + if err != nil { + return internalError() + } + if !verified { + return unauthorized() + } + token, expiresAt, err := s.sessions.Create(r.Context(), credentials.ID, s.sessionTTL) + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User User `json:"user"` + }{token, "Bearer", model.FormatTime(expiresAt), credentials.User}) + return nil +} + +func singleBasicAuth(r *http.Request) (string, string, bool) { + if len(r.Header.Values("Authorization")) != 1 { + return "", "", false + } + username, passwordValue, ok := r.BasicAuth() + return username, passwordValue, ok && username != "" +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodDelete { + return methodNotAllowed(w, http.MethodDelete) + } + _, token, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) { + return unauthorized() + } else if err != nil { + return unavailable() + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) authenticate(r *http.Request) (User, string, *apiError) { + values := r.Header.Values("Authorization") + if len(values) != 1 { + return User{}, "", unauthorized() + } + scheme, token, found := strings.Cut(values[0], " ") + if !found || !strings.EqualFold(scheme, "Bearer") || token == "" || strings.ContainsAny(token, " \t\r\n") { + return User{}, "", unauthorized() + } + userID, err := s.sessions.UserID(r.Context(), token) + if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) { + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + user, err := s.userRepository.UserByID(r.Context(), userID) + if errors.Is(err, model.ErrNotFound) { + _ = s.sessions.Delete(r.Context(), token) + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + return user, token, nil +} + +func authenticationChallenge(r *http.Request) string { + if r.URL.Path == "/api/v1/auth/sessions" { + return `Basic realm="messenger", charset="UTF-8"` + } + return `Bearer realm="messenger"` +} diff --git a/internal/messenger/httpapi/errors.go b/internal/messenger/httpapi/errors.go new file mode 100644 index 00000000..95ffd1c8 --- /dev/null +++ b/internal/messenger/httpapi/errors.go @@ -0,0 +1,61 @@ +package httpapi + +import ( + "encoding/json" + "net/http" +) + +const ( + errorCodeInvalidRequest = "invalid_request" + errorCodeUnauthorized = "unauthorized" + errorCodeNotFound = "not_found" + errorCodeInternal = "internal_error" + errorCodeUnsupportedMediaType = "unsupported_media_type" + errorCodePayloadTooLarge = "payload_too_large" + errorCodeUnavailable = "unavailable" + errorCodeConflict = "conflict" + errorCodeRateLimited = "rate_limited" +) + +type apiError struct { + status int + code string + message string +} + +func methodNotAllowed(w http.ResponseWriter, allowed string) *apiError { + w.Header().Set("Allow", allowed) + return &apiError{http.StatusMethodNotAllowed, errorCodeInvalidRequest, "method not allowed"} +} + +func invalidRequest() *apiError { + return &apiError{http.StatusBadRequest, errorCodeInvalidRequest, "invalid request"} +} + +func unauthorized() *apiError { + return &apiError{http.StatusUnauthorized, errorCodeUnauthorized, "authentication required"} +} + +func notFound() *apiError { + return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"} +} + +func internalError() *apiError { + return &apiError{http.StatusInternalServerError, errorCodeInternal, "internal server error"} +} + +func unavailable() *apiError { + return &apiError{http.StatusServiceUnavailable, errorCodeUnavailable, "service unavailable"} +} + +func writeError(w http.ResponseWriter, requestID string, err *apiError) { + writeJSON(w, err.status, map[string]any{"error": map[string]string{ + "code": err.code, "message": err.message, "request_id": requestID, + }}) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go new file mode 100644 index 00000000..b1e618fc --- /dev/null +++ b/internal/messenger/httpapi/handlers.go @@ -0,0 +1,302 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "regexp" + "strings" + "time" + "unicode/utf8" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (s *Server) live(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + if !s.ready.Load() { + return unavailable() + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := s.database.Ping(ctx); err != nil { + return unavailable() + } + if err := s.sessions.Ping(ctx); err != nil { + return unavailable() + } + if err := s.objects.Ping(ctx); err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + var input struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !usernamePattern.MatchString(input.Username) || !validText(input.DisplayName, 100) || !validPassword(input.Password) { + return invalidRequest() + } + passwordHash, err := s.passwords.Hash(input.Password) + if err != nil { + return internalError() + } + user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) + if errors.Is(err, model.ErrConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"} + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, user) + return nil +} + +func validPassword(value string) bool { + length := utf8.RuneCountInString(value) + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && length >= 12 && length <= 128 +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, err := s.authenticate(r) + if err != nil { + return err + } + writeJSON(w, http.StatusOK, user) + return nil +} + +func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if r.Method == http.MethodGet { + return s.listChats(w, r, user) + } + var input struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !validText(input.Title, 100) || len(input.MemberIDs) < 1 || len(input.MemberIDs) > 100 { + return invalidRequest() + } + members := make(map[string]struct{}, len(input.MemberIDs)) + for i, memberID := range input.MemberIDs { + canonical, valid := canonicalID(memberID) + if !valid { + return invalidRequest() + } + if _, duplicate := members[canonical]; duplicate { + return invalidRequest() + } + members[canonical] = struct{}{} + input.MemberIDs[i] = canonical + } + if _, included := members[user.ID]; !included { + return invalidRequest() + } + chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID) + if errors.Is(err, model.ErrInvalidMembers) { + return invalidRequest() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, chat) + return nil +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *apiError { + limit, cursor, err := pageParams(r, "chat", user.ID) + if err != nil { + return err + } + items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Chat{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("chat", user.ID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, chat) + return nil +} + +func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + if r.Method == http.MethodGet { + return s.listMessages(w, r, user, chatID) + } + key, valid := idempotencyKey(r) + if !valid { + return invalidRequest() + } + var input struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) { + return invalidRequest() + } + message, replay, err := s.messageRepository.CreateMessage( + r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL, + ) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if errors.Is(err, model.ErrIdempotencyConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"} + } + if err != nil { + return unavailable() + } + if replay { + s.metrics.replays.Inc() + } + writeJSON(w, http.StatusCreated, message) + return nil +} + +func validMessage(text string, attachmentIDs []string) bool { + if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 { + return false + } + return len(attachmentIDs) > 0 || strings.TrimSpace(text) != "" +} + +func canonicalAttachmentIDs(attachmentIDs []string) bool { + if len(attachmentIDs) > 10 { + return false + } + seen := make(map[string]struct{}, len(attachmentIDs)) + for index, attachmentID := range attachmentIDs { + canonical, valid := canonicalID(attachmentID) + if !valid { + return false + } + if _, duplicate := seen[canonical]; duplicate { + return false + } + seen[canonical] = struct{}{} + attachmentIDs[index] = canonical + } + return true +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError { + limit, cursor, err := pageParams(r, "message", chatID) + if err != nil { + return err + } + items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrNotFound) { + return notFound() + } + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Message{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("message", chatID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func cursorPosition(cursor *pageCursor) *model.Position { + if cursor == nil { + return nil + } + return &model.Position{CreatedAt: cursor.Time, ID: cursor.ID} +} diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go new file mode 100644 index 00000000..5fd83ebe --- /dev/null +++ b/internal/messenger/httpapi/messenger_interface.go @@ -0,0 +1,49 @@ +package httpapi + +import ( + "context" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type UserRepository interface { + CreateUser(context.Context, string, string, string) (model.User, error) + UserByID(context.Context, string) (model.User, error) + UserByUsername(context.Context, string) (model.UserCredentials, error) +} + +type ChatRepository interface { + CreateChat(context.Context, string, []string, string) (model.Chat, error) + ChatForUser(context.Context, string, string) (model.Chat, error) + ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) +} + +type MessageRepository interface { + CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) + ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) +} + +type AttachmentService interface { + Upload(context.Context, attachment.Upload) (model.Attachment, error) + Metadata(context.Context, string, string) (model.Attachment, error) + Content(context.Context, string, string) (attachment.Content, error) +} + +type HealthChecker interface { + Ping(context.Context) error +} + +type SessionStore interface { + HealthChecker + Create(context.Context, string, time.Duration) (string, time.Time, error) + UserID(context.Context, string) (string, error) + Delete(context.Context, string) error +} + +type PasswordService interface { + Hash(string) (string, error) + Verify(string, string) (bool, error) + DummyHash() string +} diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go new file mode 100644 index 00000000..1dae064e --- /dev/null +++ b/internal/messenger/httpapi/metrics.go @@ -0,0 +1,114 @@ +package httpapi + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type metrics struct { + requests *prometheus.CounterVec + duration *prometheus.HistogramVec + inFlight prometheus.Gauge + replays prometheus.Counter + attachmentUploads *prometheus.CounterVec + attachmentDuration prometheus.Histogram + attachmentBytes prometheus.Counter + handler http.Handler +} + +func newMetrics() *metrics { + registry := prometheus.NewRegistry() + result := &metrics{ + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "messenger", + Subsystem: "http", + Name: "requests_total", + Help: "Completed HTTP requests.", + }, []string{"method", "route", "status_class"}), + duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "messenger", + Subsystem: "http", + Name: "request_duration_seconds", + Help: "HTTP request duration in seconds.", + Buckets: prometheus.DefBuckets, + }, []string{"method", "route"}), + inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "messenger", + Subsystem: "http", + Name: "requests_in_flight", + Help: "HTTP requests currently being served.", + }), + replays: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", + Subsystem: "http", + Name: "idempotency_replays_total", + Help: "Message requests served from an idempotency record.", + }), + attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "messenger", + Name: "attachment_uploads_total", + Help: "Completed attachment uploads by result.", + }, []string{"result"}), + attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "messenger", + Name: "attachment_upload_duration_seconds", + Help: "Attachment upload duration in seconds.", + Buckets: prometheus.DefBuckets, + }), + attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", + Name: "attachment_upload_bytes_total", + Help: "Bytes accepted by successful attachment uploads.", + }), + } + registry.MustRegister( + result.requests, + result.duration, + result.inFlight, + result.replays, + result.attachmentUploads, + result.attachmentDuration, + result.attachmentBytes, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + return result +} + +func (m *metrics) observe(r *http.Request, status int, started time.Time) { + route := routeTemplate(r.URL.Path) + m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc() + m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds()) +} + +func routeTemplate(path string) string { + switch path { + case "/health/live", "/health/ready", "/metrics", "/api/v1/users", "/api/v1/users/me", + "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats": + return path + } + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + return "/api/v1/chats/{chat_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + return "/api/v1/chats/{chat_id}/messages" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + return "/api/v1/chats/{chat_id}/attachments" + } + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + return "/api/v1/attachments/{attachment_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + return "/api/v1/attachments/{attachment_id}/content" + } + return "unknown" +} diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go new file mode 100644 index 00000000..9ecc48d3 --- /dev/null +++ b/internal/messenger/httpapi/model.go @@ -0,0 +1,30 @@ +package httpapi + +import ( + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "github.com/google/uuid" +) + +type User = model.User +type Chat = model.Chat +type Message = model.Message +type Attachment = model.Attachment + +func newID() (string, error) { + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + return id.String(), nil +} + +func canonicalID(value string) (string, bool) { + if len(value) != 36 { + return "", false + } + id, err := uuid.Parse(value) + if err != nil { + return "", false + } + return id.String(), true +} diff --git a/internal/messenger/httpapi/request.go b/internal/messenger/httpapi/request.go new file mode 100644 index 00000000..c0a51591 --- /dev/null +++ b/internal/messenger/httpapi/request.go @@ -0,0 +1,127 @@ +package httpapi + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +const maxRequestBytes = 1 << 20 + +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError { + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "expected application/json"} + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBytes) + data, err := io.ReadAll(r.Body) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "request body too large"} + } + return invalidRequest() + } + if !utf8.Valid(data) { + return invalidRequest() + } + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil || object == nil { + return invalidRequest() + } + normalized, err := json.Marshal(object) + if err != nil { + return invalidRequest() + } + decoder := json.NewDecoder(bytes.NewReader(normalized)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return invalidRequest() + } + return nil +} + +func validText(value string, maxRunes int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes +} + +func idempotencyKey(r *http.Request) (string, bool) { + values := r.Header.Values("Idempotency-Key") + if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) { + return "", false + } + return values[0], true +} + +type pageCursor struct { + Kind string `json:"kind"` + Resource string `json:"resource"` + Time time.Time `json:"time"` + ID string `json:"id"` +} + +func pageParams(r *http.Request, kind, resource string) (int, *pageCursor, *apiError) { + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return 0, nil, invalidRequest() + } + for key := range query { + if key != "limit" && key != "cursor" { + return 0, nil, invalidRequest() + } + if len(query[key]) != 1 { + return 0, nil, invalidRequest() + } + } + limit := 50 + if value, present := query["limit"]; present { + parsed, err := strconv.Atoi(value[0]) + if err != nil || parsed < 1 || parsed > 100 { + return 0, nil, invalidRequest() + } + limit = parsed + } + if value, present := query["cursor"]; present { + cursor, valid := decodeCursor(value[0], kind, resource) + if !valid { + return 0, nil, invalidRequest() + } + return limit, cursor, nil + } + return limit, nil, nil +} + +func encodeCursor(kind, resource string, createdAt time.Time, id string) string { + data, _ := json.Marshal(pageCursor{kind, resource, createdAt, id}) + return base64.RawURLEncoding.EncodeToString(data) +} + +func decodeCursor(value, kind, resource string) (*pageCursor, bool) { + data, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + return nil, false + } + var cursor pageCursor + if err := json.Unmarshal(data, &cursor); err != nil || cursor.Kind != kind || cursor.Resource != resource { + return nil, false + } + if cursor.Time.IsZero() { + return nil, false + } + if _, valid := canonicalID(cursor.ID); !valid { + return nil, false + } + return &cursor, true +} diff --git a/internal/messenger/httpapi/routes.go b/internal/messenger/httpapi/routes.go new file mode 100644 index 00000000..f2727674 --- /dev/null +++ b/internal/messenger/httpapi/routes.go @@ -0,0 +1,47 @@ +package httpapi + +import ( + "net/http" + "strings" +) + +type requestHandler func(http.ResponseWriter, *http.Request) *apiError + +func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError { + var handler requestHandler + switch r.URL.Path { + case "/health/live": + handler = s.live + case "/health/ready": + handler = s.readiness + case "/metrics": + s.metrics.handler.ServeHTTP(w, r) + return nil + case "/api/v1/users": + handler = s.users + case "/api/v1/users/me": + handler = s.me + case "/api/v1/auth/sessions": + handler = s.createSession + case "/api/v1/auth/sessions/current": + handler = s.revokeSession + case "/api/v1/chats": + handler = s.chats + default: + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + handler = s.chat + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + handler = s.messages + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + handler = s.uploadAttachment + } else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + handler = s.attachmentMetadata + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + handler = s.attachmentContent + } else { + return notFound() + } + } + return handler(w, r) +} diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go new file mode 100644 index 00000000..9af19cff --- /dev/null +++ b/internal/messenger/httpapi/server.go @@ -0,0 +1,133 @@ +package httpapi + +import ( + "context" + "log/slog" + "net/http" + "sync/atomic" + "time" +) + +type Dependencies struct { + Users UserRepository + Chats ChatRepository + Messages MessageRepository + Attachments AttachmentService + Sessions SessionStore + Passwords PasswordService + Database HealthChecker + Objects HealthChecker +} + +type Config struct { + SessionTTL time.Duration + IdempotencyTTL time.Duration + RequestTTL time.Duration + InstanceID string + SwaggerPort string +} + +type Server struct { + userRepository UserRepository + chatRepository ChatRepository + messageRepository MessageRepository + attachments AttachmentService + sessions SessionStore + passwords PasswordService + database HealthChecker + objects HealthChecker + sessionTTL time.Duration + idempotencyTTL time.Duration + requestTTL time.Duration + instanceID string + logger *slog.Logger + swaggerPort string + metrics *metrics + ready atomic.Bool +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func NewServer( + logger *slog.Logger, + dependencies Dependencies, + config Config, +) *Server { + s := &Server{ + userRepository: dependencies.Users, chatRepository: dependencies.Chats, + messageRepository: dependencies.Messages, + attachments: dependencies.Attachments, sessions: dependencies.Sessions, + passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, + sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(), + } + s.ready.Store(true) + return s +} + +func (s *Server) SetReady(ready bool) { + s.ready.Store(ready) +} + +func (s *Server) Handler() http.Handler { + return http.HandlerFunc(s.serve) +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", s.instanceID) + requestID, err := newID() + if err != nil { + http.Error(w, "server unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("X-Request-Id", requestID) + w.Header().Set("Cache-Control", "no-store") + wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK} + started := time.Now() + s.metrics.inFlight.Inc() + defer func() { + s.metrics.inFlight.Dec() + if recovered := recover(); recovered != nil { + s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path) + if wrapped.status == http.StatusOK { + writeError(wrapped, requestID, internalError()) + } + } + s.metrics.observe(r, wrapped.status, started) + s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) + }() + if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { + wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key") + wrapped.WriteHeader(http.StatusNoContent) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.requestTTL) + defer cancel() + r = r.WithContext(ctx) + if apiErr := s.route(wrapped, r); apiErr != nil { + if apiErr.status == http.StatusUnauthorized { + wrapped.Header().Set("WWW-Authenticate", authenticationChallenge(r)) + } + writeError(wrapped, requestID, apiErr) + } +} + +func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin != "http://localhost:"+s.swaggerPort && origin != "http://127.0.0.1:"+s.swaggerPort { + return false + } + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id") + w.Header().Add("Vary", "Origin") + return true +} diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go new file mode 100644 index 00000000..8199a0d7 --- /dev/null +++ b/internal/messenger/httpapi/server_test.go @@ -0,0 +1,333 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +var errDependency = errors.New("dependency unavailable") + +type MessengerSuite struct { + suite.Suite + logger *slog.Logger +} + +func TestMessengerSuite(t *testing.T) { + suite.Run(t, new(MessengerSuite)) +} + +func (s *MessengerSuite) SetupTest() { + s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +type messengerStoreStub struct { + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error +} + +func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } +func (messengerStoreStub) CreateUser(context.Context, string, string, string) (model.User, error) { + return model.User{}, nil +} +func (s messengerStoreStub) UserByID(context.Context, string) (model.User, error) { + return s.userByID, s.userByIDErr +} +func (s messengerStoreStub) UserByUsername(context.Context, string) (model.UserCredentials, error) { + return s.credentials, s.credentialsErr +} +func (messengerStoreStub) CreateChat(context.Context, string, []string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) { + return nil, false, nil +} +func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) { + return s.message, s.messageReplay, s.messageErr +} +func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { + return nil, false, nil +} + +type objectStoreStub struct { + pingErr error +} + +func (s objectStoreStub) Ping(context.Context) error { return s.pingErr } + +type attachmentServiceStub struct { + uploaded model.Attachment + err error +} + +func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) { + return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err +} + +type sessionStub struct { + pingErr error + userID string + userIDErr error +} + +func (s sessionStub) Ping(context.Context) error { return s.pingErr } +func (sessionStub) Create(context.Context, string, time.Duration) (string, time.Time, error) { + return "token", time.Now().Add(time.Hour), nil +} +func (s sessionStub) UserID(context.Context, string) (string, error) { return s.userID, s.userIDErr } +func (sessionStub) Delete(context.Context, string) error { return nil } + +type passwordsStub struct { + verified bool + err error +} + +func (passwordsStub) Hash(string) (string, error) { return "hash", nil } +func (passwordsStub) DummyHash() string { return "hash" } +func (s passwordsStub) Verify(string, string) (bool, error) { + return s.verified, s.err +} + +func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + objects objectStoreStub + }{ + {name: "postgres", store: messengerStoreStub{pingErr: errDependency}}, + {name: "valkey", sessions: sessionStub{pingErr: errDependency}}, + {name: "s3", objects: objectStoreStub{pingErr: errDependency}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{}) + response := serveRequest(server, http.MethodGet, "/health/ready", "") + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + }{ + {name: "valkey", sessions: sessionStub{userIDErr: errDependency}}, + {name: "postgres", store: messengerStoreStub{userByIDErr: errDependency}, sessions: sessionStub{userID: "user-id"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", "Bearer valid-token") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestAuthenticationChallenges() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "") + if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` { + t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate")) + } + + store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}} + server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("user", "wrong-password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized || response.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatalf("session challenge = %d %q", response.Code, response.Header().Get("WWW-Authenticate")) + } +} + +func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest( + server, + http.MethodGet, + "/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid", + "", + ) + s.Equal(http.StatusUnauthorized, response.Code) + s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate")) +} + +func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + request.ContentLength = -1 + request.SetBasicAuth("user", "password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + for _, response := range []*httptest.ResponseRecorder{ + serveRequest(server, http.MethodGet, "/health/live", ""), + serveRequest(server, http.MethodGet, "/api/v1/users/me", ""), + } { + if value := response.Header().Get("X-Instance-Id"); value != "test-instance" { + t.Fatalf("X-Instance-Id = %q, want test-instance", value) + } + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() { + t := s.T() + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + tests := []struct { + name string + key string + }{ + {name: "missing"}, + {name: "invalid", key: "invalid key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + if tt.key != "" { + request.Header.Set("Idempotency-Key", tt.key) + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } + }) + } +} + +func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() { + t := s.T() + store := messengerStoreStub{ + userByID: model.User{ID: "user-id"}, + messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict), + } + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "same-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func (s *MessengerSuite) TestAttachmentFilenameValidation() { + for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} { + _, valid := attachmentFilename(map[string][]string{"filename": {filename}}) + s.False(valid, filename) + } + filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}}) + s.True(valid) + s.Equal("фото.png", filename) +} + +func (s *MessengerSuite) TestUploadRejectsMalformedQuery() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG", + strings.NewReader("image"), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + s.Equal(http.StatusBadRequest, response.Code) +} + +func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server { + return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{}) +} + +func (s *MessengerSuite) server( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + objects objectStoreStub, + attachments AttachmentService, +) *Server { + return NewServer(s.logger, Dependencies{ + Users: store, Chats: store, Messages: store, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: store, Objects: objects, + }, Config{ + SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second, + InstanceID: "test-instance", SwaggerPort: "8081", + }) +} + +func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, strings.NewReader(body)) + if body != "" { + request.Header.Set("Content-Type", "application/json") + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} diff --git a/internal/model/errors.go b/internal/model/errors.go new file mode 100644 index 00000000..dbe09f09 --- /dev/null +++ b/internal/model/errors.go @@ -0,0 +1,18 @@ +package model + +import "errors" + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrInvalidMembers = errors.New("invalid members") + ErrInvalidCursor = errors.New("invalid cursor") + ErrIdempotencyConflict = errors.New("idempotency conflict") + ErrInvalidToken = errors.New("invalid token") + ErrSessionNotFound = errors.New("session not found") + ErrInvalidInput = errors.New("invalid input") + ErrUnsupportedMedia = errors.New("unsupported media") + ErrPayloadTooLarge = errors.New("payload too large") + ErrCapacity = errors.New("capacity exhausted") + ErrUnavailable = errors.New("dependency unavailable") +) diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 00000000..bc71b978 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,153 @@ +package model + +import ( + "encoding/json" + "time" +) + +const ( + AttachmentPending = "pending" + AttachmentReady = "ready" + AttachmentDeleting = "deleting" +) + +const timestampLayout = "2006-01-02T15:04:05.000000Z" + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt time.Time `json:"created_at"` +} + +type UserCredentials struct { + User + PasswordHash string `json:"-"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` +} + +type StoredObject struct { + Key string + Size int64 + LastModified time.Time +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` +} + +type Position struct { + CreatedAt time.Time + ID string +} + +type userJSON struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type chatJSON struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type messageJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type attachmentJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (u User) MarshalJSON() ([]byte, error) { + return json.Marshal(userJSON{u.ID, u.Username, u.DisplayName, formatTime(u.CreatedAt)}) +} + +func (c Chat) MarshalJSON() ([]byte, error) { + return json.Marshal(chatJSON{c.ID, c.Title, c.MemberIDs, formatTime(c.CreatedAt)}) +} + +func (m Message) MarshalJSON() ([]byte, error) { + attachments := m.AttachmentIDs + if attachments == nil { + attachments = []string{} + } + return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments}) +} + +func (a Attachment) MarshalJSON() ([]byte, error) { + variants := a.Variants + if variants == nil { + variants = []Variant{} + } + return json.Marshal(attachmentJSON{ + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt), + }) +} + +func FormatTime(value time.Time) string { + return formatTime(value) +} + +func formatTime(value time.Time) string { + return value.UTC().Format(timestampLayout) +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 00000000..d3a54d9c --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,24 @@ +package model + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type ModelSuite struct { + suite.Suite +} + +func TestModelSuite(t *testing.T) { + suite.Run(t, new(ModelSuite)) +} + +func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() { + createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC) + data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt}) + s.Require().NoError(err) + s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data)) +} diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go new file mode 100644 index 00000000..ac41f65d --- /dev/null +++ b/internal/objectstore/store.go @@ -0,0 +1,176 @@ +package objectstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +var ErrNotFound = errors.New("object not found") + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseTLS bool + Timeout time.Duration +} + +type Store struct { + client *minio.Client + bucket string + timeout time.Duration +} + +func New(config Config) (*Store, error) { + endpoint, err := endpointHost(config.Endpoint, config.UseTLS) + if err != nil { + return nil, err + } + if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 { + return nil, errors.New("invalid object store configuration") + } + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), + Secure: config.UseTLS, + Region: config.Region, + }) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil +} + +func endpointHost(value string, useTLS bool) (string, error) { + value = strings.TrimSpace(value) + if !strings.Contains(value, "://") { + if value == "" { + return "", errors.New("S3 endpoint is required") + } + return value, nil + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" { + return "", errors.New("invalid S3 endpoint") + } + if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS") + } + return parsed.Host, nil +} + +func (s *Store) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return fmt.Errorf("check S3 bucket: %w", err) + } + if !exists { + return errors.New("S3 bucket does not exist") + } + return nil +} + +func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.client.PutObject( + ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, + ) + if err != nil { + return fmt.Errorf("put S3 object: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + cancel() + return nil, objectError("get S3 object", err) + } + if _, err := object.Stat(); err != nil { + cancel() + _ = object.Close() + return nil, objectError("stat S3 object", err) + } + return &timeoutReader{ReadCloser: object, cancel: cancel}, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return model.StoredObject{}, objectError("stat S3 object", err) + } + return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Delete(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + return objectError("delete S3 object", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", errors.New("object list limit must be between 1 and 1000") + } + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + objects := make([]model.StoredObject, 0, limit+1) + options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1} + for info := range s.client.ListObjects(ctx, s.bucket, options) { + if info.Err != nil { + return nil, "", fmt.Errorf("list S3 objects: %w", info.Err) + } + objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) + if len(objects) == limit+1 { + cancel() + break + } + } + var next string + if len(objects) > limit { + objects = objects[:limit] + next = objects[len(objects)-1].Key + } + return objects, next, nil +} + +type timeoutReader struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (r *timeoutReader) Close() error { + err := r.ReadCloser.Close() + r.cancel() + return err +} + +func objectError(action string, err error) error { + response := minio.ToErrorResponse(err) + if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" { + return ErrNotFound + } + return fmt.Errorf("%s: %w", action, err) +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 00000000..5dd8b1b1 --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,133 @@ +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +var ( + ErrInvalidConfig = errors.New("invalid password configuration") + ErrInvalidHash = errors.New("invalid password hash") +) + +type Config struct { + MemoryKiB uint32 + Iterations uint32 + Parallelism uint8 + SaltLength uint32 + KeyLength uint32 +} + +type Hasher struct { + config Config + dummyHash string +} + +func New(config Config) (*Hasher, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + salt := make([]byte, config.SaltLength) + digest := argon2.IDKey([]byte("invalid-user-password"), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return &Hasher{config: config, dummyHash: encode(config, salt, digest)}, nil +} + +func validateConfig(config Config) error { + if config.MemoryKiB < 8*1024 || config.MemoryKiB > 1024*1024 || + config.Iterations < 1 || config.Iterations > 20 || + config.Parallelism < 1 || + config.SaltLength < 8 || config.SaltLength > 1024 || + config.KeyLength < 16 || config.KeyLength > 1024 { + return ErrInvalidConfig + } + return nil +} + +func (h *Hasher) Hash(value string) (string, error) { + salt := make([]byte, h.config.SaltLength) + if _, err := rand.Read(salt); err != nil { + return "", fmt.Errorf("generate password salt: %w", err) + } + digest := argon2.IDKey([]byte(value), salt, h.config.Iterations, h.config.MemoryKiB, h.config.Parallelism, h.config.KeyLength) + return encode(h.config, salt, digest), nil +} + +func (h *Hasher) Verify(value, encodedHash string) (bool, error) { + config, salt, digest, err := decode(encodedHash) + if err != nil { + return false, err + } + actual := argon2.IDKey([]byte(value), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return subtle.ConstantTimeCompare(actual, digest) == 1, nil +} + +func (h *Hasher) DummyHash() string { + return h.dummyHash +} + +func encode(config Config, salt, digest []byte) string { + return fmt.Sprintf( + "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + config.MemoryKiB, + config.Iterations, + config.Parallelism, + base64.RawStdEncoding.EncodeToString(salt), + base64.RawStdEncoding.EncodeToString(digest), + ) +} + +func decode(encodedHash string) (Config, []byte, []byte, error) { + parts := strings.Split(encodedHash, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" || parts[2] != fmt.Sprintf("v=%d", argon2.Version) { + return Config{}, nil, nil, ErrInvalidHash + } + + parameters := strings.Split(parts[3], ",") + if len(parameters) != 3 { + return Config{}, nil, nil, ErrInvalidHash + } + memory, err := parameter(parameters[0], "m=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + iterations, err := parameter(parameters[1], "t=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + parallelism, err := parameter(parameters[2], "p=", 8) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + + salt, err := base64.RawStdEncoding.Strict().DecodeString(parts[4]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + digest, err := base64.RawStdEncoding.Strict().DecodeString(parts[5]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + config := Config{ + MemoryKiB: uint32(memory), Iterations: uint32(iterations), Parallelism: uint8(parallelism), + SaltLength: uint32(len(salt)), KeyLength: uint32(len(digest)), + } + if err := validateConfig(config); err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + return config, salt, digest, nil +} + +func parameter(value, prefix string, bitSize int) (uint64, error) { + if !strings.HasPrefix(value, prefix) { + return 0, ErrInvalidHash + } + return strconv.ParseUint(strings.TrimPrefix(value, prefix), 10, bitSize) +} diff --git a/internal/password/password_test.go b/internal/password/password_test.go new file mode 100644 index 00000000..8e88875a --- /dev/null +++ b/internal/password/password_test.go @@ -0,0 +1,44 @@ +package password + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type PasswordSuite struct { + suite.Suite + hasher *Hasher +} + +func TestPasswordSuite(t *testing.T) { + suite.Run(t, new(PasswordSuite)) +} + +func (s *PasswordSuite) SetupTest() { + hasher, err := New(testConfig()) + s.Require().NoError(err) + s.hasher = hasher +} + +func (s *PasswordSuite) TestHasher() { + hash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + + verified, err := s.hasher.Verify("T3st-password-123", hash) + s.Require().NoError(err) + s.True(verified) + + verified, err = s.hasher.Verify("wrong-password", hash) + s.Require().NoError(err) + s.False(verified) +} + +func (s *PasswordSuite) TestVerifyRejectsMalformedHash() { + _, err := s.hasher.Verify("password", "invalid") + s.Error(err) +} + +func testConfig() Config { + return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32} +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go new file mode 100644 index 00000000..8728ae3c --- /dev/null +++ b/internal/postgres/attachments.go @@ -0,0 +1,254 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +const attachmentColumns = ` + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, created_at, updated_at, ready_at` + +type rowScanner interface { + Scan(...any) error +} + +func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { + query := ` + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING ` + attachmentColumns + + err := scanAttachment(s.pool.QueryRow( + ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename, + attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey, + ), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + query := ` + UPDATE attachments + SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error { + const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'` + if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil { + return fmt.Errorf("delete pending attachment: %w", err) + } + return nil +} + +func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 + WHERE a.id = $1 AND a.status = 'ready'` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)` + var exists bool + if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { + return false, fmt.Errorf("check attachment by object key: %w", err) + } + return exists, nil +} + +func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") + } + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + `, + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) + FROM attachments a + WHERE a.object_key > $1 + ORDER BY a.object_key + LIMIT $2` + rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) + if err != nil { + return nil, "", fmt.Errorf("list attachments: %w", err) + } + defer rows.Close() + + attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { + var attachment model.Attachment + err := scanAttachmentWithReference(row, &attachment) + return publicAttachment(attachment), err + })) + if err != nil { + return nil, "", fmt.Errorf("scan attachments: %w", err) + } + var next string + if len(attachments) > limit { + attachments = attachments[:limit] + next = attachments[len(attachments)-1].ObjectKey + } + return attachments, next, nil +} + +func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { + return row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.Referenced, + ) +} + +func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` + var referenced bool + if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil { + return false, fmt.Errorf("check attachment references: %w", err) + } + return referenced, nil +} + +func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return false, fmt.Errorf("begin attachment deletion claim: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + var status string + var createdAt time.Time + var readyAt *time.Time + err = tx.QueryRow( + ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID, + ).Scan(&status, &createdAt, &readyAt) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("lock attachment deletion candidate: %w", err) + } + if status != candidate.Status { + return false, nil + } + var age time.Time + switch status { + case model.AttachmentPending: + age = createdAt + case model.AttachmentReady: + if readyAt == nil { + return false, nil + } + age = *readyAt + default: + return false, nil + } + if age.After(createdBefore) { + return false, nil + } + var referenced bool + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil { + return false, fmt.Errorf("recheck attachment references: %w", err) + } + if referenced { + return false, nil + } + if _, err := tx.Exec( + ctx, + `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`, + candidate.ID, candidate.Status, + ); err != nil { + return false, fmt.Errorf("mark attachment deleting: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return false, fmt.Errorf("commit attachment deletion claim: %w", err) + } + return true, nil +} + +func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) { + const query = ` + DELETE FROM attachments a + WHERE a.id = $1 + AND a.status = 'deleting' + AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)` + result, err := s.pool.Exec(ctx, query, attachmentID) + if err != nil { + return false, fmt.Errorf("delete claimed attachment: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func scanAttachment(row rowScanner, attachment *model.Attachment) error { + return row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + ) +} + +func attachmentColumnsWithAlias(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ + "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at", + alias, + ) +} + +func publicAttachment(attachment model.Attachment) model.Attachment { + attachment.Variants = []model.Variant{} + return attachment +} diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go new file mode 100644 index 00000000..0e56e166 --- /dev/null +++ b/internal/postgres/errors.go @@ -0,0 +1,11 @@ +package postgres + +import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + +var ( + ErrConflict = model.ErrConflict + ErrNotFound = model.ErrNotFound + ErrInvalidMembers = model.ErrInvalidMembers + ErrInvalidCursor = model.ErrInvalidCursor + ErrIdempotencyConflict = model.ErrIdempotencyConflict +) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go new file mode 100644 index 00000000..49f576ef --- /dev/null +++ b/internal/postgres/idempotency.go @@ -0,0 +1,364 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +const expiredIdempotencyCleanupBatch = 100 + +func (s *Store) CreateMessage( + ctx context.Context, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + var ( + tx pgx.Tx + err error + visible bool + requestHash [sha256.Size]byte + message model.Message + replay bool + ) + + tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID) + if err != nil { + return model.Message{}, false, err + } + if !visible { + return model.Message{}, false, ErrNotFound + } + validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + if !validAttachments { + return model.Message{}, false, ErrNotFound + } + if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { + return model.Message{}, false, err + } + + requestHash, err = messageRequestHash(text, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + message, replay, err = createIdempotentMessage( + ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err) + } + return message, replay, nil +} + +func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + payload, err := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, attachmentIDs}) + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err) + } + return sha256.Sum256(payload), nil +} + +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) { + if len(attachmentIDs) == 0 { + return true, nil + } + const query = ` + SELECT id + FROM attachments + WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready' + FOR UPDATE` + rows, err := tx.Query(ctx, query, chatID, attachmentIDs) + if err != nil { + return false, fmt.Errorf("validate message attachments: %w", err) + } + defer rows.Close() + count := 0 + for rows.Next() { + count++ + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("validate message attachments: %w", err) + } + return count == len(attachmentIDs), nil +} + +func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var visible bool + if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + return visible, nil +} + +func createIdempotentMessage( + ctx context.Context, + tx pgx.Tx, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + requestHash []byte, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + for { + var ( + record idempotencyRecord + found bool + err error + messageID string + reserved bool + message model.Message + ) + + record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey) + if err != nil { + return model.Message{}, false, err + } + if found && record.active { + if !bytes.Equal(record.requestHash, requestHash) { + return model.Message{}, false, ErrIdempotencyConflict + } + message, err = messageByID(ctx, tx, record.messageID) + return message, true, err + } + if found { + if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil { + return model.Message{}, false, err + } + } + + messageID = uuid.NewString() + reserved, err = reserveIdempotencyKey( + ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if !reserved { + continue + } + + message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs) + return message, false, err + } +} + +func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error { + const query = ` + WITH expired AS ( + SELECT ctid + FROM message_idempotency + WHERE expires_at <= statement_timestamp() + ORDER BY expires_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM message_idempotency records + USING expired + WHERE records.ctid = expired.ctid` + + if _, err := tx.Exec(ctx, query, limit); err != nil { + return fmt.Errorf("delete expired idempotency records: %w", err) + } + return nil +} + +type idempotencyRecord struct { + requestHash []byte + messageID string + active bool +} + +func lockIdempotencyRecord( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, +) (idempotencyRecord, bool, error) { + const query = ` + SELECT request_hash, message_id, expires_at > statement_timestamp() + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + FOR UPDATE` + + var ( + record idempotencyRecord + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key).Scan( + &record.requestHash, + &record.messageID, + &record.active, + ) + if errors.Is(err, pgx.ErrNoRows) { + return idempotencyRecord{}, false, nil + } + if err != nil { + return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err) + } + return record, true, nil +} + +func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error { + const query = ` + DELETE FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3` + if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil { + return fmt.Errorf("delete expired idempotency key: %w", err) + } + return nil +} + +func reserveIdempotencyKey( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, + requestHash []byte, + messageID string, + ttl time.Duration, +) (bool, error) { + const query = ` + INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at) + VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second') + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING true` + + var ( + inserted bool + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reserve idempotency key: %w", err) + } + return inserted, nil +} + +func insertMessage( + ctx context.Context, + tx pgx.Tx, + messageID string, + chatID string, + senderID string, + text string, + attachmentIDs []string, +) (model.Message, error) { + const query = ` + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + ) + if err != nil { + return model.Message{}, fmt.Errorf("insert message: %w", err) + } + if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil { + return model.Message{}, err + } + message.AttachmentIDs = make([]string, len(attachmentIDs)) + copy(message.AttachmentIDs, attachmentIDs) + return message, nil +} + +func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error { + const query = ` + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES ($1, $2, $3, $4)` + for position, attachmentID := range attachmentIDs { + if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil { + return fmt.Errorf("insert message attachment: %w", err) + } + } + return nil +} + +func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) { + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.id = $1` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, fmt.Errorf("idempotency result message not found") + } + if err != nil { + return model.Message{}, fmt.Errorf("get idempotency result message: %w", err) + } + return message, nil +} diff --git a/internal/postgres/store.go b/internal/postgres/store.go new file mode 100644 index 00000000..5413f28d --- /dev/null +++ b/internal/postgres/store.go @@ -0,0 +1,442 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Ping(ctx context.Context) error { + return s.pool.Ping(ctx) +} + +func (s *Store) CreateUser( + ctx context.Context, + username string, + displayName string, + passwordHash string, +) (model.User, error) { + const query = ` + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at` + + var user model.User + err := s.pool.QueryRow(ctx, query, uuid.NewString(), username, displayName, passwordHash).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if isUniqueViolation(err) { + return model.User{}, ErrConflict + } + if err != nil { + return model.User{}, fmt.Errorf("create user: %w", err) + } + + return user, nil +} + +func (s *Store) UserByID(ctx context.Context, id string) (model.User, error) { + const query = ` + SELECT id, username, display_name, created_at + FROM users + WHERE id = $1` + + var user model.User + err := s.pool.QueryRow(ctx, query, id).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.User{}, ErrNotFound + } + if err != nil { + return model.User{}, fmt.Errorf("get user by id: %w", err) + } + + return user, nil +} + +func (s *Store) UserByUsername(ctx context.Context, username string) (model.UserCredentials, error) { + const query = ` + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1` + + var credentials model.UserCredentials + err := s.pool.QueryRow(ctx, query, username).Scan( + &credentials.ID, + &credentials.Username, + &credentials.DisplayName, + &credentials.CreatedAt, + &credentials.PasswordHash, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.UserCredentials{}, ErrNotFound + } + if err != nil { + return model.UserCredentials{}, fmt.Errorf("get user by username: %w", err) + } + + return credentials, nil +} + +func (s *Store) CreateChat( + ctx context.Context, + title string, + memberIDs []string, + callerID string, +) (model.Chat, error) { + if !validMembers(memberIDs, callerID) { + return model.Chat{}, ErrInvalidMembers + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + valid, err := allUsersExist(ctx, tx, memberIDs) + if err != nil { + return model.Chat{}, err + } + if !valid { + return model.Chat{}, ErrInvalidMembers + } + + chat, err := insertChat(ctx, tx, title) + if err != nil { + return model.Chat{}, err + } + if err := insertMembers(ctx, tx, chat.ID, memberIDs); err != nil { + return model.Chat{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Chat{}, fmt.Errorf("commit create chat transaction: %w", err) + } + + chat.MemberIDs = append([]string(nil), memberIDs...) + return chat, nil +} + +func validMembers(memberIDs []string, callerID string) bool { + callerUUID, err := uuid.Parse(callerID) + if err != nil { + return false + } + + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + callerIncluded := false + for _, memberID := range memberIDs { + memberUUID, err := uuid.Parse(memberID) + if err != nil { + return false + } + if _, exists := seen[memberUUID]; exists { + return false + } + + seen[memberUUID] = struct{}{} + callerIncluded = callerIncluded || memberUUID == callerUUID + } + + return callerIncluded +} + +func allUsersExist(ctx context.Context, tx pgx.Tx, memberIDs []string) (bool, error) { + const query = ` + SELECT count(*) + FROM users + WHERE id::text = ANY($1)` + + var count int + if err := tx.QueryRow(ctx, query, memberIDs).Scan(&count); err != nil { + return false, fmt.Errorf("check chat members: %w", err) + } + + return count == len(memberIDs), nil +} + +func insertChat(ctx context.Context, tx pgx.Tx, title string) (model.Chat, error) { + const query = ` + INSERT INTO chats (id, title) + VALUES ($1, $2) + RETURNING id, title, created_at` + + var chat model.Chat + if err := tx.QueryRow(ctx, query, uuid.NewString(), title).Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil { + return model.Chat{}, fmt.Errorf("insert chat: %w", err) + } + + return chat, nil +} + +func insertMembers(ctx context.Context, tx pgx.Tx, chatID string, memberIDs []string) error { + const query = ` + INSERT INTO chat_members (chat_id, user_id, position) + VALUES ($1, $2, $3)` + + batch := &pgx.Batch{} + for position, memberID := range memberIDs { + batch.Queue(query, chatID, memberID, position) + } + + results := tx.SendBatch(ctx, batch) + defer func() { _ = results.Close() }() + for range memberIDs { + if _, err := results.Exec(); err != nil { + return fmt.Errorf("insert chat member: %w", err) + } + } + if err := results.Close(); err != nil { + return fmt.Errorf("close chat members batch: %w", err) + } + + return nil +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (model.Chat, error) { + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id::text ORDER BY cm.position) + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = $1 + AND EXISTS ( + SELECT 1 + FROM chat_members visible + WHERE visible.chat_id = c.id AND visible.user_id = $2 + ) + GROUP BY c.id, c.title, c.created_at` + + var chat model.Chat + err := s.pool.QueryRow(ctx, query, chatID, userID).Scan( + &chat.ID, + &chat.Title, + &chat.CreatedAt, + &chat.MemberIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Chat{}, ErrNotFound + } + if err != nil { + return model.Chat{}, fmt.Errorf("get chat for user: %w", err) + } + + return chat, nil +} + +func (s *Store) ListChats( + ctx context.Context, + userID string, + after *model.Position, + limit int, +) ([]model.Chat, bool, error) { + if after != nil { + valid, err := s.visibleChatPositionExists(ctx, userID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(all_members.user_id::text ORDER BY all_members.position) + FROM chats c + JOIN chat_members visible ON visible.chat_id = c.id AND visible.user_id = $1 + JOIN chat_members all_members ON all_members.chat_id = c.id + WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at, c.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, userID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list chats: %w", err) + } + defer rows.Close() + + chats, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Chat](func(row pgx.CollectableRow) (model.Chat, error) { + var chat model.Chat + err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + return chat, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan chats: %w", err) + } + + return page(chats, limit) +} + +func (s *Store) visibleChatPositionExists( + ctx context.Context, + userID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1 AND c.created_at = $2 AND c.id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, userID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate chat cursor: %w", err) + } + + return exists, nil +} + +func (s *Store) ListMessages( + ctx context.Context, + chatID string, + userID string, + after *model.Position, + limit int, +) ([]model.Message, bool, error) { + visible, err := s.chatVisible(ctx, chatID, userID) + if err != nil { + return nil, false, err + } + if !visible { + return nil, false, ErrNotFound + } + if after != nil { + valid, err := s.messagePositionExists(ctx, chatID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.chat_id = $1 + AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid)) + ORDER BY m.created_at, m.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, chatID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list messages: %w", err) + } + defer rows.Close() + + messages, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Message](func(row pgx.CollectableRow) (model.Message, error) { + var message model.Message + err := row.Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + return message, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan messages: %w", err) + } + + return page(messages, limit) +} + +func (s *Store) chatVisible(ctx context.Context, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, userID).Scan(&exists); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + + return exists, nil +} + +func (s *Store) messagePositionExists( + ctx context.Context, + chatID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM messages + WHERE chat_id = $1 AND created_at = $2 AND id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate message cursor: %w", err) + } + + return exists, nil +} + +func page[T any](items []T, limit int) ([]T, bool, error) { + if limit < 1 { + return nil, false, fmt.Errorf("limit must be positive") + } + if len(items) <= limit { + return items, false, nil + } + + return items[:limit], true, nil +} + +func isUniqueViolation(err error) bool { + var postgresError *pgconn.PgError + return errors.As(err, &postgresError) && postgresError.Code == "23505" +} diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go new file mode 100644 index 00000000..5974d280 --- /dev/null +++ b/internal/postgres/store_integration_test.go @@ -0,0 +1,401 @@ +package postgres_test + +import ( + "context" + "errors" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +type PostgresSuite struct { + suite.Suite + pool *pgxpool.Pool + store *postgres.Store +} + +func TestPostgresSuite(t *testing.T) { + suite.Run(t, new(PostgresSuite)) +} + +func (s *PostgresSuite) SetupSuite() { + s.pool = openTestDatabase(s.T()) + s.store = postgres.New(s.pool) +} + +func (s *PostgresSuite) TearDownSuite() { + if s.pool != nil { + s.pool.Close() + } +} + +func (s *PostgresSuite) TestConcurrentUsernameCreation() { + t := s.T() + store := s.store + username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] + + const workers = 8 + results := make(chan error, workers) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + _, err := store.CreateUser(context.Background(), username, "Race user", "password-hash") + results <- err + }() + } + group.Wait() + close(results) + + var created, conflicts int + for err := range results { + switch { + case err == nil: + created++ + case errors.Is(err, postgres.ErrConflict): + conflicts++ + default: + t.Fatalf("CreateUser() error = %v", err) + } + } + if created != 1 || conflicts != workers-1 { + t.Fatalf("created = %d, conflicts = %d", created, conflicts) + } +} + +func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() { + t := s.T() + pool := s.pool + store := s.store + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + + before := chatCount(t, pool) + _, err = store.CreateChat(context.Background(), "Invalid chat", []string{owner.ID, uuid.NewString()}, owner.ID) + if !errors.Is(err, postgres.ErrInvalidMembers) { + t.Fatalf("CreateChat() error = %v, want ErrInvalidMembers", err) + } + if after := chatCount(t, pool); after != before { + t.Fatalf("chat count changed from %d to %d", before, after) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + key := uuid.NewString() + + first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour) + if err != nil || replay { + t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err) + } + second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour) + if err != nil || !replay { + t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err) + } + s.Equal(first, second) + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { + t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash") + s.Require().NoError(err) + attachment := model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + } + pending, err := s.store.CreatePendingAttachment(context.Background(), attachment) + s.Require().NoError(err) + created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + s.Equal("ready", created.Status) + s.Empty(created.Variants) + s.Nil(created.ErrorCode) + + visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID) + s.Require().NoError(err) + s.Equal(created, visible) + _, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID) + s.ErrorIs(err, postgres.ErrNotFound) + + message, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + s.Equal([]string{created.ID}, message.AttachmentIDs) + + messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10) + s.Require().NoError(err) + s.Len(messages, 1) + s.Equal([]string{created.ID}, messages[0].AttachmentIDs) + referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) + s.Require().NoError(err) + s.True(referenced) + deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(deleted) +} + +func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID) + s.Require().NoError(err) + s.True(deleted) + + pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + + claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(claimed) + visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, visible.Status) +} + +func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { + owner, firstChatID := createOwnerAndChat(s.T(), s.store) + secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) + s.Require().NoError(err) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + _, _, err = s.store.CreateMessage( + context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrNotFound) +} + +func (s *PostgresSuite) TestConcurrentMessageIdempotency() { + t := s.T() + pool := s.pool + stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)} + owner, chatID := createOwnerAndChat(t, stores[0]) + key := uuid.NewString() + + const workers = 8 + type result struct { + message model.Message + replay bool + err error + } + results := make(chan result, workers) + var group sync.WaitGroup + for worker := range workers { + group.Add(1) + go func() { + defer group.Done() + message, replay, err := stores[worker%len(stores)].CreateMessage( + context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour, + ) + results <- result{message: message, replay: replay, err: err} + }() + } + group.Wait() + close(results) + + var messageID string + var created int + for result := range results { + if result.err != nil { + t.Fatalf("CreateMessage() error = %v", result.err) + } + if messageID == "" { + messageID = result.message.ID + } + if result.message.ID != messageID { + t.Fatalf("message ID = %s, want %s", result.message.ID, messageID) + } + if !result.replay { + created++ + } + } + if created != 1 { + t.Fatalf("new message results = %d, want 1", created) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + key := uuid.NewString() + + if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil { + t.Fatalf("owner CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil { + t.Fatalf("second user CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil { + t.Fatalf("other chat CreateMessage() error = %v", err) + } + if _, err := pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour) + if err != nil || replay || message.Text != "after expiry" { + t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err) + } +} + +func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { + t := s.T() + owner, chatID := createOwnerAndChat(t, s.store) + expiredKey := uuid.NewString() + if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil { + t.Fatalf("create expired message: %v", err) + } + if _, err := s.pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + if _, _, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour, + ); err != nil { + t.Fatalf("create new message: %v", err) + } + + var remaining int + if err := s.pool.QueryRow(context.Background(), ` + SELECT count(*) + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil { + t.Fatalf("count expired idempotency records: %v", err) + } + if remaining != 0 { + t.Fatalf("expired idempotency records = %d, want 0", remaining) + } +} + +func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) { + t.Helper() + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + return owner, chat.ID +} + +func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + return count +} + +func openTestDatabase(t *testing.T) *pgxpool.Pool { + t.Helper() + url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE") + pool, err := database.Open(context.Background(), database.Config{ + URL: url, MinConnections: 0, MaxConnections: 10, + MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute, + HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("database.Open() error = %v", err) + } + if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil { + pool.Close() + t.Fatalf("database.Migrate() error = %v", err) + } + return pool +} + +func testSecret(t *testing.T, valueName, fileName string) string { + t.Helper() + if value := strings.TrimSpace(os.Getenv(valueName)); value != "" { + return value + } + path := strings.TrimSpace(os.Getenv(fileName)) + if path == "" { + t.Skipf("set %s or %s to run integration tests", valueName, fileName) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", fileName, err) + } + return strings.TrimSpace(string(data)) +} + +func uniqueUsername() string { + return "u_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] +} + +func chatCount(t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM chats").Scan(&count); err != nil { + t.Fatalf("count chats: %v", err) + } + return count +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 00000000..267b39d3 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,350 @@ +package reconcile + +import ( + "context" + "errors" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type MetadataStore interface { + ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentExistsByObjectKey(context.Context, string) (bool, error) + ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) + DeleteAttachmentIfDeleting(context.Context, string) (bool, error) +} + +type ObjectStore interface { + List(context.Context, string, string, int) ([]model.StoredObject, string, error) + Delete(context.Context, string) error +} + +type Config struct { + Prefix string + PageSize int + GraceTime time.Duration + UnreferencedRetention time.Duration + Apply bool + Now func() time.Time +} + +type Report struct { + MetadataScanned int `json:"metadata_scanned"` + ObjectsScanned int `json:"objects_scanned"` + Consistent int `json:"consistent"` + FreshPendingAttachments int `json:"fresh_pending_attachments"` + OldPendingAttachments int `json:"old_pending_attachments"` + DeletingAttachments int `json:"deleting_attachments"` + FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"` + OldUnreferencedAttachments int `json:"old_unreferenced_attachments"` + DeletedAttachmentObjects int `json:"deleted_attachment_objects"` + FreshOrphanObjects int `json:"fresh_orphan_objects"` + OldOrphanObjects int `json:"old_orphan_objects"` + DeletedOrphanObjects int `json:"deleted_orphan_objects"` + FreshMissingObjects int `json:"fresh_missing_objects"` + ReferencedMissingObjects int `json:"referenced_missing_objects"` + OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` + DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` +} + +type Reconciler struct { + metadata MetadataStore + objects ObjectStore + config Config +} + +func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) { + if config.PageSize < 1 || config.PageSize > 1000 { + return nil, errors.New("page size must be between 1 and 1000") + } + if config.GraceTime < 0 { + return nil, errors.New("grace time must not be negative") + } + if config.UnreferencedRetention <= 0 { + return nil, errors.New("unreferenced retention must be positive") + } + if config.Now == nil { + config.Now = time.Now + } + return &Reconciler{metadata: metadata, objects: objects, config: config}, nil +} + +func (r *Reconciler) Run(ctx context.Context) (Report, error) { + metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize) + if err != nil { + return Report{}, err + } + objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize) + if err != nil { + return Report{}, err + } + + now := r.config.Now() + anomalyCutoff := now.Add(-r.config.GraceTime) + retentionCutoff := now.Add(-r.config.UnreferencedRetention) + var report Report + for metadata.current != nil || objects.current != nil { + switch { + case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey: + if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + case objects.current == nil || metadata.current.ObjectKey < objects.current.Key: + if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + default: + if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + } + } + return report, nil +} + +func (r *Reconciler) handlePair( + ctx context.Context, + metadata model.Attachment, + retentionCutoff time.Time, + anomalyCutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + report.ObjectsScanned++ + switch metadata.Status { + case model.AttachmentPending: + if metadata.CreatedAt.After(anomalyCutoff) { + report.FreshPendingAttachments++ + report.Consistent++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, anomalyCutoff, report) + case model.AttachmentDeleting: + report.DeletingAttachments++ + return r.deleteClaimedPair(ctx, metadata, report) + case model.AttachmentReady: + if metadata.Referenced { + report.Consistent++ + return nil + } + age := metadata.CreatedAt + if metadata.ReadyAt != nil { + age = *metadata.ReadyAt + } + if age.After(retentionCutoff) { + report.FreshUnreferencedAttachments++ + report.Consistent++ + return nil + } + report.OldUnreferencedAttachments++ + return r.deletePair(ctx, metadata, retentionCutoff, report) + default: + return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status) + } +} + +func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error { + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff) + if err != nil { + return fmt.Errorf("claim attachment %q: %w", attachment.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedPair(ctx, attachment, report) +} + +func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err) + } + report.DeletedAttachmentObjects++ + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +func (r *Reconciler) handleOrphanObject( + ctx context.Context, + object model.StoredObject, + cutoff time.Time, + report *Report, +) error { + report.ObjectsScanned++ + if object.LastModified.After(cutoff) { + report.FreshOrphanObjects++ + return nil + } + report.OldOrphanObjects++ + if !r.config.Apply { + return nil + } + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + return nil + } + if err := r.objects.Delete(ctx, object.Key); err != nil { + return fmt.Errorf("delete orphan object %q: %w", object.Key, err) + } + report.DeletedOrphanObjects++ + return nil +} + +func (r *Reconciler) handleMissingObject( + ctx context.Context, + metadata model.Attachment, + cutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + if metadata.Status == model.AttachmentDeleting { + return r.deleteClaimedMetadata(ctx, metadata, report) + } + if metadata.CreatedAt.After(cutoff) { + report.FreshMissingObjects++ + return nil + } + if metadata.Referenced { + report.ReferencedMissingObjects++ + return nil + } + report.OldUnreferencedMetadata++ + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff) + if err != nil { + return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedMetadata(ctx, metadata, report) +} + +func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error { + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) + if err != nil { + return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +type metadataIterator struct { + store MetadataStore + pageSize int + after string + page []model.Attachment + index int + current *model.Attachment + done bool +} + +func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) { + iterator := &metadataIterator{store: store, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *metadataIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list attachment metadata: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("metadata page is empty but has continuation") + } + } +} + +type objectIterator struct { + store ObjectStore + prefix string + pageSize int + after string + page []model.StoredObject + index int + current *model.StoredObject + done bool +} + +func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) { + iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *objectIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("object page is empty but has continuation") + } + } +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 00000000..26c2d922 --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,180 @@ +package reconcile + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func TestReconcilerLifecycleAndRepeat(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + fresh := now.Add(-10 * time.Minute) + metadata := &metadataStub{items: map[string]model.Attachment{ + "attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false), + "attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true), + "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), + "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), + "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, + "attachments/b/original": {Key: "attachments/b/original", LastModified: old}, + "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, + "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, + "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, report.OldPendingAttachments) + require.Equal(t, 1, report.OldUnreferencedAttachments) + require.Equal(t, 1, report.FreshUnreferencedAttachments) + require.Equal(t, 1, report.DeletedOrphanObjects) + require.Equal(t, 2, report.DeletedAttachmentObjects) + require.Equal(t, 3, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, "attachments/b/original") + require.Contains(t, metadata.items, "attachments/d/original") + require.NotContains(t, metadata.items, "attachments/a/original") + require.NotContains(t, metadata.items, "attachments/c/original") + require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, objects.items, "attachments/orphan/original") + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedOrphanObjects) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { + now := time.Now().UTC() + metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedOrphanObjects) + require.Contains(t, objects.items, "attachments/a/original") +} + +func TestNewRejectsInvalidRetention(t *testing.T) { + _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) + require.Error(t, err) +} + +func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment { + readyAt := createdAt + return model.Attachment{ + ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt, + ReadyAt: &readyAt, Referenced: referenced, + } +} + +type metadataStub struct { + items map[string]model.Attachment + appeared map[string]bool +} + +func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.Attachment, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) { + if s.appeared[key] { + return true, nil + } + _, exists := s.items[key] + return exists, nil +} + +func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { + for key, item := range s.items { + if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { + continue + } + age := item.CreatedAt + if item.Status == model.AttachmentReady && item.ReadyAt != nil { + age = *item.ReadyAt + } + if age.After(cutoff) { + return false, nil + } + item.Status = model.AttachmentDeleting + s.items[key] = item + return true, nil + } + return false, nil +} + +func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + for key, item := range s.items { + if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { + delete(s.items, key) + return true, nil + } + } + return false, nil +} + +type objectStub struct { + items map[string]model.StoredObject +} + +func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.StoredObject, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *objectStub) Delete(_ context.Context, key string) error { + delete(s.items, key) + return nil +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000..b10261be --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,163 @@ +package session + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/valkey-io/valkey-go" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +const ( + tokenLength = 32 + keyPrefix = "session:" +) + +var ( + ErrNotFound = model.ErrSessionNotFound + ErrInvalidToken = model.ErrInvalidToken + ErrInvalidRecord = errors.New("invalid session record") + ErrInvalidTTL = errors.New("invalid session TTL") + ErrTokenGeneration = errors.New("session token generation failed") + ErrStore = model.ErrUnavailable +) + +type record struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Store struct { + client valkey.Client +} + +func New(rawURL string) (*Store, error) { + option, err := valkey.ParseURL(rawURL) + if err != nil { + return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err) + } + + client, err := valkey.NewClient(option) + if err != nil { + return nil, fmt.Errorf("%w: create Valkey client: %w", ErrStore, err) + } + + return &Store{client: client}, nil +} + +func (s *Store) Create(ctx context.Context, userID string, ttl time.Duration) (string, time.Time, error) { + if strings.TrimSpace(userID) == "" { + return "", time.Time{}, ErrInvalidRecord + } + if ttl < time.Second { + return "", time.Time{}, ErrInvalidTTL + } + + token, err := newToken() + if err != nil { + return "", time.Time{}, err + } + + record := record{ + UserID: userID, + ExpiresAt: time.Now().UTC().Add(ttl).Truncate(time.Microsecond), + } + payload, err := json.Marshal(record) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: encode session: %w", ErrStore, err) + } + + command := s.client.B().Set().Key(s.key(token)).Value(valkey.BinaryString(payload)).Ex(ttl).Build() + if err := s.client.Do(ctx, command).Error(); err != nil { + return "", time.Time{}, fmt.Errorf("%w: create session: %w", ErrStore, err) + } + + return token, record.ExpiresAt, nil +} + +func newToken() (string, error) { + random := make([]byte, tokenLength) + if _, err := rand.Read(random); err != nil { + return "", fmt.Errorf("%w: %v", ErrTokenGeneration, err) + } + return base64.RawURLEncoding.EncodeToString(random), nil +} + +func (s *Store) UserID(ctx context.Context, token string) (string, error) { + key, err := s.sessionKey(token) + if err != nil { + return "", err + } + + payload, err := s.client.Do(ctx, s.client.B().Get().Key(key).Build()).AsBytes() + if valkey.IsValkeyNil(err) { + return "", ErrNotFound + } + if err != nil { + return "", fmt.Errorf("%w: get session: %w", ErrStore, err) + } + + var record record + if err := json.Unmarshal(payload, &record); err != nil { + return "", fmt.Errorf("%w: decode session: %w", ErrStore, err) + } + if strings.TrimSpace(record.UserID) == "" || record.ExpiresAt.IsZero() { + return "", fmt.Errorf("%w: %w", ErrStore, ErrInvalidRecord) + } + if !record.ExpiresAt.After(time.Now()) { + return "", ErrNotFound + } + + return record.UserID, nil +} + +func (s *Store) Delete(ctx context.Context, token string) error { + key, err := s.sessionKey(token) + if err != nil { + return err + } + + deleted, err := s.client.Do(ctx, s.client.B().Del().Key(key).Build()).AsInt64() + if err != nil { + return fmt.Errorf("%w: delete session: %w", ErrStore, err) + } + if deleted == 0 { + return ErrNotFound + } + + return nil +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.client.Do(ctx, s.client.B().Ping().Build()).Error(); err != nil { + return fmt.Errorf("%w: ping Valkey: %w", ErrStore, err) + } + return nil +} + +func (s *Store) Close() error { + s.client.Close() + return nil +} + +func (s *Store) sessionKey(token string) (string, error) { + decoded, err := base64.RawURLEncoding.Strict().DecodeString(token) + if err != nil || len(decoded) != tokenLength { + return "", ErrInvalidToken + } + return s.key(token), nil +} + +func (s *Store) key(token string) string { + digest := sha256.Sum256([]byte(token)) + return keyPrefix + hex.EncodeToString(digest[:]) +} diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go new file mode 100644 index 00000000..c7d874e9 --- /dev/null +++ b/internal/session/session_integration_test.go @@ -0,0 +1,72 @@ +package session_test + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" +) + +type SessionSuite struct { + suite.Suite + store *session.Store +} + +func TestSessionSuite(t *testing.T) { + suite.Run(t, new(SessionSuite)) +} + +func (s *SessionSuite) SetupSuite() { + rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) + if rawURL == "" { + path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE")) + if path == "" { + s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") + } + data, err := os.ReadFile(path) + s.Require().NoError(err) + rawURL = strings.TrimSpace(string(data)) + } + store, err := session.New(rawURL) + s.Require().NoError(err) + s.store = store +} + +func (s *SessionSuite) TearDownSuite() { + if s.store != nil { + s.Require().NoError(s.store.Close()) + } +} + +func (s *SessionSuite) TestSessionLifecycle() { + userID := uuid.NewString() + token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute) + s.Require().NoError(err) + s.NotEmpty(token) + s.True(expiresAt.After(time.Now())) + + actualUserID, err := s.store.UserID(context.Background(), token) + s.Require().NoError(err) + s.Equal(userID, actualUserID) + s.Require().NoError(s.store.Delete(context.Background(), token)) + + _, err = s.store.UserID(context.Background(), token) + s.ErrorIs(err, session.ErrNotFound) +} + +func (s *SessionSuite) TestSessionExpires() { + token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second) + s.Require().NoError(err) + + s.Eventually(func() bool { + _, err = s.store.UserID(context.Background(), token) + return errors.Is(err, session.ErrNotFound) + }, 3*time.Second, 50*time.Millisecond) +} diff --git a/lab5.env.example b/lab5.env.example new file mode 100644 index 00000000..5ad34dcb --- /dev/null +++ b/lab5.env.example @@ -0,0 +1,100 @@ +# Адрес и внутренний порт HTTP-сервера. +APP_HOST=0.0.0.0 +APP_PORT=8080 + +# Адреса публикации сервисов на локальной машине. +API_BIND_ADDRESS=127.0.0.1:8080 +SWAGGER_BIND_ADDRESS=127.0.0.1:8081 +PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 +GRAFANA_BIND_ADDRESS=127.0.0.1:3000 + +# Время жизни сессии, идемпотентности и используемое хранилище. +SESSION_TTL_SECONDS=3600 +IDEMPOTENCY_TTL_SECONDS=86400 +SESSION_STORE=valkey + +# Число API-реплик за Nginx. +API_REPLICAS=2 + +# Учётные данные локального PostgreSQL и строка подключения API. +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD=local-postgres-password +DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable + +# Пароль локального Valkey и строка подключения API. +VALKEY_PASSWORD=local-valkey-password +CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 + +# Административные данные MinIO используются только при bootstrap. +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=local-minio-admin-password + +# API использует отдельного пользователя с доступом только к bucket вложений. +S3_ENDPOINT=s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_ACCESS_KEY=messenger-app +S3_SECRET_KEY=local-messenger-s3-password +S3_USE_TLS=false +S3_TIMEOUT_SECONDS=5 +ATTACHMENT_MAX_BYTES=10485760 +ATTACHMENT_MAX_PIXELS=40000000 +ATTACHMENT_BODY_MEMORY_BYTES=67108864 +ATTACHMENT_DECODE_MEMORY_BYTES=402653184 + +# Размер пула и обслуживание соединений PostgreSQL. +DB_MIN_CONNS=1 +DB_MAX_CONNS=10 +DB_MAX_CONNECTION_AGE_SECONDS=1800 +DB_MAX_CONNECTION_IDLE_SECONDS=300 +DB_HEALTH_CHECK_SECONDS=30 + +# Таймауты зависимостей, миграций и HTTP-запросов. +DEPENDENCY_TIMEOUT_SECONDS=5 +MIGRATION_TIMEOUT_SECONDS=30 +REQUEST_TIMEOUT_SECONDS=5 +HTTP_READ_HEADER_TIMEOUT_SECONDS=5 +HTTP_READ_TIMEOUT_SECONDS=10 +HTTP_WRITE_TIMEOUT_SECONDS=10 +HTTP_IDLE_TIMEOUT_SECONDS=60 +HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 + +# Параметры Argon2id для новых хешей паролей. +PASSWORD_MEMORY_KIB=65536 +PASSWORD_ITERATIONS=3 +PASSWORD_PARALLELISM=2 +PASSWORD_SALT_LENGTH=16 +PASSWORD_KEY_LENGTH=32 + +# Порт Swagger внутри конфигурации CORS API. +SWAGGER_PORT=8081 + +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=512m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=0.5 +GRAFANA_MEMORY=256m +POSTGRES_CPUS=2 +POSTGRES_MEMORY=1g +VALKEY_CPUS=1 +VALKEY_MEMORY=256m +S3_CPUS=1 +S3_MEMORY=512m +S3_BOOTSTRAP_CPUS=0.25 +S3_BOOTSTRAP_MEMORY=256m + +# Пароль локального администратора Grafana. +GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/migrations/0001_initial.sql b/migrations/0001_initial.sql new file mode 100644 index 00000000..2f9b90b4 --- /dev/null +++ b/migrations/0001_initial.sql @@ -0,0 +1,50 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_display_name_not_blank CHECK (display_name ~ '[^[:space:]]'), + CONSTRAINT users_password_hash_length CHECK (char_length(password_hash) BETWEEN 1 AND 1024), + CONSTRAINT users_password_hash_not_blank CHECK (password_hash ~ '[^[:space:]]') +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100), + CONSTRAINT chats_title_not_blank CHECK (title ~ '[^[:space:]]') +); + +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_position_unique UNIQUE (chat_id, position), + CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE +); + +CREATE INDEX chat_members_user_id_chat_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_text_not_blank CHECK (text ~ '[^[:space:]]'), + CONSTRAINT messages_sender_membership_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); + +CREATE INDEX messages_chat_id_created_at_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0002_message_idempotency.sql b/migrations/0002_message_idempotency.sql new file mode 100644 index 00000000..91443a48 --- /dev/null +++ b/migrations/0002_message_idempotency.sql @@ -0,0 +1,19 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL, + chat_id UUID NOT NULL, + key TEXT NOT NULL, + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32), + CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at) +); + +CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..91d1cc92 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,47 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_length, + DROP CONSTRAINT messages_text_not_blank; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id); diff --git a/migrations/0004_attachment_lifecycle.sql b/migrations/0004_attachment_lifecycle.sql new file mode 100644 index 00000000..1596c99d --- /dev/null +++ b/migrations/0004_attachment_lifecycle.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachments + ADD COLUMN status TEXT NOT NULL DEFAULT 'ready', + ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp(); + +UPDATE attachments +SET ready_at = created_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status IN ('ready', 'deleting') AND ready_at IS NOT NULL) + ); + +CREATE INDEX attachments_status_created_at_object_key_idx + ON attachments (status, created_at, object_key); diff --git a/migrations/0005_allow_pending_deletion.sql b/migrations/0005_allow_pending_deletion.sql new file mode 100644 index 00000000..0b20acb2 --- /dev/null +++ b/migrations/0005_allow_pending_deletion.sql @@ -0,0 +1,9 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status = 'ready' AND ready_at IS NOT NULL) OR + status = 'deleting' + ); diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 00000000..08bbf60c --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import "embed" + +// FS contains database migrations applied by the application at startup. +// +//go:embed *.sql +var FS embed.FS diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json new file mode 100644 index 00000000..0478e84b --- /dev/null +++ b/monitoring/grafana/dashboards/messenger.json @@ -0,0 +1,143 @@ +{ + "annotations": {"list": []}, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "type": "timeseries", "title": "API requests per second", "id": 1, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API RPS by replica", "id": 2, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API responses by status class", "id": 3, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Healthy API replicas", "id": 4, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API latency p95 by replica and route", "id": 5, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API in flight", "id": 6, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, + "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Go heap by replica", "id": 7, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Idempotency replays", "id": 8, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, + "targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Nginx connections and requests", "id": 9, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, + "targets": [ + {"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"}, + {"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "PostgreSQL connections", "id": 10, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, + "targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}] + }, + { + "type": "timeseries", "title": "PostgreSQL transactions", "id": 11, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, + "targets": [ + {"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"}, + {"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "PostgreSQL database size", "id": 12, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Valkey memory", "id": 13, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Valkey keys and commands", "id": 14, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, + "targets": [ + {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, + {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "Attachment uploads by result", "id": 15, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 56}, + "targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Attachment upload latency p95", "id": 16, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Attachment upload traffic", "id": 17, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 64}, + "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}, + "targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO objects", "id": 18, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 64}, + "targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO requests and errors", "id": 19, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 64}, + "targets": [ + {"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"}, + {"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "MinIO availability", "id": 20, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 72}, + "targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO storage", "id": 21, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 72}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [ + {"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"}, + {"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"} + ] + } + ], + "refresh": "5s", + "schemaVersion": 41, + "tags": ["highload", "lab4", "nginx", "failover"], + "templating": {"list": []}, + "time": {"from": "now-15m", "to": "now"}, + "timezone": "browser", + "title": "Messenger lab4 overview", + "uid": "messenger-overview", + "version": 3 +} diff --git a/monitoring/grafana/provisioning/dashboards/messenger.yml b/monitoring/grafana/provisioning/dashboards/messenger.yml new file mode 100644 index 00000000..fa7934cb --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/messenger.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Messenger + folder: Highload + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..8704d9e5 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 00000000..83721c48 --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,28 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: messenger + dns_sd_configs: + - names: ["api"] + type: A + port: 8080 + refresh_interval: 2s + + - job_name: nginx + static_configs: + - targets: ["nginx-exporter:9113"] + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: valkey + static_configs: + - targets: ["valkey-exporter:9121"] + + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: ["s3:9000"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..d62dcd65 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,57 @@ +user nginx; +worker_processes auto; +error_log /dev/stderr notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + + log_format structured escape=json + '{"time":"$time_iso8601","request_id":"$http_x_request_id",' + '"method":"$request_method","uri":"$request_uri","status":$status,' + '"request_time":$request_time,"upstream":"$upstream_addr",' + '"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}'; + + access_log /dev/stdout structured; + + upstream messenger { + zone messenger 64k; + include /etc/nginx/conf.d/upstream.conf; + keepalive 32; + } + + server { + listen 8080; + client_max_body_size 11m; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering off; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } +} diff --git a/nginx/upstream.conf b/nginx/upstream.conf new file mode 100644 index 00000000..f6131cb4 --- /dev/null +++ b/nginx/upstream.conf @@ -0,0 +1 @@ +server api:8080 resolve max_fails=1 fail_timeout=3s; diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a724583d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + +docker compose stop nginx api valkey postgres s3 +docker compose start postgres valkey s3 +docker compose run --rm s3-bootstrap +docker compose run --rm migrate +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..d02dc02d --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,223 @@ +#!/bin/sh + +set -eu + +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +manifest=$1 +deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"} +current_manifest=$deploy_dir/current-images.env +active_containers=$deploy_dir/active-api-containers +previous_containers=$deploy_dir/previous-api-containers +upstream_file=$deploy_dir/upstream.conf +upstream_backup=$deploy_dir/upstream.previous.conf +candidate_file=$deploy_dir/candidate-api-containers +replicas=${API_REPLICAS:-2} +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5} +switched=false +nginx_created=false +upstream_changed=false +old_pool="" + +mkdir -p "$deploy_dir" +: > "$candidate_file" +export NGINX_UPSTREAM_FILE=$upstream_file + +load_manifest() { + set -a + # shellcheck disable=SC1090 + . "$1" + set +a +} + +compose() { + docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@" +} + +container_name() { + name=$(docker inspect --format '{{.Name}}' "$1") + printf '%s\n' "${name#/}" +} + +running_container() { + [ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ] +} + +write_upstream() { + output=$1 + containers=$2 + : > "$output" + for container in $containers; do + name=$(container_name "$container") + printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + done +} + +wait_for_containers() { + containers=$1 + deadline=$(($(date +%s) + 120)) + + while [ "$(date +%s)" -lt "$deadline" ]; do + count=0 + healthy=0 + for container in $containers; do + count=$((count + 1)) + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container") + if [ "$status" = healthy ]; then + healthy=$((healthy + 1)) + fi + done + if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then + return 0 + fi + sleep 1 + done + + printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2 + return 1 +} + +smoke_check() { + compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready +} + +remove_containers() { + containers=$1 + for container in $containers; do + docker rm -f "$container" >/dev/null 2>&1 || true + done +} + +restore_upstream() { + if [ -s "$upstream_backup" ]; then + cp "$upstream_backup" "$upstream_file" + compose exec -T nginx nginx -t + compose exec -T nginx nginx -s reload + fi +} + +rollback() { + status=$? + trap - EXIT INT TERM + candidates=$(cat "$candidate_file" 2>/dev/null || true) + + if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then + printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2 + restore_upstream || true + if [ "$switched" = true ]; then + smoke_check || true + fi + elif [ "$nginx_created" = true ]; then + compose stop nginx >/dev/null 2>&1 || true + fi + remove_containers "$candidates" + exit "$status" +} + +trap rollback EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +load_manifest "$manifest" +new_api_image=$API_IMAGE + +if [ -f "$previous_containers" ]; then + remove_containers "$(cat "$previous_containers")" + rm -f "$previous_containers" +fi + +if [ -s "$active_containers" ]; then + for container in $(cat "$active_containers"); do + if running_container "$container"; then + old_pool="$old_pool $container" + fi + done +fi +if [ -z "$old_pool" ]; then + old_pool=$(compose ps -q api) +fi + +if [ -s "$upstream_file" ]; then + : +elif [ -n "$old_pool" ]; then + write_upstream "$upstream_file" "$old_pool" +else + cp nginx/upstream.conf "$upstream_file" +fi +cp "$upstream_file" "$upstream_backup" + +compose pull api nginx swagger prometheus grafana s3 s3-bootstrap +compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3 +compose run --rm --no-deps s3-bootstrap +compose run --rm --no-deps migrate + +nginx_container=$(compose ps -q nginx) +if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then + compose up -d --no-build --no-deps --force-recreate nginx + nginx_container=$(compose ps -q nginx) +fi +if [ -n "$nginx_container" ]; then + running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container") + if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then + printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2 + exit 1 + fi +fi + +API_IMAGE=$new_api_image +export API_IMAGE +short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12) +if [ -z "$short_digest" ]; then + short_digest=$(date +%s) +fi +generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"} + +candidate_pool="" +index=1 +while [ "$index" -le "$replicas" ]; do + name="${project}-api-${short_digest}-${generation}-${index}" + container=$(compose run -d --no-deps --name "$name" api) + candidate_pool="$candidate_pool $container" + printf '%s\n' "$container" >> "$candidate_file" + index=$((index + 1)) +done +wait_for_containers "$candidate_pool" + +write_upstream "$upstream_file.next" "$candidate_pool" +cp "$upstream_file.next" "$upstream_file" +upstream_changed=true + +if [ -n "$(compose ps -q nginx)" ]; then + compose exec -T nginx nginx -t + switched=true + compose exec -T nginx nginx -s reload +else + nginx_created=true + compose up -d --no-build --no-deps nginx + switched=true +fi +smoke_check + +application_network="${project}_application" +for container in $candidate_pool; do + docker network disconnect "$application_network" "$container" + docker network connect --alias api "$application_network" "$container" +done +smoke_check + +compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana + +printf '%s\n' "$candidate_pool" > "$active_containers" +cp "$manifest" "$current_manifest" +rm -f "$candidate_file" "$upstream_file.next" +trap - EXIT INT TERM + +if [ -n "$old_pool" ]; then + for container in $old_pool; do + docker stop --time 6 "$container" >/dev/null || true + done + printf '%s\n' "$old_pool" > "$previous_containers" +fi diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..5327251e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2 +exit 1 -- 2.54.0 From 42336ce7c24ba4333f877f74157af60cd32b0b77 Mon Sep 17 00:00:00 2001 From: dizi Date: Thu, 22 Oct 2026 17:22:28 +0300 Subject: [PATCH 02/11] feat: add durable image processing pipeline --- cmd/messenger/main.go | 11 +- go.mod | 3 + go.sum | 10 + internal/bootstrap/pipeline.go | 200 +++++++ internal/bootstrap/reconciliation.go | 1 + internal/bootstrap/replay.go | 39 ++ internal/bootstrap/server.go | 2 +- internal/broker/redisstream/broker.go | 280 +++++++++ internal/broker/redisstream/broker_test.go | 74 +++ internal/broker/redisstream/envelope.go | 147 +++++ internal/broker/redisstream/envelope_test.go | 74 +++ internal/config/config.go | 6 + internal/config/pipeline.go | 273 +++++++++ .../messenger/attachment/pipeline/metrics.go | 102 ++++ internal/messenger/attachment/service.go | 74 +-- internal/messenger/attachment/service_test.go | 50 +- .../messenger/attachment/transform/exif.go | 146 +++++ .../attachment/transform/processor.go | 237 ++++++++ .../attachment/transform/processor_test.go | 236 ++++++++ .../messenger/attachment/transform/resize.go | 105 ++++ .../messenger/attachment/worker/worker.go | 452 ++++++++++++++ .../attachment/worker/worker_test.go | 56 ++ internal/messenger/httpapi/attachments.go | 45 +- .../messenger/httpapi/messenger_interface.go | 2 +- internal/messenger/httpapi/server_test.go | 8 +- internal/model/errors.go | 6 + internal/model/model.go | 118 +++- internal/outbox/dispatcher.go | 174 ++++++ internal/outbox/dispatcher_test.go | 181 ++++++ internal/postgres/attachment_jobs.go | 560 ++++++++++++++++++ internal/postgres/attachment_recovery.go | 210 +++++++ internal/postgres/attachments.go | 97 ++- internal/postgres/errors.go | 6 + internal/postgres/idempotency.go | 28 +- internal/postgres/outbox.go | 113 ++++ internal/postgres/store_integration_test.go | 189 ++++++ internal/reconcile/reconcile.go | 40 ++ internal/reconcile/reconcile_test.go | 15 +- migrations/0006_async_attachment_pipeline.sql | 135 +++++ 39 files changed, 4379 insertions(+), 126 deletions(-) create mode 100644 internal/bootstrap/pipeline.go create mode 100644 internal/bootstrap/replay.go create mode 100644 internal/broker/redisstream/broker.go create mode 100644 internal/broker/redisstream/broker_test.go create mode 100644 internal/broker/redisstream/envelope.go create mode 100644 internal/broker/redisstream/envelope_test.go create mode 100644 internal/config/pipeline.go create mode 100644 internal/messenger/attachment/pipeline/metrics.go create mode 100644 internal/messenger/attachment/transform/exif.go create mode 100644 internal/messenger/attachment/transform/processor.go create mode 100644 internal/messenger/attachment/transform/processor_test.go create mode 100644 internal/messenger/attachment/transform/resize.go create mode 100644 internal/messenger/attachment/worker/worker.go create mode 100644 internal/messenger/attachment/worker/worker_test.go create mode 100644 internal/outbox/dispatcher.go create mode 100644 internal/outbox/dispatcher_test.go create mode 100644 internal/postgres/attachment_jobs.go create mode 100644 internal/postgres/attachment_recovery.go create mode 100644 internal/postgres/outbox.go create mode 100644 migrations/0006_async_attachment_pipeline.sql diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index c6ab63ec..80230b80 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -24,8 +24,17 @@ func run(logger *slog.Logger, arguments []string) error { if len(arguments) == 1 && arguments[0] == "migrate" { return bootstrap.RunMigrations() } + if len(arguments) == 1 && arguments[0] == "dispatch" { + return bootstrap.RunDispatcher(logger) + } + if len(arguments) == 1 && arguments[0] == "worker" { + return bootstrap.RunWorker(logger) + } if arguments[0] == "reconcile-attachments" { return bootstrap.RunReconciliation(arguments[1:], os.Stdout) } - return errors.New("usage: messenger [migrate|reconcile-attachments]") + if arguments[0] == "replay-attachment" { + return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout) + } + return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]") } diff --git a/go.mod b/go.mod index 10f19e48..563b37a8 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module gitea.privcloud.tech/dizi/mtusi2026lab5 go 1.26.0 require ( + github.com/deepteams/webp v1.2.8 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 github.com/minio/minio-go/v7 v7.0.99 github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 @@ -18,6 +20,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect diff --git a/go.sum b/go.sum index 5aa456c6..edfd0ad8 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA= +github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -64,6 +72,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go new file mode 100644 index 00000000..2f794c50 --- /dev/null +++ b/internal/bootstrap/pipeline.go @@ -0,0 +1,200 @@ +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +func RunDispatcher(logger *slog.Logger) error { + settings, err := config.LoadDispatcher() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher", + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + metrics := pipeline.NewMetrics() + dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ + BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, + PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, + RetryDelay: settings.RetryDelay, + }) + if err != nil { + return err + } + logger.Info("outbox dispatcher started", "stream", settings.Stream) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run) +} + +func RunWorker(logger *slog.Logger) error { + settings, err := config.LoadWorker() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + hostname, err := os.Hostname() + if err != nil { + return err + } + consumer := hostname + "-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + return err + } + metrics := pipeline.NewMetrics() + processorWorker, err := worker.New( + logger, postgres.New(pool), broker, objects, processor, metrics, + worker.Config{ + WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, + ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + return err + } + logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run) +} + +func newRedisClient(rawURL string) (*redis.Client, error) { + options, err := redis.ParseURL(rawURL) + if err != nil { + return nil, err + } + client := redis.NewClient(options) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + return client, nil +} + +func runPipelineProcess( + logger *slog.Logger, + metricsAddress string, + metricsHandler http.Handler, + run func(context.Context) error, +) error { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler) + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + listener, err := net.Listen("tcp", metricsAddress) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + runResult := make(chan error, 1) + go func() { runResult <- run(ctx) }() + + var result error + select { + case result = <-runResult: + stop() + case result = <-serveResult: + stop() + case <-ctx.Done(): + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil && result == nil { + result = err + } + if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) { + return nil + } + logger.Info("pipeline process stopped") + return result +} + +func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func workerDatabaseConfig(settings config.WorkerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go index 359ddbd0..447dd23d 100644 --- a/internal/bootstrap/reconciliation.go +++ b/internal/bootstrap/reconciliation.go @@ -64,6 +64,7 @@ func RunReconciliation(arguments []string, output io.Writer) error { } reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + OrphanPrefixes: []string{"staging/"}, UnreferencedRetention: *retention, Apply: *apply, }) if err != nil { diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go new file mode 100644 index 00000000..56b9f2ed --- /dev/null +++ b/internal/bootstrap/replay.go @@ -0,0 +1,39 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "io" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +func RunAttachmentReplay(arguments []string, output io.Writer) error { + if len(arguments) != 3 { + return errors.New("usage: messenger replay-attachment ") + } + settings, err := config.LoadJobAdmin() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + job, err := postgres.New(pool).ReplayFailedAttachment( + context.Background(), arguments[0], arguments[1], arguments[2], + ) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt) + return err +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index 7073922c..121e328c 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -54,7 +54,7 @@ func RunServer(logger *slog.Logger) error { attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, - CleanupTimeout: settings.RequestTimeout, + CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion, }) if err != nil { return err diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go new file mode 100644 index 00000000..74a53745 --- /dev/null +++ b/internal/broker/redisstream/broker.go @@ -0,0 +1,280 @@ +package redisstream + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" +) + +const payloadField = "payload" + +var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + +var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer") + +var _ outbox.Publisher = (*Broker)(nil) + +type Config struct { + Stream string + Group string + Consumer string + FailedStream string + FailedMaxLen int64 + Block time.Duration + ClaimMinIdle time.Duration + ReadCount int64 +} + +type Broker struct { + client redis.UniversalClient + config Config +} + +type Delivery struct { + ID string + Envelope Envelope + DecodeError error + + broker *Broker + payload []byte +} + +type FailedDelivery struct { + Code string + FailedAt time.Time +} + +func New(client redis.UniversalClient, config Config) (*Broker, error) { + if client == nil { + return nil, errors.New("redis client is required") + } + if err := config.validate(); err != nil { + return nil, err + } + return &Broker{client: client, config: config}, nil +} + +func (c Config) validate() error { + switch { + case c.Stream == "": + return errors.New("stream is required") + case c.Group == "": + return errors.New("consumer group is required") + case c.Consumer == "": + return errors.New("consumer name is required") + case c.FailedStream == "": + return errors.New("failed stream is required") + case c.FailedStream == c.Stream: + return errors.New("failed stream must differ from job stream") + case c.FailedMaxLen <= 0: + return errors.New("failed stream maximum length must be positive") + case c.Block < 0: + return errors.New("block duration must not be negative") + case c.ClaimMinIdle <= 0: + return errors.New("claim minimum idle time must be positive") + case c.ReadCount <= 0: + return errors.New("read count must be positive") + default: + return nil + } +} + +func (b *Broker) Ensure(ctx context.Context) error { + err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err() + if err != nil && !isBusyGroup(err) { + return fmt.Errorf("create redis stream consumer group: %w", err) + } + return nil +} + +func isBusyGroup(err error) bool { + return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP") +} + +func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + envelope, err := DecodeEnvelope(payload) + if err != nil { + return "", err + } + payload, err = EncodeEnvelope(envelope) + if err != nil { + return "", err + } + + id, err := b.client.XAdd(ctx, &redis.XAddArgs{ + Stream: b.config.Stream, + Values: map[string]any{payloadField: string(payload)}, + }).Result() + if err != nil { + return "", fmt.Errorf("publish redis stream job: %w", err) + } + return id, nil +} + +func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: b.config.Group, + Consumer: b.config.Consumer, + Streams: []string{b.config.Stream, ">"}, + Count: b.config.ReadCount, + Block: b.config.Block, + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read redis stream jobs: %w", err) + } + return b.deliveries(streams) +} + +func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) { + messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: b.config.Stream, + Group: b.config.Group, + Consumer: b.config.Consumer, + MinIdle: b.config.ClaimMinIdle, + Start: start, + Count: b.config.ReadCount, + }).Result() + if err != nil { + return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err) + } + + deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}}) + if err != nil { + return nil, start, err + } + return deliveries, next, nil +} + +func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { + var deliveries []Delivery + for _, stream := range streams { + for _, message := range stream.Messages { + delivery := Delivery{ID: message.ID, broker: b} + payload, err := messagePayload(message) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + envelope, err := DecodeEnvelope(payload) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.Envelope = envelope + delivery.payload, err = EncodeEnvelope(envelope) + if err != nil { + return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err) + } + deliveries = append(deliveries, delivery) + } + } + return deliveries, nil +} + +func messagePayload(message redis.XMessage) ([]byte, error) { + value, ok := message.Values[payloadField] + if !ok { + return nil, errors.New("payload field is missing") + } + switch payload := value.(type) { + case string: + return []byte(payload), nil + case []byte: + return payload, nil + default: + return nil, fmt.Errorf("payload has type %T", value) + } +} + +func (d Delivery) Ack(ctx context.Context) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, + ).Int64() + if err != nil { + return fmt.Errorf("ack redis stream job: %w", err) + } + if acked == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const ackOwnedScript = ` +local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[1], ARGV[2]) +end +return acked +` + +func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + if !failureCodePattern.MatchString(failure.Code) { + return errors.New("failure code is invalid") + } + if failure.FailedAt.IsZero() { + return errors.New("failure time is required") + } + + payload := d.payload + if d.DecodeError == nil && len(payload) == 0 { + var err error + payload, err = EncodeEnvelope(d.Envelope) + if err != nil { + return err + } + } + moved, err := d.broker.client.Eval(ctx, moveToFailedScript, + []string{d.broker.config.FailedStream, d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code, + failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt), + strconv.FormatInt(d.broker.config.FailedMaxLen, 10), + ).Int64() + if err != nil { + return fmt.Errorf("move redis stream job to failed stream: %w", err) + } + if moved == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const moveToFailedScript = ` +local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*', + 'payload', ARGV[4], + 'source_message_id', ARGV[2], + 'error_code', ARGV[5], + 'failed_at', ARGV[6], + 'attempt', ARGV[7]) +local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[2], ARGV[2]) +end +return acked +` diff --git a/internal/broker/redisstream/broker_test.go b/internal/broker/redisstream/broker_test.go new file mode 100644 index 00000000..5e2f4c64 --- /dev/null +++ b/internal/broker/redisstream/broker_test.go @@ -0,0 +1,74 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestBrokerDecodesDeliveries(t *testing.T) { + payload, err := EncodeEnvelope(validEnvelope()) + require.NoError(t, err) + + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: string(payload)}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Equal(t, "1-0", deliveries[0].ID) + require.Equal(t, validEnvelope(), deliveries[0].Envelope) +} + +func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) { + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: `{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"private/object" + }`}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Empty(t, deliveries[0].payload) +} + +func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) { + broker := &Broker{} + _, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`)) + require.Error(t, err) +} + +func TestConfigValidation(t *testing.T) { + config := Config{ + Stream: "attachment-jobs", + Group: "attachment-workers", + Consumer: "worker-1", + FailedStream: "attachment-jobs-failed", + FailedMaxLen: 1000, + Block: time.Second, + ClaimMinIdle: time.Minute, + ReadCount: 10, + } + require.NoError(t, config.validate()) + + config.ClaimMinIdle = 0 + require.Error(t, config.validate()) +} diff --git a/internal/broker/redisstream/envelope.go b/internal/broker/redisstream/envelope.go new file mode 100644 index 00000000..44ffb4a0 --- /dev/null +++ b/internal/broker/redisstream/envelope.go @@ -0,0 +1,147 @@ +package redisstream + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +const ( + SchemaVersion = 1 + JobType = "attachment.process.v1" +) + +var ( + errInvalidEnvelope = errors.New("invalid job envelope") + traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +) + +type Envelope struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func (e Envelope) Validate() error { + switch { + case e.SchemaVersion != SchemaVersion: + return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope) + case e.Type != JobType: + return fmt.Errorf("%w: unsupported type", errInvalidEnvelope) + case e.JobID == uuid.Nil: + return fmt.Errorf("%w: job_id is required", errInvalidEnvelope) + case e.AttachmentID == uuid.Nil: + return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope) + case e.PipelineVersion < 1: + return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope) + case e.Attempt < 1: + return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope) + case e.CreatedAt.IsZero(): + return fmt.Errorf("%w: created_at is required", errInvalidEnvelope) + case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent): + return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope) + case utf8.RuneCountInString(e.Tracestate) > 512: + return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope) + default: + return nil + } +} + +func EncodeEnvelope(envelope Envelope) ([]byte, error) { + if err := envelope.Validate(); err != nil { + return nil, err + } + + payload, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("encode job envelope: %w", err) + } + + return payload, nil +} + +func DecodeEnvelope(payload []byte) (Envelope, error) { + var wire envelopeJSON + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err) + } + if err := ensureJSONEnd(decoder); err != nil { + return Envelope{}, err + } + + jobID, err := parseUUID(wire.JobID, "job_id") + if err != nil { + return Envelope{}, err + } + attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id") + if err != nil { + return Envelope{}, err + } + createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt) + if err != nil { + return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope) + } + envelope := Envelope{ + SchemaVersion: wire.SchemaVersion, + JobID: jobID, + Type: wire.Type, + AttachmentID: attachmentID, + PipelineVersion: wire.PipelineVersion, + Attempt: wire.Attempt, + CreatedAt: createdAt, + Traceparent: wire.Traceparent, + Tracestate: wire.Tracestate, + } + if err := envelope.Validate(); err != nil { + return Envelope{}, err + } + + return envelope, nil +} + +type envelopeJSON struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func parseUUID(value, field string) (uuid.UUID, error) { + if !uuidPattern.MatchString(value) { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + parsed, err := uuid.Parse(value) + if err != nil { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + return parsed, nil +} + +func ensureJSONEnd(decoder *json.Decoder) error { + var value any + if err := decoder.Decode(&value); !errors.Is(err, io.EOF) { + return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope) + } + return nil +} diff --git a/internal/broker/redisstream/envelope_test.go b/internal/broker/redisstream/envelope_test.go new file mode 100644 index 00000000..f24b68c0 --- /dev/null +++ b/internal/broker/redisstream/envelope_test.go @@ -0,0 +1,74 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestEnvelopeRoundTrip(t *testing.T) { + envelope := validEnvelope() + payload, err := EncodeEnvelope(envelope) + require.NoError(t, err) + + decoded, err := DecodeEnvelope(payload) + require.NoError(t, err) + require.Equal(t, envelope, decoded) +} + +func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) { + payload := []byte(`{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"must-not-enter-the-broker" + }`) + + _, err := DecodeEnvelope(payload) + require.Error(t, err) +} + +func TestEnvelopeValidation(t *testing.T) { + tests := []struct { + name string + mutate func(*Envelope) + }{ + {name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }}, + {name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }}, + {name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }}, + {name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }}, + {name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }}, + {name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }}, + {name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }}, + {name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }}, + {name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + envelope := validEnvelope() + tt.mutate(&envelope) + require.Error(t, envelope.Validate()) + }) + } +} + +func validEnvelope() Envelope { + return Envelope{ + SchemaVersion: SchemaVersion, + JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"), + Type: JobType, + AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"), + PipelineVersion: 1, + Attempt: 1, + CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC), + Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + Tracestate: "vendor=value", + } +} diff --git a/internal/config/config.go b/internal/config/config.go index dcf5b9a1..b27025c5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,6 +46,7 @@ type ServerConfig struct { AttachmentMaxPixels int64 AttachmentBodyMemory int64 AttachmentDecodeMemory int64 + PipelineVersion int } type MigrationConfig struct { @@ -207,6 +208,10 @@ func LoadServer() (ServerConfig, error) { if int64(attachmentDecodeMemory) < minimumDecodeMemory { return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) } + pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000) + if err != nil { + return ServerConfig{}, err + } return ServerConfig{ Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, @@ -226,6 +231,7 @@ func LoadServer() (ServerConfig, error) { AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), AttachmentBodyMemory: int64(attachmentBodyMemory), AttachmentDecodeMemory: int64(attachmentDecodeMemory), + PipelineVersion: pipelineVersion, }, nil } diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go new file mode 100644 index 00000000..572d59bf --- /dev/null +++ b/internal/config/pipeline.go @@ -0,0 +1,273 @@ +package config + +import ( + "errors" + "net" + "time" + + "github.com/spf13/viper" +) + +type DispatcherConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + MetricsAddress string +} + +type WorkerConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + Concurrency int + Prefetch int + LeaseDuration time.Duration + ProcessingTimeout time.Duration + ReadBlock time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + MemoryBytes int64 + MetricsAddress string +} + +type JobAdminConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration +} + +type brokerConfig struct { + url string + stream string + consumerGroup string + failedStream string + failedStreamMaxLength int +} + +func LoadJobAdmin() (JobAdminConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return JobAdminConfig{}, err + } + return JobAdminConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + }, nil +} + +func LoadDispatcher() (DispatcherConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600) + if err != nil { + return DispatcherConfig{}, err + } + pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60) + if err != nil { + return DispatcherConfig{}, err + } + recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600) + if err != nil { + return DispatcherConfig{}, err + } + retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600) + if err != nil { + return DispatcherConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return DispatcherConfig{}, err + } + return DispatcherConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, + FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress, + }, nil +} + +func LoadWorker() (WorkerConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return WorkerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return WorkerConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return WorkerConfig{}, err + } + concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64) + if err != nil { + return WorkerConfig{}, err + } + prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024) + if err != nil { + return WorkerConfig{}, err + } + leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600) + if err != nil { + return WorkerConfig{}, err + } + processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600) + if err != nil { + return WorkerConfig{}, err + } + if processingTimeout >= leaseDuration { + return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS") + } + readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60) + if err != nil { + return WorkerConfig{}, err + } + reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600) + if err != nil { + return WorkerConfig{}, err + } + maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100) + if err != nil { + return WorkerConfig{}, err + } + retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600) + if err != nil { + return WorkerConfig{}, err + } + retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400) + if err != nil { + return WorkerConfig{}, err + } + if retryMax < retryBase { + return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS") + } + attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + if err != nil { + return WorkerConfig{}, err + } + memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return WorkerConfig{}, err + } + return WorkerConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, + S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream, + ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, + FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, + ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + MemoryBytes: int64(memoryBytes), + MetricsAddress: metricsAddress, + }, nil +} + +func metricsAddress(values *viper.Viper) (string, error) { + host, err := requiredString(values, "PIPELINE_METRICS_HOST") + if err != nil { + return "", err + } + port, err := requiredPort(values, "PIPELINE_METRICS_PORT") + if err != nil { + return "", err + } + return net.JoinHostPort(host, port), nil +} + +func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { + url, err := requiredString(values, "BROKER_URL") + if err != nil { + return brokerConfig{}, err + } + stream, err := requiredString(values, "BROKER_STREAM") + if err != nil { + return brokerConfig{}, err + } + consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP") + if err != nil { + return brokerConfig{}, err + } + failedStream, err := requiredString(values, "BROKER_FAILED_STREAM") + if err != nil { + return brokerConfig{}, err + } + failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000) + if err != nil { + return brokerConfig{}, err + } + return brokerConfig{ + url: url, stream: stream, consumerGroup: consumerGroup, + failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength, + }, nil +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go new file mode 100644 index 00000000..ad76eca2 --- /dev/null +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -0,0 +1,102 @@ +package pipeline + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type Metrics struct { + workerInFlight prometheus.Gauge + workerJobs *prometheus.CounterVec + workerDuration *prometheus.HistogramVec + workerRetries prometheus.Counter + workerReclaimed prometheus.Counter + outboxPublished prometheus.Counter + outboxPublishFailure prometheus.Counter + outboxRecovered prometheus.Counter + handler http.Handler +} + +func NewMetrics() *Metrics { + registry := prometheus.NewRegistry() + metrics := &Metrics{ + workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight", + Help: "Attachment jobs currently processed by this worker.", + }), + workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total", + Help: "Completed attachment job deliveries by result.", + }, []string{"result"}), + workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds", + Help: "Attachment job delivery processing duration.", Buckets: prometheus.DefBuckets, + }, []string{"result"}), + workerRetries: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total", + Help: "Attachment jobs scheduled for another attempt.", + }), + workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total", + Help: "Redis Stream deliveries reclaimed after their idle timeout.", + }), + outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "published_total", + Help: "Outbox events confirmed by Redis Streams.", + }), + outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total", + Help: "Outbox events whose Redis Streams publish failed.", + }), + outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total", + Help: "Expired outbox leases returned for publication.", + }), + } + registry.MustRegister( + metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration, + metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished, + metrics.outboxPublishFailure, metrics.outboxRecovered, + collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + return metrics +} + +func (m *Metrics) Handler() http.Handler { + return m.handler +} + +func (m *Metrics) ObserveStarted() { + m.workerInFlight.Inc() +} + +func (m *Metrics) ObserveFinished(result string, duration time.Duration) { + m.workerInFlight.Dec() + m.workerJobs.WithLabelValues(result).Inc() + m.workerDuration.WithLabelValues(result).Observe(duration.Seconds()) +} + +func (m *Metrics) ObserveRetry() { + m.workerRetries.Inc() +} + +func (m *Metrics) ObserveReclaimed(count int) { + m.workerReclaimed.Add(float64(count)) +} + +func (m *Metrics) ObservePublished() { + m.outboxPublished.Inc() +} + +func (m *Metrics) ObservePublishFailure() { + m.outboxPublishFailure.Inc() +} + +func (m *Metrics) ObserveRecovered(count int) { + m.outboxRecovered.Add(float64(count)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index f5a1c963..7e0bbf8e 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -25,10 +25,10 @@ import ( type Repository interface { ChatForUser(context.Context, string, string) (model.Chat, error) CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) - MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) + QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error) DeletePendingAttachment(context.Context, string, string) error - AttachmentByStorage(context.Context, string, string) (model.Attachment, error) AttachmentForUser(context.Context, string, string) (model.Attachment, error) + AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) } type ObjectStore interface { @@ -43,6 +43,7 @@ type Config struct { BodyMemoryBytes int64 DecodeMemoryBytes int64 CleanupTimeout time.Duration + PipelineVersion int } type Upload struct { @@ -52,10 +53,13 @@ type Upload struct { ContentType string ContentLength int64 Body io.Reader + Traceparent string + Tracestate string } type Content struct { Attachment model.Attachment + Variant model.Variant Body io.ReadCloser } @@ -69,7 +73,7 @@ type Service struct { } func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { - if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 { return nil, errors.New("invalid attachment service configuration") } if config.BodyMemoryBytes < config.MaxBytes { @@ -96,58 +100,38 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e defer release() attachmentID := uuid.NewString() - objectKey := "attachments/" + uuid.NewString() + "/original" + stagingKey := "staging/" + attachmentID + "/" + uuid.NewString() digest := sha256.Sum256(data) pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), - ObjectKey: objectKey, Status: model.AttachmentPending, + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, + PipelineVersion: s.config.PipelineVersion, }) if err != nil { return model.Attachment{}, mapDependencyError(err) } - if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil { - s.deletePending(pending.ID, objectKey) + if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { + s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(pending.ID, stagingKey) return model.Attachment{}, model.ErrUnavailable } - attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey) - if err == nil { - return attachment, nil + createdAt := time.Now().UTC() + queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, + Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, + }, input.Traceparent, input.Tracestate) + if err != nil { + s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + return model.Attachment{}, mapDependencyError(err) } - return s.resolveReadyOutcome(pending.ID, objectKey) + return queued, nil } -func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) { +func (s *Service) deletePending(attachmentID, stagingKey string) { cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) defer cancel() - stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) - if err == nil && stored.Status == model.AttachmentReady { - return stored, nil - } - if err == nil && stored.Status == model.AttachmentPending { - if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { - s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr) - return model.Attachment{}, model.ErrUnavailable - } - if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil { - s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr) - } - return model.Attachment{}, model.ErrUnavailable - } - if errors.Is(err, model.ErrNotFound) { - if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { - s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr) - } - return model.Attachment{}, model.ErrUnavailable - } - s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err) - return model.Attachment{}, model.ErrUnavailable -} - -func (s *Service) deletePending(attachmentID, objectKey string) { - cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) - defer cancel() - if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil { + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) } } @@ -160,16 +144,16 @@ func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (mo return attachment, nil } -func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) { - attachment, err := s.Metadata(ctx, attachmentID, userID) +func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) { + attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName) if err != nil { - return Content{}, err + return Content{}, mapDependencyError(err) } - body, err := s.objects.Get(ctx, attachment.ObjectKey) + body, err := s.objects.Get(ctx, variant.ObjectKey) if err != nil { return Content{}, model.ErrUnavailable } - return Content{Attachment: attachment, Body: body}, nil + return Content{Attachment: attachment, Variant: variant, Body: body}, nil } func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index f06a895d..da14eb0c 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -27,8 +27,8 @@ func TestUploadLifecycle(t *testing.T) { result, err := service.Upload(context.Background(), upload(data, "image/png")) require.NoError(t, err) - require.Equal(t, model.AttachmentReady, result.Status) - require.Equal(t, []string{"pending", "put", "ready"}, events) + require.Equal(t, model.AttachmentQueued, result.Status) + require.Equal(t, []string{"pending", "put", "queued"}, events) } func TestUploadValidation(t *testing.T) { @@ -86,7 +86,7 @@ func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, Config{ MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, - DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1, }, ) require.Error(t, err) @@ -103,16 +103,16 @@ func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { require.True(t, repository.pendingDeleted) } -func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) { +func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) { data := pngFixture(t, 2, 2) - repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true} + repository := &repositoryStub{queueErr: errors.New("connection lost")} objects := &objectStub{} service := newTestService(t, repository, objects, 1024, 4) - result, err := service.Upload(context.Background(), upload(data, "image/png")) - require.NoError(t, err) - require.Equal(t, model.AttachmentReady, result.Status) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) require.False(t, objects.deleted) + require.False(t, repository.pendingDeleted) } func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { @@ -121,7 +121,7 @@ func newTestService(t *testing.T, repository Repository, objects ObjectStore, ma slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, Config{ MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, - DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1, }, ) require.NoError(t, err) @@ -137,8 +137,7 @@ func upload(data []byte, contentType string) Upload { type repositoryStub struct { events *[]string - markErr error - lookupReady bool + queueErr error pendingDeleted bool pending model.Attachment } @@ -154,12 +153,19 @@ func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment m return attachment, nil } -func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) { - if s.markErr != nil { - return model.Attachment{}, s.markErr +func (s *repositoryStub) QueueAttachmentWithJobAndOutbox( + context.Context, + string, + string, + model.AttachmentJob, + string, + string, +) (model.Attachment, error) { + if s.queueErr != nil { + return model.Attachment{}, s.queueErr } - s.pending.Status = model.AttachmentReady - s.record("ready") + s.pending.Status = model.AttachmentQueued + s.record("queued") return s.pending, nil } @@ -168,18 +174,14 @@ func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string return nil } -func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { - if s.lookupReady { - s.pending.Status = model.AttachmentReady - return s.pending, nil - } - return model.Attachment{}, model.ErrNotFound -} - func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { return s.pending, nil } +func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) { + return s.pending, model.Variant{}, nil +} + func (s *repositoryStub) record(event string) { if s.events != nil { *s.events = append(*s.events, event) diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go new file mode 100644 index 00000000..35a5c664 --- /dev/null +++ b/internal/messenger/attachment/transform/exif.go @@ -0,0 +1,146 @@ +package transform + +import ( + "encoding/binary" + "errors" + "image" + "image/color" +) + +var errInvalidEXIF = errors.New("invalid EXIF metadata") + +func jpegOrientation(data []byte) (int, error) { + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + return 0, errInvalidEXIF + } + for offset := 2; offset < len(data); { + if data[offset] != 0xff { + return 0, errInvalidEXIF + } + for offset < len(data) && data[offset] == 0xff { + offset++ + } + if offset >= len(data) { + return 0, errInvalidEXIF + } + marker := data[offset] + offset++ + if marker == 0xd9 || marker == 0xda { + return 1, nil + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + if offset+2 > len(data) { + return 0, errInvalidEXIF + } + length := int(binary.BigEndian.Uint16(data[offset : offset+2])) + if length < 2 || offset+length > len(data) { + return 0, errInvalidEXIF + } + payload := data[offset+2 : offset+length] + offset += length + if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { + continue + } + return tiffOrientation(payload[6:]) + } + return 1, nil +} + +func tiffOrientation(data []byte) (int, error) { + if len(data) < 8 { + return 0, errInvalidEXIF + } + var order binary.ByteOrder + switch string(data[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 0, errInvalidEXIF + } + if order.Uint16(data[2:4]) != 42 { + return 0, errInvalidEXIF + } + ifdOffset := uint64(order.Uint32(data[4:8])) + if ifdOffset > uint64(len(data)-2) { + return 0, errInvalidEXIF + } + count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2])) + entriesStart := ifdOffset + 2 + if count > (uint64(len(data))-entriesStart)/12 { + return 0, errInvalidEXIF + } + for index := uint64(0); index < count; index++ { + entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] + if order.Uint16(entry[:2]) != 0x0112 { + continue + } + if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 { + return 0, errInvalidEXIF + } + orientation := int(order.Uint16(entry[8:10])) + if orientation < 1 || orientation > 8 { + return 0, errInvalidEXIF + } + return orientation, nil + } + return 1, nil +} + +func applyOrientation(source image.Image, orientation int) *image.NRGBA { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + outputWidth, outputHeight := width, height + if orientation >= 5 { + outputWidth, outputHeight = height, width + } + output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) + for y := 0; y < outputHeight; y++ { + for x := 0; x < outputWidth; x++ { + sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) + output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) + } + } + return output +} + +func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { + switch orientation { + case 2: + return width - 1 - x, y + case 3: + return width - 1 - x, height - 1 - y + case 4: + return x, height - 1 - y + case 5: + return y, x + case 6: + return y, height - 1 - x + case 7: + return width - 1 - y, height - 1 - x + case 8: + return width - 1 - y, x + default: + return x, y + } +} + +func flattenWhite(source image.Image) *image.NRGBA { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + for x := 0; x < bounds.Dx(); x++ { + r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() + output.SetNRGBA(x, y, color.NRGBA{ + R: uint8((r + 0xffff - a) / 0x101), + G: uint8((g + 0xffff - a) / 0x101), + B: uint8((b + 0xffff - a) / 0x101), + A: 0xff, + }) + } + } + return output +} diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go new file mode 100644 index 00000000..a8dde17d --- /dev/null +++ b/internal/messenger/attachment/transform/processor.go @@ -0,0 +1,237 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/jpeg" + _ "image/png" + "math" + "net/http" + + "github.com/deepteams/webp" +) + +const ( + // DefaultJPEGQuality is used for thumbnail and crop variants. + DefaultJPEGQuality = 85 + // DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size. + DefaultWebPQuality = 82 + // DefaultWebPMethod selects the encoder's balanced quality and CPU mode. + DefaultWebPMethod = 4 +) + +var ( + ErrInvalidConfig = errors.New("invalid image transform configuration") + ErrUnsupportedMedia = errors.New("unsupported image media type") + ErrPayloadTooLarge = errors.New("image exceeds configured limits") + ErrInvalidImage = errors.New("invalid image data") +) + +type Config struct { + MaxBytes int64 + MaxPixels int64 + JPEGQuality int + WebPQuality float32 + WebPMethod int +} + +type Variant struct { + Bytes []byte + ContentType string + Width int + Height int + SHA256 string +} + +type Result struct { + Original Variant + Thumbnail Variant + Crop Variant + Optimized Variant + Orientation int +} + +type Processor struct { + config Config +} + +func DefaultConfig(maxBytes, maxPixels int64) Config { + return Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, + JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod, + } +} + +func NewProcessor(config Config) (*Processor, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 || + config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 { + return nil, ErrInvalidConfig + } + return &Processor{config: config}, nil +} + +func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) { + if err := contextError(ctx); err != nil { + return Result{}, err + } + img, format, width, height, orientation, err := p.decode(data, declaredContentType) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + oriented := applyOrientation(img, orientation) + if err := contextError(ctx); err != nil { + return Result{}, err + } + opaque := flattenWhite(oriented) + if err := contextError(ctx); err != nil { + return Result{}, err + } + result := Result{ + Original: variant(data, format, width, height), + Orientation: orientation, + } + + thumbnail := fit(opaque, 256, 256, false) + result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + crop := centeredSquare(opaque) + crop = resize(crop, 128, 128) + result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + optimized := fit(opaque, 1280, 1280, false) + result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + return result, nil +} + +func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) { + if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + if len(data) == 0 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(len(data)) > p.config.MaxBytes { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + if http.DetectContentType(data) != declaredContentType { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + config, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + img, decodedFormat, err := image.Decode(bytes.NewReader(data)) + if err != nil || decodedFormat != format { + return nil, "", 0, 0, 0, ErrInvalidImage + } + orientation := 1 + if format == "jpeg" { + orientation, err = jpegOrientation(data) + if err != nil { + return nil, "", 0, 0, 0, ErrInvalidImage + } + } + return img, declaredContentType, config.Width, config.Height, orientation, nil +} + +func contextError(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func contentType(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + default: + return "" + } +} + +func encodeJPEG(img image.Image, quality int) (Variant, error) { + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil +} + +func encodeWebP(img image.Image, quality float32, method int) (Variant, error) { + var output bytes.Buffer + options := &webp.EncoderOptions{Quality: quality, Method: method} + if err := webp.Encode(&output, img, options); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil +} + +func variant(data []byte, contentType string, width, height int) Variant { + digest := sha256.Sum256(data) + return Variant{ + Bytes: data, ContentType: contentType, Width: width, Height: height, + SHA256: hex.EncodeToString(digest[:]), + } +} + +// EstimateMemoryWeight returns a conservative reservation for the input, decoded source, +// orientation and alpha copies, variant working buffer, and encoded variant buffers. +func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) { + if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) { + return 0, ErrInvalidImage + } + pixels := int64(width) * int64(height) + optimizedPixels := fittedPixels(width, height, 1280, 1280) + thumbnailPixels := fittedPixels(width, height, 256, 256) + const cropPixels int64 = 128 * 128 + if pixels > (math.MaxInt64-inputBytes)/16 { + return 0, ErrPayloadTooLarge + } + weight := inputBytes + pixels*16 + variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4 + if variants > math.MaxInt64-weight { + return 0, ErrPayloadTooLarge + } + return weight + variants, nil +} + +func fittedPixels(width, height, maxWidth, maxHeight int) int64 { + newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false) + return int64(newWidth) * int64(newHeight) +} diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go new file mode 100644 index 00000000..972cb6d6 --- /dev/null +++ b/internal/messenger/attachment/transform/processor_test.go @@ -0,0 +1,236 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" + + "github.com/deepteams/webp" +) + +func TestProcessProducesExpectedVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 320*200) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Original, "image/png", 320, 200) + assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) + assertVariant(t, result.Optimized, "image/webp", 320, 200) + if !bytes.Equal(result.Original.Bytes, data) { + t.Fatal("original bytes changed") + } + digest := sha256.Sum256(data) + if result.Original.SHA256 != hex.EncodeToString(digest[:]) { + t.Fatalf("unexpected original digest: %s", result.Original.SHA256) + } + if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil { + t.Fatalf("decode optimized WebP: %v", err) + } +} + +func TestProcessDoesNotUpscaleFitVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 64, 32)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 64*32) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32) + assertVariant(t, result.Optimized, "image/webp", 64, 32) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) +} + +func TestFlattenWhiteCompositesAlpha(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) + source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) + + result := flattenWhite(source) + if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { + t.Fatalf("transparent pixel: got %#v", got) + } + if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) { + t.Fatalf("semi-transparent pixel: got %#v", got) + } +} + +func TestProcessAppliesEXIFOrientation(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + fillGradient(source) + data := injectOrientation(t, encodeJPEGForTest(t, source), 6) + processor := newTestProcessor(t, int64(len(data))+1, 3*2) + + result, err := processor.Process(context.Background(), data, "image/jpeg") + if err != nil { + t.Fatalf("process image: %v", err) + } + + if result.Orientation != 6 { + t.Fatalf("orientation: got %d, want 6", result.Orientation) + } + assertVariant(t, result.Original, "image/jpeg", 3, 2) + assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3) + assertVariant(t, result.Optimized, "image/webp", 2, 3) +} + +func TestProcessRejectsInvalidInputs(t *testing.T) { + valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10))) + + tests := []struct { + name string + data []byte + contentType string + maxPixels int64 + want error + }{ + {name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge}, + {name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels) + _, err := processor.Process(context.Background(), test.data, test.contentType) + if !errors.Is(err, test.want) { + t.Fatalf("error: got %v, want %v", err, test.want) + } + }) + } +} + +func TestProcessIsDeterministic(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 48, 31)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 48*31) + + first, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("first process: %v", err) + } + second, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("second process: %v", err) + } + + for name, variants := range map[string][2]Variant{ + "thumbnail": {first.Thumbnail, second.Thumbnail}, + "crop": {first.Crop, second.Crop}, + "optimized": {first.Optimized, second.Optimized}, + } { + if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) { + t.Fatalf("%s bytes differ", name) + } + } +} + +func TestProcessHonorsCanceledContext(t *testing.T) { + processor := newTestProcessor(t, 1024, 100) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := processor.Process(ctx, []byte("unused"), "image/png") + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +func TestEstimateMemoryWeight(t *testing.T) { + weight, err := EstimateMemoryWeight(1000, 320, 200) + if err != nil { + t.Fatalf("estimate memory: %v", err) + } + want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4) + if weight != want { + t.Fatalf("weight: got %d, want %d", weight, want) + } +} + +func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor { + t.Helper() + processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels)) + if err != nil { + t.Fatalf("new processor: %v", err) + } + return processor +} + +func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) { + t.Helper() + if variant.ContentType != contentType || variant.Width != width || variant.Height != height { + t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d", + variant.ContentType, variant.Width, variant.Height, contentType, width, height) + } + if len(variant.Bytes) == 0 || variant.SHA256 == "" { + t.Fatal("variant has no bytes or digest") + } +} + +func fillGradient(img *image.NRGBA) { + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } +} + +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return output.Bytes() +} + +func encodeJPEGForTest(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil { + t.Fatalf("encode JPEG: %v", err) + } + return output.Bytes() +} + +func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte { + t.Helper() + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + t.Fatal("not a JPEG") + } + tiff := make([]byte, 26) + copy(tiff[:2], "II") + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + payload := append([]byte("Exif\x00\x00"), tiff...) + segment := []byte{0xff, 0xe1, 0, 0} + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + result := make([]byte, 0, len(data)+len(segment)+len(payload)) + result = append(result, data[:2]...) + result = append(result, segment...) + result = append(result, payload...) + return append(result, data[2:]...) +} diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go new file mode 100644 index 00000000..4526b5ef --- /dev/null +++ b/internal/messenger/attachment/transform/resize.go @@ -0,0 +1,105 @@ +package transform + +import ( + "image" + "image/color" +) + +func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA { + bounds := source.Bounds() + width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) + return resize(source, width, height) +} + +func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { + if !upscale && width <= maxWidth && height <= maxHeight { + return width, height + } + if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) { + newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width))) + return maxWidth, newHeight + } + newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height))) + return newWidth, maxHeight +} + +func centeredSquare(source image.Image) image.Image { + bounds := source.Bounds() + size := min(bounds.Dx(), bounds.Dy()) + left := bounds.Min.X + (bounds.Dx()-size)/2 + top := bounds.Min.Y + (bounds.Dy()-size)/2 + return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)} +} + +type croppedImage struct { + source image.Image + bounds image.Rectangle + offset image.Point +} + +func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() } +func (i croppedImage) Bounds() image.Rectangle { return i.bounds } +func (i croppedImage) At(x, y int) color.Color { + return i.source.At(i.offset.X+x, i.offset.Y+y) +} + +func resize(source image.Image, width, height int) *image.NRGBA { + bounds := source.Bounds() + if bounds.Dx() == width && bounds.Dy() == height { + return copyNRGBA(source) + } + output := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) + for x := 0; x < width; x++ { + x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) + c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) + c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) + c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA) + c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA) + output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) + } + } + return output +} + +func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { + coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15 + if coordinate <= 0 { + return 0, min(1, sourceSize-1), 0 + } + maximum := int64(sourceSize-1) << 16 + if coordinate >= maximum { + return sourceSize - 1, sourceSize - 1, 0 + } + first := int(coordinate >> 16) + return first, first + 1, uint32(coordinate & 0xffff) +} + +func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA { + return color.NRGBA{ + R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy), + G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy), + B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy), + A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy), + } +} + +func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { + const one uint64 = 1 << 16 + top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx) + bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx) + value := top*(one-uint64(fy)) + bottom*uint64(fy) + return uint8((value + 1<<31) >> 32) +} + +func copyNRGBA(source image.Image) *image.NRGBA { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + for x := 0; x < bounds.Dx(); x++ { + output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + return output +} diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go new file mode 100644 index 00000000..9fa0082a --- /dev/null +++ b/internal/messenger/attachment/worker/worker.go @@ -0,0 +1,452 @@ +package worker + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "path" + "strings" + "sync" + "time" + + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" +) + +type Repository interface { + AttachmentJob(context.Context, string) (model.AttachmentJob, error) + ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) + FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) + RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) +} + +type Broker interface { + Read(context.Context) ([]redisstream.Delivery, error) + Reclaim(context.Context, string) ([]redisstream.Delivery, string, error) +} + +type ObjectStore interface { + Get(context.Context, string) (io.ReadCloser, error) + Put(context.Context, string, string, []byte) error + Delete(context.Context, string) error +} + +type ImageProcessor interface { + Process(context.Context, []byte, string) (transform.Result, error) +} + +type Metrics interface { + ObserveStarted() + ObserveFinished(string, time.Duration) + ObserveRetry() + ObserveReclaimed(int) +} + +type Config struct { + WorkerID string + Concurrency int + LeaseDuration time.Duration + ProcessingTimeout time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + MaxBytes int64 + MaxPixels int64 + MemoryBytes int64 +} + +type FaultPoint string + +const ( + FaultNone FaultPoint = "" + FaultBeforeSideEffect FaultPoint = "before_side_effect" + FaultAfterSideEffect FaultPoint = "after_side_effect" + FaultAfterCommit FaultPoint = "after_commit" +) + +var ErrInjectedFailure = errors.New("injected worker failure") + +type Worker struct { + logger *slog.Logger + repository Repository + broker Broker + objects ObjectStore + processor ImageProcessor + metrics Metrics + config Config + memory *semaphore.Weighted + now func() time.Time +} + +func New( + logger *slog.Logger, + repository Repository, + broker Broker, + objects ObjectStore, + processor ImageProcessor, + metrics Metrics, + config Config, +) (*Worker, error) { + if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil { + return nil, errors.New("worker dependencies are required") + } + if err := validateConfig(config); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Worker{ + logger: logger, repository: repository, broker: broker, objects: objects, + processor: processor, metrics: metrics, config: config, + memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now, + }, nil +} + +func validateConfig(config Config) error { + if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || + config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || + config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { + return errors.New("invalid worker configuration") + } + const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128 + if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 || + config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes { + return errors.New("worker memory must fit one maximum image") + } + return nil +} + +func (w *Worker) Run(ctx context.Context) error { + lastReclaim := time.Time{} + reclaimCursor := "0-0" + for ctx.Err() == nil { + deliveries, err := w.broker.Read(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.processBatch(ctx, deliveries) + if w.now().Sub(lastReclaim) < w.config.ReclaimInterval { + continue + } + recovery, err := w.repository.RecoverExpiredAttachmentJobs( + ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase), + ) + if err != nil { + return fmt.Errorf("recover expired attachment jobs: %w", err) + } + if recovery.Requeued > 0 || recovery.Failed > 0 { + w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed) + } + deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.metrics.ObserveReclaimed(len(deliveries)) + w.processBatch(ctx, deliveries) + reclaimCursor = next + if reclaimCursor == "" { + reclaimCursor = "0-0" + } + lastReclaim = w.now() + } + return nil +} + +func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) { + limit := make(chan struct{}, w.config.Concurrency) + var group sync.WaitGroup + for _, delivery := range deliveries { + if ctx.Err() != nil { + break + } + limit <- struct{}{} + group.Add(1) + go func(delivery redisstream.Delivery) { + defer group.Done() + defer func() { <-limit }() + if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil { + w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err) + } + }(delivery) + } + group.Wait() +} + +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error { + started := w.now() + w.metrics.ObserveStarted() + result := "transient_error" + defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }() + + if delivery.DecodeError != nil { + result = "invalid_job" + return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + } + envelope := delivery.Envelope + lease, err := w.repository.ClaimAttachmentJob( + ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, + w.config.WorkerID, w.config.LeaseDuration, + ) + if err != nil { + result, err = w.resolveClaim(ctx, delivery, envelope, err) + return err + } + if fault == FaultBeforeSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + + processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout) + variants, err := w.process(processingCtx, lease) + cancel() + if err != nil { + result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) + return err + } + if fault == FaultAfterSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants) + if err != nil { + return fmt.Errorf("complete attachment job: %w", err) + } + if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { + if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { + w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + } + } + if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { + return fmt.Errorf("unexpected completion result %q", mutation) + } + if fault == FaultAfterCommit { + result = "injected_failure" + return ErrInjectedFailure + } + if err := delivery.Ack(ctx); err != nil { + return err + } + result = "ready" + return nil +} + +func (w *Worker) resolveClaim( + ctx context.Context, + delivery redisstream.Delivery, + envelope redisstream.Envelope, + claimErr error, +) (string, error) { + switch { + case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput): + return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobBusy): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobNotDue): + return "not_due", nil + case errors.Is(claimErr, model.ErrConflict): + job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String()) + if err != nil { + return "transient_error", fmt.Errorf("inspect conflicting job: %w", err) + } + switch job.Status { + case model.AttachmentJobReady: + return "duplicate", delivery.Ack(ctx) + case model.AttachmentJobFailed: + code := "processing_failed" + if job.ErrorCode != nil { + code = *job.ErrorCode + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } +} + +func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) { + body, err := w.objects.Get(ctx, lease.Attachment.StagingKey) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > w.config.MaxBytes { + return nil, transform.ErrPayloadTooLarge + } + imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, transform.ErrInvalidImage + } + weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) + if err != nil { + return nil, err + } + if weight > w.config.MemoryBytes { + return nil, errWorkerCapacity + } + if err := w.memory.Acquire(ctx, weight); err != nil { + return nil, err + } + defer w.memory.Release(weight) + + processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType) + if err != nil { + return nil, err + } + variants := []struct { + name string + value transform.Variant + }{ + {name: "original", value: processed.Original}, + {name: "thumbnail", value: processed.Thumbnail}, + {name: "crop", value: processed.Crop}, + {name: "optimized", value: processed.Optimized}, + } + result := make([]model.Variant, 0, len(variants)) + for _, item := range variants { + objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value) + if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil { + return nil, err + } + result = append(result, model.Variant{ + Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)), + SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height, + ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion, + }) + } + return result, nil +} + +var errWorkerCapacity = errors.New("image exceeds worker memory budget") + +func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { + extension := ".bin" + switch value.ContentType { + case "image/png": + extension = ".png" + case "image/jpeg": + extension = ".jpg" + case "image/webp": + extension = ".webp" + } + return path.Join( + "attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion), + name, value.SHA256+extension, + ) +} + +func (w *Worker) resolveProcessingFailure( + ctx context.Context, + delivery redisstream.Delivery, + lease model.AttachmentJobLease, + processingErr error, +) (string, error) { + code, permanent := processingError(processingErr) + if permanent || lease.Job.Attempt >= w.config.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + mutation, err := w.repository.FailAttachmentJob(ctx, lease, code) + if err != nil { + return "transient_error", fmt.Errorf("fail attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected failure result %q", mutation) + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + } + nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) + mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt) + if err != nil { + return "transient_error", fmt.Errorf("retry attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) + } + w.metrics.ObserveRetry() + return "retry", delivery.Ack(ctx) +} + +func processingError(err error) (string, bool) { + switch { + case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia): + return "invalid_image", true + case errors.Is(err, transform.ErrPayloadTooLarge): + return "image_too_large", true + case errors.Is(err, objectstore.ErrNotFound): + return "source_missing", true + case errors.Is(err, errWorkerCapacity): + return "worker_capacity", true + default: + return "processing_failed", false + } +} + +func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { + delay := w.config.RetryBase + for step := 1; step < attempt && delay < w.config.RetryMax; step++ { + if delay > w.config.RetryMax/2 { + delay = w.config.RetryMax + break + } + delay *= 2 + } + if delay > w.config.RetryMax { + delay = w.config.RetryMax + } + var hash uint64 + for index := range jobID { + hash = hash*131 + uint64(jobID[index]) + } + jitterRange := min(delay/4, w.config.RetryMax-delay) + if jitterRange <= 0 { + return delay + } + jitter := time.Duration(hash % uint64(jitterRange)) + if delay > time.Duration(math.MaxInt64)-jitter { + return delay + } + return delay + jitter +} + +type nopMetrics struct{} + +func (nopMetrics) ObserveStarted() {} +func (nopMetrics) ObserveFinished(string, time.Duration) {} +func (nopMetrics) ObserveRetry() {} +func (nopMetrics) ObserveReclaimed(int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go new file mode 100644 index 00000000..77a1d5f5 --- /dev/null +++ b/internal/messenger/attachment/worker/worker_test.go @@ -0,0 +1,56 @@ +package worker + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" +) + +func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { + config := validConfig() + require.NoError(t, validateConfig(config)) + + config.MemoryBytes = 384 * 1024 * 1024 + require.ErrorContains(t, validateConfig(config), "memory must fit") +} + +func TestRetryDelayIsBounded(t *testing.T) { + worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}} + for attempt := 1; attempt <= 10; attempt++ { + delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt) + require.GreaterOrEqual(t, delay, time.Second) + require.LessOrEqual(t, delay, 4*time.Second) + } +} + +func TestProcessingErrorClassification(t *testing.T) { + tests := []struct { + err error + code string + permanent bool + }{ + {err: transform.ErrInvalidImage, code: "invalid_image", permanent: true}, + {err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true}, + {err: objectstore.ErrNotFound, code: "source_missing", permanent: true}, + {err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false}, + } + for _, test := range tests { + code, permanent := processingError(test.err) + require.Equal(t, test.code, code) + require.Equal(t, test.permanent, permanent) + } +} + +func validConfig() Config { + return Config{ + WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, + ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, + } +} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index bab14f6d..f9250c85 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -6,6 +6,7 @@ import ( "mime" "net/http" "net/url" + "regexp" "strconv" "strings" "time" @@ -46,6 +47,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ ChatID: chatID, OwnerID: user.ID, Filename: filename, ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")), }) if err != nil { if errors.Is(err, model.ErrCapacity) { @@ -58,7 +60,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr } result = "success" s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) - writeJSON(w, http.StatusCreated, uploaded) + writeJSON(w, http.StatusAccepted, uploaded) return nil } @@ -108,20 +110,21 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE return authErr } query, err := url.ParseQuery(r.URL.RawQuery) - if err != nil || !originalVariant(query) { + variant, valid := attachmentVariant(query) + if err != nil || !valid { return invalidRequest() } attachmentID, apiErr := attachmentIDFromRequest(r) if apiErr != nil { return apiErr } - content, err := s.attachments.Content(r.Context(), attachmentID, user.ID) + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant) if err != nil { return attachmentError(err) } defer func() { _ = content.Body.Close() }() - w.Header().Set("Content-Type", content.Attachment.ContentType) - w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10)) + w.Header().Set("Content-Type", content.Variant.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10)) w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, content.Body); err != nil { @@ -130,11 +133,35 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE return nil } -func originalVariant(query url.Values) bool { +func attachmentVariant(query url.Values) (string, bool) { if len(query) == 0 { - return true + return "original", true } - return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" + if len(query) != 1 || len(query["variant"]) != 1 { + return "", false + } + switch query["variant"][0] { + case "original", "thumbnail", "crop", "optimized": + return query["variant"][0], true + default: + return "", false + } +} + +var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +func traceparent(value string) string { + if traceparentPattern.MatchString(value) { + return value + } + return "" +} + +func tracestate(value string) string { + if len(value) <= 512 { + return value + } + return "" } func attachmentIDFromRequest(r *http.Request) (string, *apiError) { @@ -158,6 +185,8 @@ func attachmentError(err error) *apiError { return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} case errors.Is(err, model.ErrCapacity): return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + case errors.Is(err, model.ErrConflict): + return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"} default: return unavailable() } diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go index 5fd83ebe..055b80b1 100644 --- a/internal/messenger/httpapi/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -28,7 +28,7 @@ type MessageRepository interface { type AttachmentService interface { Upload(context.Context, attachment.Upload) (model.Attachment, error) Metadata(context.Context, string, string) (model.Attachment, error) - Content(context.Context, string, string) (attachment.Content, error) + Content(context.Context, string, string, string) (attachment.Content, error) } type HealthChecker interface { diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index 8199a0d7..00467d34 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -87,8 +87,12 @@ func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { return s.uploaded, s.err } -func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) { - return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err +func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) { + return attachment.Content{ + Attachment: s.uploaded, + Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, + Body: io.NopCloser(strings.NewReader("content")), + }, s.err } type sessionStub struct { diff --git a/internal/model/errors.go b/internal/model/errors.go index dbe09f09..83105b27 100644 --- a/internal/model/errors.go +++ b/internal/model/errors.go @@ -15,4 +15,10 @@ var ( ErrPayloadTooLarge = errors.New("payload too large") ErrCapacity = errors.New("capacity exhausted") ErrUnavailable = errors.New("dependency unavailable") + ErrStaleLease = errors.New("stale lease") + ErrStaleAttempt = errors.New("stale attempt") + ErrStalePipeline = errors.New("stale pipeline") + ErrJobBusy = errors.New("job is already leased") + ErrJobComplete = errors.New("job is already complete") + ErrJobNotDue = errors.New("job is not due") ) diff --git a/internal/model/model.go b/internal/model/model.go index bc71b978..83f79956 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -6,9 +6,21 @@ import ( ) const ( - AttachmentPending = "pending" - AttachmentReady = "ready" - AttachmentDeleting = "deleting" + AttachmentPending = "pending" + AttachmentQueued = "queued" + AttachmentProcessing = "processing" + AttachmentReady = "ready" + AttachmentFailed = "failed" + AttachmentDeleting = "deleting" + + AttachmentJobQueued = "queued" + AttachmentJobProcessing = "processing" + AttachmentJobReady = "ready" + AttachmentJobFailed = "failed" + + JobMutationApplied JobMutationResult = "applied" + JobMutationAlreadyComplete JobMutationResult = "already_complete" + JobMutationStale JobMutationResult = "stale" ) const timestampLayout = "2006-01-02T15:04:05.000000Z" @@ -42,21 +54,23 @@ type Message struct { } type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []Variant `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ObjectKey string `json:"-"` - ReadyAt *time.Time `json:"-"` - Referenced bool `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + StagingKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` + PipelineVersion int `json:"-"` } type StoredObject struct { @@ -66,12 +80,68 @@ type StoredObject struct { } type Variant struct { - Name string `json:"name"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Width int `json:"width"` - Height int `json:"height"` + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` +} + +type AttachmentJob struct { + ID string + AttachmentID string + PipelineVersion int + Attempt int + Status string + AvailableAt time.Time + LeaseToken string + LeaseOwner string + LeaseUntil *time.Time + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentJobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type AttachmentJobLease struct { + Job AttachmentJob + Attachment Attachment +} + +type OutboxEvent struct { + ID string + Topic string + PartitionKey string + Payload json.RawMessage + JobID string + Attempt int + AvailableAt time.Time + PublishTries int + LeaseToken string + LeaseUntil *time.Time + LastErrorCode *string + CreatedAt time.Time +} + +type JobMutationResult string + +type RecoveryResult struct { + Requeued int + Failed int } type Position struct { diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go new file mode 100644 index 00000000..17778e3c --- /dev/null +++ b/internal/outbox/dispatcher.go @@ -0,0 +1,174 @@ +package outbox + +import ( + "context" + "errors" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +var errLeaseLost = errors.New("outbox lease lost") + +type Store interface { + ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) + MarkOutboxPublished(context.Context, string, string) (bool, error) + ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) + RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) +} + +type Publisher interface { + PublishConfirmed(context.Context, []byte) (string, error) +} + +type Metrics interface { + ObservePublished() + ObservePublishFailure() + ObserveRecovered(int) +} + +type Config struct { + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration +} + +type Dispatcher struct { + store Store + publisher Publisher + metrics Metrics + config Config + now func() time.Time +} + +func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) { + if store == nil { + return nil, errors.New("outbox store is required") + } + if publisher == nil { + return nil, errors.New("outbox publisher is required") + } + if err := config.validate(); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil +} + +func (c Config) validate() error { + switch { + case c.BatchSize <= 0: + return errors.New("outbox batch size must be positive") + case c.LeaseDuration <= 0: + return errors.New("outbox lease duration must be positive") + case c.PollInterval <= 0: + return errors.New("outbox poll interval must be positive") + case c.RecoveryInterval <= 0: + return errors.New("outbox recovery interval must be positive") + case c.RetryDelay <= 0: + return errors.New("outbox retry delay must be positive") + default: + return nil + } +} + +func (d *Dispatcher) Run(ctx context.Context) error { + if ctx.Err() != nil { + return nil + } + + pollTicker := time.NewTicker(d.config.PollInterval) + recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + defer pollTicker.Stop() + defer recoveryTicker.Stop() + + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + for { + if _, err := d.DispatchOnce(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + + select { + case <-ctx.Done(): + return nil + case <-pollTicker.C: + case <-recoveryTicker.C: + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + } + } +} + +func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { + events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) + if err != nil { + return 0, fmt.Errorf("claim due outbox events: %w", err) + } + + published := 0 + for _, event := range events { + if err := ctx.Err(); err != nil { + return published, nil + } + _, err := d.publisher.PublishConfirmed(ctx, event.Payload) + if err != nil { + if ctx.Err() != nil { + return published, nil + } + d.metrics.ObservePublishFailure() + nextAttemptAt := d.now().Add(d.config.RetryDelay) + released, releaseErr := d.store.ReleaseOutboxEvent( + ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + ) + if releaseErr != nil { + return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) + } + if !released { + return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost) + } + continue + } + marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken) + if err != nil { + return published, fmt.Errorf("mark outbox event published: %w", err) + } + if !marked { + return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) + } + d.metrics.ObservePublished() + published++ + } + return published, nil +} + +func (d *Dispatcher) RecoverExpired(ctx context.Context) error { + recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now()) + if err != nil { + return fmt.Errorf("recover expired outbox leases: %w", err) + } + d.metrics.ObserveRecovered(recovered) + return nil +} + +type nopMetrics struct{} + +func (nopMetrics) ObservePublished() {} +func (nopMetrics) ObservePublishFailure() {} +func (nopMetrics) ObserveRecovered(int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go new file mode 100644 index 00000000..036741e8 --- /dev/null +++ b/internal/outbox/dispatcher_test.go @@ -0,0 +1,181 @@ +package outbox + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "github.com/stretchr/testify/require" +) + +func TestDispatcherPublishesAndMarksEvent(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, []string{"event-1:lease-1"}, store.marked) +} + +func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) { + event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()} + store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + _, err := dispatcher.DispatchOnce(context.Background()) + require.Error(t, err) + require.Equal(t, 1, publisher.calls) + + store.markErr = nil + store.events[0].LeaseToken = "lease-2" + publisher.messageID = "2-0" + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, 2, publisher.calls) + require.JSONEq(t, string(event.Payload), string(publisher.payloads[0])) + require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1])) + require.Equal(t, []string{"event-1:lease-2"}, store.marked) +} + +func TestDispatcherReleasesPublishFailure(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{err: errors.New("redis unavailable")} + dispatcher := newTestDispatcher(t, store, publisher) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Zero(t, published) + require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released) + require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt) +} + +func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { + store := &storeStub{ + events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}}, + markLost: true, + } + dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"}) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.ErrorIs(t, err, errLeaseLost) + require.Zero(t, published) +} + +func TestDispatcherStopsAfterCancellation(t *testing.T) { + store := &storeStub{} + publisher := &publisherStub{} + dispatcher := newTestDispatcher(t, store, publisher) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { done <- dispatcher.Run(ctx) }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("dispatcher did not stop after cancellation") + } +} + +func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher { + t.Helper() + dispatcher, err := New(store, publisher, nil, Config{ + BatchSize: 10, + LeaseDuration: time.Minute, + PollInterval: time.Millisecond, + RecoveryInterval: time.Second, + RetryDelay: time.Second, + }) + require.NoError(t, err) + return dispatcher +} + +func testPayload() json.RawMessage { + return json.RawMessage(`{ + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-10-22T12:00:00Z" + }`) +} + +type storeStub struct { + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int +} + +func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]model.OutboxEvent(nil), s.events...), nil +} + +func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.markErr != nil { + return false, s.markErr + } + if s.markLost { + return false, nil + } + s.marked = append(s.marked, id+":"+leaseToken) + return true, nil +} + +func (s *storeStub) ReleaseOutboxEvent( + _ context.Context, + id string, + leaseToken string, + code string, + nextAttemptAt time.Time, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.released = id + ":" + leaseToken + ":" + code + s.nextAttemptAt = nextAttemptAt + return true, nil +} + +func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recoveredCalls++ + return 0, nil +} + +type publisherStub struct { + mu sync.Mutex + messageID string + err error + calls int + payloads [][]byte +} + +func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.calls++ + p.payloads = append(p.payloads, append([]byte(nil), payload...)) + return p.messageID, p.err +} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go new file mode 100644 index 00000000..34f914e7 --- /dev/null +++ b/internal/postgres/attachment_jobs.go @@ -0,0 +1,560 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +const attachmentJobTopic = "attachment.jobs" + +const attachmentJobColumns = ` + id, attachment_id, pipeline_version, attempt, status, available_at, + lease_token, lease_owner, lease_until, error_code, created_at, updated_at` + +func (s *Store) QueueAttachmentWithJobAndOutbox( + ctx context.Context, + attachmentID string, + stagingKey string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 1 { + return model.Attachment{}, model.ErrInvalidInput + } + if job.Attempt == 0 { + job.Attempt = 1 + } + if job.Attempt != 1 { + return model.Attachment{}, model.ErrInvalidInput + } + now := time.Now().UTC() + if job.CreatedAt.IsZero() { + job.CreatedAt = now + } + if job.AvailableAt.IsZero() { + job.AvailableAt = now + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + query := ` + UPDATE attachments + SET status = 'queued', pipeline_version = $3, staging_key = $2, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + var attachment model.Attachment + err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job) + } + if err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment: %w", err) + } + + job.AttachmentID = attachmentID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = job.CreatedAt + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) QueueAttachmentPipelineVersion( + ctx context.Context, + attachmentID string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 2 { + return model.Attachment{}, model.ErrInvalidInput + } + job.Attempt = 1 + now := time.Now().UTC() + job.CreatedAt = now + job.AvailableAt = now + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err) + } + if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 { + return model.Attachment{}, ErrConflict + } + job.AttachmentID = attachment.ID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = now + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', pipeline_version = $2, staging_key = object_key, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 + RETURNING ` + attachmentColumns + if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) + } + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) alreadyQueuedAttachment( + ctx context.Context, + tx pgx.Tx, + attachmentID string, + stagingKey string, + job model.AttachmentJob, +) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready') + AND j.id = $3 AND j.pipeline_version = $4` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrConflict + } + return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error { + const query = ` + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)` + if _, err := tx.Exec( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt, + ); err != nil { + return fmt.Errorf("insert attachment job: %w", err) + } + return nil +} + +func insertAttachmentJobOutbox( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + traceparent string, + tracestate string, +) error { + payload, err := json.Marshal(model.AttachmentJobMessage{ + SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1", + AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, CreatedAt: job.CreatedAt, + Traceparent: traceparent, Tracestate: tracestate, + }) + if err != nil { + return fmt.Errorf("marshal attachment job event: %w", err) + } + const query = ` + INSERT INTO outbox_events ( + id, topic, partition_key, payload, job_id, attempt, available_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload, + job.ID, job.Attempt, job.AvailableAt, + ); err != nil { + return fmt.Errorf("insert attachment job outbox event: %w", err) + } + return nil +} + +func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) { + const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1` + var job model.AttachmentJob + if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrNotFound + } + return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err) + } + return job, nil +} + +func (s *Store) ClaimAttachmentJob( + ctx context.Context, + jobID string, + attachmentID string, + pipelineVersion int, + attempt int, + workerID string, + leaseDuration time.Duration, +) (model.AttachmentJobLease, error) { + if leaseDuration <= 0 || workerID == "" { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJobLease{}, err + } + if attachment.ID != attachmentID || job.AttachmentID != attachmentID { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady { + return model.AttachmentJobLease{}, model.ErrJobComplete + } + if job.Attempt != attempt { + return model.AttachmentJobLease{}, model.ErrStaleAttempt + } + if job.Status == model.AttachmentJobProcessing { + return model.AttachmentJobLease{}, model.ErrJobBusy + } + if job.Status != model.AttachmentJobQueued { + return model.AttachmentJobLease{}, ErrConflict + } + if job.AvailableAt.After(time.Now()) { + return model.AttachmentJobLease{}, model.ErrJobNotDue + } + + job.LeaseToken = uuid.NewString() + job.LeaseOwner = workerID + leaseUntil := time.Now().UTC().Add(leaseDuration) + job.LeaseUntil = &leaseUntil + job.Status = model.AttachmentJobProcessing + const updateJob = ` + UPDATE attachment_jobs + SET status = 'processing', lease_token = $2, lease_owner = $3, + lease_until = $4, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err) + } + const updateAttachment = ` + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'` + result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err) + } + if result.RowsAffected() != 1 { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err) + } + attachment.Status = model.AttachmentProcessing + attachment.ErrorCode = nil + return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil +} + +func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) { + query := ` + SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + ` + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = $1 + FOR UPDATE OF j, a` + var job model.AttachmentJob + var attachment model.Attachment + var leaseToken, leaseOwner *string + var stagingKey *string + err := tx.QueryRow(ctx, query, jobID).Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, model.Attachment{}, ErrNotFound + } + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err) + } + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return job, attachment, nil +} + +func prefixedAttachmentJobColumns(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+ + "%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+ + "%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at", + alias, + ) +} + +func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error { + var leaseToken, leaseOwner *string + err := row.Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + ) + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + return err +} + +func (s *Store) CompleteAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + variants []model.Variant, +) (model.JobMutationResult, error) { + if !validCompletedVariants(variants) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin complete attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const deleteVariants = ` + DELETE FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("replace attachment variants: %w", err) + } + + var originalKey string + const insertVariant = ` + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + sha256, width, height, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + for _, variant := range variants { + if variant.Name == "original" { + originalKey = variant.ObjectKey + } + if _, err := tx.Exec( + ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name, + variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width, + variant.Height, variant.ObjectKey, + ); err != nil { + return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err) + } + } + + const completeJob = ` + UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil { + return "", fmt.Errorf("complete attachment job: %w", err) + } + const completeAttachment = ` + UPDATE attachments + SET status = 'ready', object_key = $3, ready_at = clock_timestamp(), + staging_key = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { + return "", fmt.Errorf("complete attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit completed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func validCompletedVariants(variants []model.Variant) bool { + seen := make(map[string]struct{}, len(variants)) + for _, variant := range variants { + if variant.Name == "" || variant.ObjectKey == "" { + return false + } + if _, duplicate := seen[variant.Name]; duplicate { + return false + } + seen[variant.Name] = struct{}{} + } + _, hasOriginal := seen["original"] + _, hasThumbnail := seen["thumbnail"] + return hasOriginal && hasThumbnail +} + +func (s *Store) RetryAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, + nextAttemptAt time.Time, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin retry attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = nextAttemptAt.UTC() + job.ErrorCode = &errorCode + const retryJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $3, available_at = $4, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $5, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec( + ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode, + ); err != nil { + return "", fmt.Errorf("retry attachment job: %w", err) + } + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("requeue attachment: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit attachment job retry: %w", err) + } + return model.JobMutationApplied, nil +} + +func (s *Store) FailAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin fail attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $3, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil { + return "", fmt.Errorf("fail attachment job: %w", err) + } + const failAttachment = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return "", fmt.Errorf("fail attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit failed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func lockJobMutation( + ctx context.Context, + tx pgx.Tx, + lease model.AttachmentJobLease, +) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) { + job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID) + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, "", err + } + if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion { + return job, attachment, model.JobMutationStale, nil + } + if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady { + return job, attachment, model.JobMutationAlreadyComplete, nil + } + if job.PipelineVersion != lease.Job.PipelineVersion || + job.Attempt != lease.Job.Attempt || + job.Status != model.AttachmentJobProcessing || + job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken || + job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) { + return job, attachment, model.JobMutationStale, nil + } + return job, attachment, model.JobMutationApplied, nil +} + +func validErrorCode(errorCode string) bool { + return len(errorCode) >= 1 && len(errorCode) <= 64 +} diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go new file mode 100644 index 00000000..d7d84caa --- /dev/null +++ b/internal/postgres/attachment_recovery.go @@ -0,0 +1,210 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) RecoverExpiredAttachmentJobs( + ctx context.Context, + limit int, + maxAttempts int, + retryAt time.Time, +) (model.RecoveryResult, error) { + if limit < 1 || limit > 1000 || maxAttempts < 1 { + return model.RecoveryResult{}, model.ErrInvalidInput + } + if retryAt.IsZero() { + retryAt = time.Now().UTC() + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectExpired = ` + SELECT id + FROM attachment_jobs + WHERE status = 'processing' AND lease_until <= clock_timestamp() + ORDER BY lease_until, id + LIMIT $1 + FOR UPDATE SKIP LOCKED` + rows, err := tx.Query(ctx, selectExpired, limit) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err) + } + jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err) + } + + var recovered model.RecoveryResult + for _, jobID := range jobIDs { + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.RecoveryResult{}, err + } + if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) { + continue + } + if job.PipelineVersion != attachment.PipelineVersion { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + if job.Attempt >= maxAttempts { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = retryAt.UTC() + const requeueJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = 'lease_expired', updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err) + } + const requeueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.RecoveryResult{}, err + } + recovered.Requeued++ + } + if err := tx.Commit(ctx); err != nil { + return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err) + } + return recovered, nil +} + +func markExpiredJobFailed( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + attachment model.Attachment, + errorCode string, + failAttachment bool, +) error { + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $2, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { + return fmt.Errorf("fail expired attachment job: %w", err) + } + if !failAttachment { + return nil + } + const failAttachmentQuery = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return fmt.Errorf("fail attachment after expired lease: %w", err) + } + return nil +} + +func (s *Store) ReplayFailedAttachment( + ctx context.Context, + attachmentID string, + actor string, + reason string, +) (model.AttachmentJob, error) { + if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 { + return model.AttachmentJob{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectJob = ` + SELECT j.id + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed' + FOR UPDATE OF j, a` + var jobID string + if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrConflict + } + return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err) + } + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJob{}, err + } + previousAttempt := job.Attempt + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = time.Now().UTC() + job.ErrorCode = nil + job.LeaseToken = "" + job.LeaseOwner = "" + job.LeaseUntil = nil + const replayJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err) + } + const replayAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err) + } + const insertAudit = ` + INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID, + previousAttempt, job.Attempt, actor, reason, + ); err != nil { + return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.AttachmentJob{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err) + } + return job, nil +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 8728ae3c..44d60dff 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -13,7 +13,7 @@ import ( const attachmentColumns = ` id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, status, created_at, updated_at, ready_at` + object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key` type rowScanner interface { Scan(...any) error @@ -74,7 +74,7 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri SELECT ` + attachmentColumnsWithAlias("a") + ` FROM attachments a JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 - WHERE a.id = $1 AND a.status = 'ready'` + WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')` var attachment model.Attachment err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) @@ -84,9 +84,68 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri if err != nil { return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) } + if attachment.Status == model.AttachmentReady { + attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion) + if err != nil { + return model.Attachment{}, err + } + } return publicAttachment(attachment), nil } +func (s *Store) AttachmentVariantForUser( + ctx context.Context, + attachmentID string, + userID string, + variantName string, +) (model.Attachment, model.Variant, error) { + attachment, err := s.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, model.Variant{}, err + } + if attachment.Status != model.AttachmentReady { + return model.Attachment{}, model.Variant{}, ErrConflict + } + for _, variant := range attachment.Variants { + if variant.Name == variantName { + return attachment, variant, nil + } + } + if variantName == "original" && attachment.ObjectKey != "" { + return attachment, model.Variant{ + Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey, + PipelineVersion: attachment.PipelineVersion, + }, nil + } + return model.Attachment{}, model.Variant{}, ErrNotFound +} + +func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) { + const query = ` + SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY CASE name + WHEN 'original' THEN 0 + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END` + rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion) + if err != nil { + return nil, fmt.Errorf("list attachment variants: %w", err) + } + defer rows.Close() + + variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant]) + if err != nil { + return nil, fmt.Errorf("scan attachment variants: %w", err) + } + return variants, nil +} + func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` @@ -102,7 +161,12 @@ func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey } func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { - const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)` + const query = ` + SELECT EXISTS ( + SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1 + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1 + )` var exists bool if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { return false, fmt.Errorf("check attachment by object key: %w", err) @@ -144,12 +208,18 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi } func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { - return row.Scan( + var stagingKey *string + err := row.Scan( &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, &attachment.Referenced, ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err } func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { @@ -232,23 +302,36 @@ func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID str } func scanAttachment(row rowScanner, attachment *model.Attachment) error { - return row.Scan( + var stagingKey *string + err := row.Scan( &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err } func attachmentColumnsWithAlias(alias string) string { return fmt.Sprintf( "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ - "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at", + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+ + "%[1]s.error_code, %[1]s.staging_key", alias, ) } func publicAttachment(attachment model.Attachment) model.Attachment { - attachment.Variants = []model.Variant{} + variants := make([]model.Variant, 0, len(attachment.Variants)) + for _, variant := range attachment.Variants { + if variant.Name != "original" { + variants = append(variants, variant) + } + } + attachment.Variants = variants return attachment } diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index 0e56e166..e4daccb1 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -8,4 +8,10 @@ var ( ErrInvalidMembers = model.ErrInvalidMembers ErrInvalidCursor = model.ErrInvalidCursor ErrIdempotencyConflict = model.ErrIdempotencyConflict + ErrStaleLease = model.ErrStaleLease + ErrStaleAttempt = model.ErrStaleAttempt + ErrStalePipeline = model.ErrStalePipeline + ErrJobBusy = model.ErrJobBusy + ErrJobComplete = model.ErrJobComplete + ErrJobNotDue = model.ErrJobNotDue ) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index 49f576ef..a8168f52 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -51,13 +51,16 @@ func (s *Store) CreateMessage( if !visible { return model.Message{}, false, ErrNotFound } - validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) if err != nil { return model.Message{}, false, err } if !validAttachments { return model.Message{}, false, ErrNotFound } + if failedAttachment { + return model.Message{}, false, ErrConflict + } if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { return model.Message{}, false, err } @@ -92,28 +95,35 @@ func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, return sha256.Sum256(payload), nil } -func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) { +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) { if len(attachmentIDs) == 0 { - return true, nil + return true, false, nil } const query = ` - SELECT id + SELECT id, status FROM attachments - WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready' + WHERE chat_id = $1 AND id::text = ANY($2) + AND status IN ('queued', 'processing', 'ready', 'failed') FOR UPDATE` rows, err := tx.Query(ctx, query, chatID, attachmentIDs) if err != nil { - return false, fmt.Errorf("validate message attachments: %w", err) + return false, false, fmt.Errorf("validate message attachments: %w", err) } defer rows.Close() - count := 0 + var count int + var failed bool for rows.Next() { + var id, status string + if err := rows.Scan(&id, &status); err != nil { + return false, false, fmt.Errorf("scan message attachment: %w", err) + } count++ + failed = failed || status == model.AttachmentFailed } if err := rows.Err(); err != nil { - return false, fmt.Errorf("validate message attachments: %w", err) + return false, false, fmt.Errorf("validate message attachments: %w", err) } - return count == len(attachmentIDs), nil + return count == len(attachmentIDs), failed, nil } func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go new file mode 100644 index 00000000..6bfcc107 --- /dev/null +++ b/internal/postgres/outbox.go @@ -0,0 +1,113 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) ClaimOutboxEvents( + ctx context.Context, + limit int, + leaseDuration time.Duration, +) ([]model.OutboxEvent, error) { + if limit < 1 || limit > 1000 || leaseDuration <= 0 { + return nil, model.ErrInvalidInput + } + claimToken := uuid.NewString() + query := ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at IS NULL + AND available_at <= clock_timestamp() + AND (claim_until IS NULL OR claim_until <= clock_timestamp()) + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbox_events event + SET claim_token = $2, + claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond', + publish_tries = event.publish_tries + 1 + FROM candidates + WHERE event.id = candidates.id + RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id, + event.attempt, event.available_at, event.publish_tries, event.claim_token, + event.claim_until, event.last_error_code, event.created_at` + rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds()) + if err != nil { + return nil, fmt.Errorf("claim outbox events: %w", err) + } + defer rows.Close() + + events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) { + var event model.OutboxEvent + err := row.Scan( + &event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID, + &event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken, + &event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt, + ) + return event, err + })) + if err != nil { + return nil, fmt.Errorf("scan claimed outbox events: %w", err) + } + return events, nil +} + +func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) { + const query = ` + UPDATE outbox_events + SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL, + last_error_code = NULL + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken) + if err != nil { + return false, fmt.Errorf("mark outbox event published: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) ReleaseOutboxEvent( + ctx context.Context, + eventID string, + leaseToken string, + errorCode string, + nextAttemptAt time.Time, +) (bool, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE outbox_events + SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4 + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode) + if err != nil { + return false, fmt.Errorf("release outbox event: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) { + if now.IsZero() { + now = time.Now().UTC() + } + const query = ` + UPDATE outbox_events + SET claim_token = NULL, claim_until = NULL + WHERE published_at IS NULL AND claim_until <= $1` + result, err := s.pool.Exec(ctx, query, now.UTC()) + if err != nil { + return 0, fmt.Errorf("recover expired outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 5974d280..89ce5882 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -2,6 +2,7 @@ package postgres_test import ( "context" + "encoding/json" "errors" "os" "strings" @@ -159,6 +160,165 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { s.False(deleted) } +func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + s.Equal(model.AttachmentQueued, attachment.Status) + + events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute) + s.Require().NoError(err) + s.Require().NotEmpty(events) + var event model.OutboxEvent + for _, candidate := range events { + if candidate.JobID == job.ID { + event = candidate + break + } + } + s.Require().NotEmpty(event.ID) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(event.Payload, &message)) + s.Equal(job.ID, message.JobID) + s.Equal(attachment.ID, message.AttachmentID) + s.Equal(1, message.Attempt) + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + + _, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "queued attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute, + ) + s.ErrorIs(err, model.ErrInvalidInput) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + s.Equal(model.AttachmentProcessing, lease.Attachment.Status) + s.NotEmpty(lease.Attachment.StagingKey) + + variants := []model.Variant{ + { + Name: "original", ContentType: "image/png", SizeBytes: 128, + SHA256: strings.Repeat("a", 64), Width: 320, Height: 200, + ObjectKey: "attachments/" + attachment.ID + "/v1/original.png", + }, + { + Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64, + SHA256: strings.Repeat("b", 64), Width: 256, Height: 160, + ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg", + }, + } + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationAlreadyComplete, result) + + metadata, variant, err := s.store.AttachmentVariantForUser( + context.Background(), attachment.ID, owner.ID, "thumbnail", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, metadata.Status) + s.Len(metadata.Variants, 1) + s.Equal(variants[1].ObjectKey, variant.ObjectKey) + + v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2} + queuedV2, err := s.store.QueueAttachmentPipelineVersion( + context.Background(), attachment.ID, v2, "", "", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentQueued, queuedV2.Status) + s.Equal(2, queuedV2.PipelineVersion) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + + _, _, err = s.store.CreateMessage( + context.Background(), chatID, owner.ID, "failed attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrConflict) + metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentFailed, metadata.Status) + s.Require().NotNil(metadata.ErrorCode) + s.Equal("invalid_image", *metadata.ErrorCode) + + replayed, err := s.store.ReplayFailedAttachment( + context.Background(), attachment.ID, "operator@example", "input repaired", + ) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + s.Equal(model.AttachmentJobQueued, replayed.Status) + + var auditCount int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID, + ).Scan(&auditCount) + s.Require().NoError(err) + s.Equal(1, auditCount) +} + +func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now()) + s.Require().NoError(err) + s.Equal(1, recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(2, current.Attempt) + s.Equal(model.AttachmentJobQueued, current.Status) + + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, []model.Variant{ + { + Name: "original", ContentType: "image/png", SizeBytes: 128, + SHA256: strings.Repeat("a", 64), Width: 320, Height: 200, + ObjectKey: "attachments/" + attachment.ID + "/v1/original.png", + }, + { + Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64, + SHA256: strings.Repeat("b", 64), Width: 256, Height: 160, + ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg", + }, + }) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { owner, chatID := createOwnerAndChat(s.T(), s.store) pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ @@ -344,6 +504,35 @@ func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string return owner, chat.ID } +func createQueuedAttachment( + t *testing.T, + store *postgres.Store, + ownerID string, + chatID string, +) (model.Attachment, model.AttachmentJob) { + t.Helper() + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: stagingKey, + }) + if err != nil { + t.Fatalf("CreatePendingAttachment() error = %v", err) + } + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1, + } + queued, err := store.QueueAttachmentWithJobAndOutbox( + context.Background(), pending.ID, stagingKey, job, "", "", + ) + if err != nil { + t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err) + } + return queued, job +} + func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { t.Helper() var count int diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 267b39d3..6f84b585 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -23,6 +23,7 @@ type ObjectStore interface { type Config struct { Prefix string + OrphanPrefixes []string PageSize int GraceTime time.Duration UnreferencedRetention time.Duration @@ -65,6 +66,11 @@ func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconcile if config.UnreferencedRetention <= 0 { return nil, errors.New("unreferenced retention must be positive") } + for _, prefix := range config.OrphanPrefixes { + if prefix == "" || prefix == config.Prefix { + return nil, errors.New("orphan prefix must be non-empty and distinct") + } + } if config.Now == nil { config.Now = time.Now } @@ -113,9 +119,35 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) { } } } + for _, prefix := range r.config.OrphanPrefixes { + if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil { + return report, err + } + } return report, nil } +func (r *Reconciler) scanOrphanPrefix( + ctx context.Context, + prefix string, + cutoff time.Time, + report *Report, +) error { + objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize) + if err != nil { + return err + } + for objects.current != nil { + if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil { + return err + } + if err := objects.advance(ctx); err != nil { + return err + } + } + return nil +} + func (r *Reconciler) handlePair( ctx context.Context, metadata model.Attachment, @@ -226,6 +258,14 @@ func (r *Reconciler) handleMissingObject( report *Report, ) error { report.MetadataScanned++ + if metadata.Status == model.AttachmentPending { + if metadata.CreatedAt.After(cutoff) { + report.FreshPendingAttachments++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, cutoff, report) + } if metadata.Status == model.AttachmentDeleting { return r.deleteClaimedMetadata(ctx, metadata, report) } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 26c2d922..a468274a 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -21,6 +21,7 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) { "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), + "staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false), }} objects := &objectStub{items: map[string]model.StoredObject{ "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, @@ -28,27 +29,33 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) { "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + "staging/orphan/original": {Key: "staging/orphan/original", LastModified: old}, + "staging/pending/original": {Key: "staging/pending/original", LastModified: old}, }} reconciler, err := New(metadata, objects, Config{ Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + OrphanPrefixes: []string{"staging/"}, UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, }) require.NoError(t, err) report, err := reconciler.Run(context.Background()) require.NoError(t, err) - require.Equal(t, 1, report.OldPendingAttachments) + require.Equal(t, 2, report.OldPendingAttachments) require.Equal(t, 1, report.OldUnreferencedAttachments) require.Equal(t, 1, report.FreshUnreferencedAttachments) - require.Equal(t, 1, report.DeletedOrphanObjects) - require.Equal(t, 2, report.DeletedAttachmentObjects) - require.Equal(t, 3, report.DeletedUnreferencedRows) + require.Equal(t, 2, report.DeletedOrphanObjects) + require.Equal(t, 3, report.DeletedAttachmentObjects) + require.Equal(t, 4, report.DeletedUnreferencedRows) require.Contains(t, metadata.items, "attachments/b/original") require.Contains(t, metadata.items, "attachments/d/original") require.NotContains(t, metadata.items, "attachments/a/original") require.NotContains(t, metadata.items, "attachments/c/original") require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, metadata.items, "staging/pending/original") require.NotContains(t, objects.items, "attachments/orphan/original") + require.NotContains(t, objects.items, "staging/orphan/original") + require.NotContains(t, objects.items, "staging/pending/original") repeated, err := reconciler.Run(context.Background()) require.NoError(t, err) diff --git a/migrations/0006_async_attachment_pipeline.sql b/migrations/0006_async_attachment_pipeline.sql new file mode 100644 index 00000000..2b061b68 --- /dev/null +++ b/migrations/0006_async_attachment_pipeline.sql @@ -0,0 +1,135 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status, + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code TEXT, + ADD COLUMN staging_key TEXT; + +UPDATE attachments +SET staging_key = object_key +WHERE status = 'pending'; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK ( + status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting') + ), + ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1), + ADD CONSTRAINT attachments_error_code CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR + (status <> 'failed' AND error_code IS NULL) + ), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'ready' AND ready_at IS NOT NULL) OR + (status <> 'ready') + ), + ADD CONSTRAINT attachments_staging_key_length CHECK ( + staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024 + ); + +CREATE INDEX attachments_status_updated_at_idx + ON attachments (status, updated_at, id); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued', + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + lease_token UUID, + lease_owner TEXT, + lease_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_lease CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR + (status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL) + ), + CONSTRAINT attachment_jobs_error_code CHECK ( + error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (available_at, id) + WHERE status = 'queued'; + +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until, id) + WHERE status = 'processing'; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key), + CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE TABLE outbox_events ( + id UUID PRIMARY KEY, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload JSONB NOT NULL, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + publish_tries INTEGER NOT NULL DEFAULT 0, + claim_token UUID, + claim_until TIMESTAMPTZ, + last_error_code TEXT, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt), + CONSTRAINT outbox_events_attempt CHECK (attempt >= 1), + CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0), + CONSTRAINT outbox_events_claim CHECK ( + (claim_token IS NULL AND claim_until IS NULL) OR + (claim_token IS NOT NULL AND claim_until IS NOT NULL) + ), + CONSTRAINT outbox_events_last_error_code CHECK ( + last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX outbox_events_due_idx + ON outbox_events (available_at, id) + WHERE published_at IS NULL; + +CREATE TABLE attachment_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + previous_attempt INTEGER NOT NULL, + new_attempt INTEGER NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1), + CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128), + CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512) +); + +CREATE INDEX attachment_replay_audit_job_created_at_idx + ON attachment_replay_audit (job_id, created_at); -- 2.54.0 From bc090c0f8b4e9e4d462222808c819d2e2fbe7c5f Mon Sep 17 00:00:00 2001 From: dizi Date: Thu, 22 Oct 2026 17:22:32 +0300 Subject: [PATCH 03/11] test: verify image pipeline recovery --- Makefile | 8 +- REPORT.md | 191 +++++++- cmd/lab4experiment/main.go | 676 ----------------------------- cmd/lab5experiment/api_scenario.go | 298 +++++++++++++ cmd/lab5experiment/main.go | 605 ++++++++++++++++++++++++++ evidence/lab5-idempotency.json | 42 ++ evidence/lab5-resilience.json | 1 + scripts/test-lab5-acceptance.sh | 19 + scripts/test-lab5.sh | 12 + 9 files changed, 1152 insertions(+), 700 deletions(-) delete mode 100644 cmd/lab4experiment/main.go create mode 100644 cmd/lab5experiment/api_scenario.go create mode 100644 cmd/lab5experiment/main.go create mode 100644 evidence/lab5-idempotency.json create mode 100644 evidence/lab5-resilience.json create mode 100755 scripts/test-lab5-acceptance.sh create mode 100755 scripts/test-lab5.sh diff --git a/Makefile b/Makefile index 5d86d171..43e0a2a9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,9 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh diff --git a/REPORT.md b/REPORT.md index e98f75e9..a084953a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,182 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Пластинин Денис, БВТ2303. +- Целевая оценка: 5. +- Выбранный трек: 2, доказанная идемпотентность versioned pipeline. +- Ветка решения: `solution/bvt2303-plastinin`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Локальные пароли из `lab5.env.example` предназначены только для изолированного учебного стенда. -## Архитектура и решения +```sh +cp lab5.env.example .env +docker compose config --quiet +docker compose up -d --build --scale api=2 --wait --wait-timeout 180 +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +После запуска доступны: + +- API через Nginx: `http://localhost:8080`; +- Swagger UI: `http://localhost:8081`; +- Prometheus: `http://localhost:9090`; +- Grafana: `http://localhost:3000`, пользователь `admin`, пароль берётся из `GRAFANA_ADMIN_PASSWORD`. + +Интеграционные проверки PostgreSQL и Valkey запускаются отдельно: + +```sh +docker compose --profile tests run --rm --build tests +``` + +Эксперимент трека 5 временно останавливает обычные dispatcher и worker, запускает управляемые сбои, затем возвращает процессы в работу: + +```sh +make test-lab5 +``` + +Обязательный сценарий с остановленным worker и разрывом commit → publish: + +```sh +make test-lab5-acceptance +``` + +Обычная остановка сохраняет volumes: + +```sh +docker compose down +``` + +Намеренное удаление всех учебных данных: + +```sh +docker compose down -v +``` + +## Архитектура + +```mermaid +flowchart LR + Client --> Nginx + Nginx --> API1[API replica 1] + Nginx --> API2[API replica 2] + API1 --> PostgreSQL + API2 --> PostgreSQL + API1 --> Valkey[Valkey sessions] + API2 --> Valkey + API1 --> S3[(MinIO / S3)] + API2 --> S3 + PostgreSQL -->|outbox lease| Dispatcher + Dispatcher -->|XADD| Redis[(Redis Streams + AOF)] + Redis -->|XREADGROUP| Worker + Worker --> S3 + Worker -->|lease + result transaction| PostgreSQL + Worker -->|XACK after commit| Redis +``` + +PostgreSQL является источником истины для metadata, состояния job, попытки, lease, версий pipeline, вариантов и outbox. MinIO хранит исходник в staging и готовые объекты. Valkey хранит только сессии. Отдельный Redis используется как durable broker: включены AOF, `appendfsync always`, volume и политика `noeviction`. + +Temporal не используется. В работе один короткий тип workflow без ожидания внешних событий и компенсаций между множеством сервисов. Transactional outbox, job lease и Redis consumer group закрывают требуемые окна отказа с меньшим числом компонентов. Добавление Temporal потребовало бы Temporal Service и отдельного persistent store, при этом PostgreSQL job/outbox всё равно понадобился бы для атомарной связи с attachment metadata. + +## Граница ответа 202 + +API полностью читает и проверяет PNG/JPEG: MIME, фактический формат, размер и число пикселей. Затем создаётся `pending` metadata, а байты записываются в приватный ключ `staging//` в S3. После успешной записи одна транзакция PostgreSQL переводит attachment в `queued`, создаёт job и outbox event. Только после commit клиент получает `202 Accepted` со snapshot состояния `queued`. + +Если запись staging не удалась, `202` не возвращается. Если соединение потеряно после commit, сохранённый outbox остаётся источником восстановления. Бинарные данные, S3 ключи и credentials в Redis не передаются: envelope содержит только schema version, job ID, attachment ID, pipeline version, attempt, время и trace context. + +## Обработка изображений + +Worker повторно проверяет доверенный staging объект и создаёт четыре content-addressed объекта: + +| Вариант | Формат | Правило | +| --- | --- | --- | +| `original` | исходный PNG/JPEG | байты не изменяются | +| `thumbnail` | JPEG quality 85 | fit в `256×256`, без увеличения | +| `crop` | JPEG quality 85 | центральный квадрат `128×128`, увеличение разрешено | +| `optimized` | WebP quality 82, method 4 | fit в `1280×1280`, без увеличения | + +EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments//pipelines///.`, поэтому повтор той же операции записывает тот же объект. + +Concurrency и prefetch ограничены значениями `WORKER_CONCURRENCY` и `WORKER_PREFETCH`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `WORKER_MEMORY_BYTES` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. + +## Доставка, retry и восстановление + +Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. + +Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями. + +Временная ошибка переводит job обратно в `queued`, увеличивает `attempt` и в той же транзакции создаёт новый outbox event. Задержка растёт экспоненциально и получает детерминированный jitter до 25%. Постоянные ошибки (`invalid_image`, `image_too_large`, `source_missing`) и исчерпание пяти попыток переводят attachment в `failed`. Явный replay выполняется командой: + +```sh +docker compose run --rm api /app/messenger replay-attachment +``` + +Replay допустим только для `failed`, увеличивает attempt и сохраняет audit record. Повтор `ready` job становится no-op. `XAUTOCLAIM` возвращает зависшие Redis deliveries, а PostgreSQL recovery возвращает job с истёкшим lease в очередь. Старый consumer не может подтвердить delivery после передачи новому consumer: Lua script проверяет владельца PEL перед `XACK` или переносом в failed stream. + +## Эксперимент трека 5 + +### Гипотеза + +At-least-once доставка не создаст второй логический результат, если effect адресуется по `(attachment_id, pipeline_version, variant)`, а публикация metadata защищена `attempt` и lease token. Старая версия pipeline не сможет перезаписать текущую. + +### Методика + +Go-программа `cmd/lab5experiment` работает с настоящими PostgreSQL, Redis Streams и MinIO из Compose. Используется PNG `320×200`. Обычные worker и dispatcher на время прогона остановлены, чтобы порядок событий задавался экспериментом. + +Автоматически выполняются: + +1. crash после claim, но до side effect; +2. crash после записи всех S3 объектов, но до commit результата; +3. crash после commit результата, но до consumer ACK; +4. две дополнительные доставки уже завершённой job; +5. постоянная ошибка повреждённого staging и явный replay; +6. завершение pipeline v2, после которого доставляется устаревшая job v1. + +Для первых двух сценариев ожидается истечение lease, увеличение attempt и завершение следующей попыткой. В третьем и четвёртом сценариях повтор должен подтвердиться как no-op. Повреждённый staging дважды завершается одинаковым безопасным `invalid_image`, причём replay увеличивает attempt и проходит только через audit/outbox. После готовой версии v2 поздняя v1 не меняет metadata. + +### Результат + +Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidence/lab5-resilience.json`. + +| Воздействие | До восстановления | После восстановления | +| --- | --- | --- | +| crash до side effect | 0 конечных объектов | attempt 2, `ready`, 4 объекта | +| crash после side effect | 4 объекта, DB ещё `processing` | attempt 2, `ready`, те же 4 объекта | +| crash после commit до ACK | DB уже `ready`, 4 объекта | attempt 1, повтор no-op, 4 объекта | +| две duplicate deliveries | 4 объекта | 4 объекта | +| permanent failure и replay | `failed`, `invalid_image` | replay attempt 2, снова `failed`, `invalid_image` | +| stale v1 после готовой v2 | `ready` v2 | остаётся `ready` v2, 8 объектов для двух версий | + +Все пять подтверждённых logical jobs дошли до терминального состояния: четыре `ready`, одна ожидаемо `failed`. Инварианты `accepted_jobs_reach_terminal_state`, `duplicate_effect_is_unique` и `old_pipeline_cannot_replace_new` получили `true`. Прогон занял 6981 ms. + +Это не exactly-once доставка: Redis может доставить одно сообщение несколько раз, а dispatcher может повторить `XADD`. Exactly-once относится только к наблюдаемому эффекту, который достигается идемпотентными ключами и compare-and-set в PostgreSQL. + +## Мониторинг и CI + +Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard содержит RPS, status classes, p95 API latency, Go heap, PostgreSQL, Valkey, MinIO, throughput pipeline по результатам, p95 обработки, in-flight jobs, retry/reclaim, outbox publish failures и Redis broker memory/commands. + +CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда | Результат | +| --- | --- | --- | +| Публичный API lab5 | `make test` | PASS: queued → ready, original и thumbnail проверены | +| PostgreSQL и Valkey | `docker compose --profile tests run --rm --build tests` | оба пакета PASS | +| Transform | `go test ./internal/messenger/attachment/transform` | thumbnail/crop/WebP, EXIF, alpha и лимиты PASS | +| Redis и outbox | `go test ./internal/broker/redisstream ./internal/outbox` | envelope, poison, owner fencing и lost lease PASS | +| Worker остановлен, outbox gap, restart broker/API | `make test-lab5-acceptance` | queued link и 409 до worker; после restart `ready`, original сохранён | +| Трек 5 | `make test-lab5` | 5/5 jobs terminal, 4 ready, 1 expected failed, все три инварианта true | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Ограничения -## Эксперименты и отказы +Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Lease должен быть больше максимального времени обработки; для больших production изображений нужен heartbeat продления lease. Deadline проверяется между стадиями, но библиотечный вызов WebP не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Источники и библиотеки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Redis Streams и consumer groups: . +- Redis persistence: . +- PostgreSQL `SKIP LOCKED`: . +- MinIO Go SDK: . +- OpenTelemetry trace context: . +- Go-пакеты: `pgx/v5`, `go-redis/v9`, `minio-go/v7`, `deepteams/webp`, Prometheus client. diff --git a/cmd/lab4experiment/main.go b/cmd/lab4experiment/main.go deleted file mode 100644 index 9ce5d5a7..00000000 --- a/cmd/lab4experiment/main.go +++ /dev/null @@ -1,676 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "flag" - "fmt" - "image" - "image/jpeg" - "io" - "net/http" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" - - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" -) - -const experimentPassword = "Lab4-experiment-password" - -type apiClient struct { - baseURL string - client *http.Client - token string -} - -type attachment struct { - ID string `json:"id"` -} - -type setup struct { - client *apiClient - chatID string -} - -func main() { - if err := run(os.Args[1:], os.Stdout); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func run(arguments []string, output io.Writer) error { - if len(arguments) == 0 { - return errors.New("usage: lab4experiment memory|consistency [flags]") - } - switch arguments[0] { - case "memory": - return runMemory(arguments[1:], output) - case "consistency": - return runConsistency(arguments[1:], output) - default: - return errors.New("usage: lab4experiment memory|consistency [flags]") - } -} - -type memoryResult struct { - Replicas int `json:"replicas"` - Concurrency int `json:"concurrency"` - Rounds int `json:"rounds"` - SettleTimeMillis int64 `json:"settle_time_ms"` - ImageBytes int `json:"image_bytes"` - UploadRateBytes int64 `json:"upload_rate_bytes_per_second"` - SuccessfulUploads int64 `json:"successful_uploads"` - FailedUploads int64 `json:"failed_uploads"` - DurationMillis int64 `json:"duration_ms"` - HeapBaselineBytes int64 `json:"heap_baseline_bytes"` - HeapPeakBytes int64 `json:"heap_peak_bytes"` - HeapDeltaBytes int64 `json:"heap_delta_bytes"` - RSSBaselineBytes int64 `json:"rss_baseline_bytes"` - RSSPeakBytes int64 `json:"rss_peak_bytes"` - RSSDeltaBytes int64 `json:"rss_delta_bytes"` -} - -func runMemory(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("memory", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL") - replicas := flags.Int("replicas", 2, "expected API replicas") - concurrency := flags.Int("concurrency", 12, "number of concurrent uploads") - rounds := flags.Int("rounds", 4, "number of upload waves") - width := flags.Int("width", 2048, "image width") - height := flags.Int("height", 2048, "image height") - uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload") - settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid memory experiment flags") - } - if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 { - return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive") - } - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - fixture, err := jpegFixture(*width, *height) - if err != nil { - return err - } - if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil { - return err - } - settle := time.NewTimer(*settleTime) - defer settle.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-settle.C: - } - heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - return err - } - rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - return err - } - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - - var successful atomic.Int64 - var failed atomic.Int64 - var firstError struct { - sync.Mutex - err error - } - heapPeak := heapBaseline - rssPeak := rssBaseline - loadDone := make(chan struct{}) - sampleDone := make(chan error, 1) - go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone) - started := time.Now() - for round := range *rounds { - var workers sync.WaitGroup - workers.Add(*concurrency) - for index := range *concurrency { - go func(round, index int) { - defer workers.Done() - reader := newRateReader(fixture, *uploadRate) - filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil { - failed.Add(1) - firstError.Lock() - if firstError.err == nil { - firstError.err = err - } - firstError.Unlock() - return - } - successful.Add(1) - }(round, index) - } - workers.Wait() - } - close(loadDone) - if err := <-sampleDone; err != nil { - return err - } - result := memoryResult{ - Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(), - ImageBytes: len(fixture), UploadRateBytes: *uploadRate, - SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(), - HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline, - RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline, - } - if result.FailedUploads > 0 { - return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err) - } - return writeJSON(output, result) -} - -func sampleMemory( - ctx context.Context, - prometheusURL string, - expectedReplicas int, - done <-chan struct{}, - heapPeak *int64, - rssPeak *int64, - result chan<- error, -) { - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - result <- ctx.Err() - return - case <-done: - result <- nil - return - case <-ticker.C: - replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err != nil { - result <- err - return - } - if replicas != int64(expectedReplicas) { - result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas) - return - } - heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - result <- err - return - } - rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - result <- err - return - } - *heapPeak = max(*heapPeak, heap) - *rssPeak = max(*rssPeak, rss) - } - } -} - -type consistencyResult struct { - PageSize int `json:"page_size"` - SeededConsistent int `json:"seeded_consistent"` - FreshApply reconciliationRun `json:"fresh_apply"` - DryRun reconciliationRun `json:"dry_run"` - Apply reconciliationRun `json:"apply"` - RepeatedApply reconciliationRun `json:"repeated_apply"` - FreshPreserved bool `json:"fresh_preserved"` - OrphanDeleted bool `json:"orphan_deleted"` - UnusedDeleted bool `json:"unused_metadata_deleted"` - InUsePreserved bool `json:"in_use_metadata_preserved"` - AbandonedDeleted bool `json:"abandoned_attachment_deleted"` - TotalDurationMillis int64 `json:"total_duration_ms"` -} - -type reconciliationRun struct { - DurationMillis int64 `json:"duration_ms"` - Report reconcile.Report `json:"report"` -} - -func runConsistency(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("consistency", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - pageSize := flags.Int("page-size", 10, "reconciliation page size") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid consistency experiment flags") - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - settings, err := config.LoadReconciliation() - if err != nil { - return err - } - pool, err := database.Open(ctx, database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - objects, err := objectstore.New(objectstore.Config{ - Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, - AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, - }) - if err != nil { - return err - } - metadata := postgres.New(pool) - clean, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - initial, err := clean.Run(ctx) - if err != nil { - return err - } - if hasAnomalies(initial) { - return errors.New("consistency experiment requires a stack without existing attachment anomalies") - } - - started := time.Now() - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - fixture, err := jpegFixture(320, 200) - if err != nil { - return err - } - seededConsistent := *pageSize + 1 - for index := range seededConsistent { - filename := fmt.Sprintf("consistent-%03d.jpg", index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil { - return err - } - } - unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil { - return err - } - abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("prepare abandoned pending attachment: %w", err) - } - keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) - if err != nil { - return err - } - if err := objects.Delete(ctx, keys[unused.ID]); err != nil { - return err - } - if err := objects.Delete(ctx, keys[inUse.ID]); err != nil { - return err - } - orphanKey := "attachments/experiment-" + uuid.NewString() + "/original" - if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil { - return err - } - - fresh, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, - UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - freshRun, err := measuredReconciliation(ctx, fresh) - if err != nil { - return err - } - freshPreserved := objectExists(ctx, objects, orphanKey) && - attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) && - attachmentExists(ctx, pool, abandoned.ID) - if _, err := pool.Exec(ctx, ` - UPDATE attachments - SET created_at = clock_timestamp() - INTERVAL '2 hours' - WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("age abandoned attachment: %w", err) - } - - dryRun, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - dryRunResult, err := measuredReconciliation(ctx, dryRun) - if err != nil { - return err - } - apply, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - applyResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - repeatedResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - - result := consistencyResult{ - PageSize: *pageSize, SeededConsistent: seededConsistent, - FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, - FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), - UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), - AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), - TotalDurationMillis: time.Since(started).Milliseconds(), - } - if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || - repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 { - return fmt.Errorf("unexpected reconciliation result: %+v", result) - } - return writeJSON(output, result) -} - -func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) { - started := time.Now() - report, err := reconciler.Run(ctx) - return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err -} - -func hasAnomalies(report reconcile.Report) bool { - return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || - report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || - report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || - report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0 -} - -func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { - rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids) - if err != nil { - return nil, fmt.Errorf("query attachment keys: %w", err) - } - defer rows.Close() - keys := make(map[string]string, len(ids)) - for rows.Next() { - var id string - var key string - if err := rows.Scan(&id, &key); err != nil { - return nil, fmt.Errorf("scan attachment key: %w", err) - } - keys[id] = key - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate attachment keys: %w", err) - } - if len(keys) != len(ids) { - return nil, errors.New("not all attachment keys were found") - } - return keys, nil -} - -func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool { - var exists bool - return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists -} - -func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool { - _, err := objects.Stat(ctx, key) - return err == nil -} - -func prepare(ctx context.Context, baseURL string) (setup, error) { - client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}} - username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "") - var user struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{ - "username": username, "display_name": "Lab 4 experiment", "password": experimentPassword, - }, &user, ""); err != nil { - return setup{}, err - } - credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword)) - var session struct { - Token string `json:"token"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil { - return setup{}, err - } - client.token = session.Token - var chat struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{ - "title": "Lab 4 experiment", "member_ids": []string{user.ID}, - }, &chat, "Bearer "+client.token); err != nil { - return setup{}, err - } - return setup{client: client, chatID: chat.ID}, nil -} - -func (c *apiClient) upload( - ctx context.Context, - chatID string, - filename string, - body io.Reader, - contentLength int64, -) (attachment, error) { - path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename - request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body) - if err != nil { - return attachment{}, err - } - request.ContentLength = contentLength - request.Header.Set("Authorization", "Bearer "+c.token) - request.Header.Set("Content-Type", "image/jpeg") - response, err := c.client.Do(request) - if err != nil { - return attachment{}, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusCreated { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message) - } - var result attachment - if err := json.NewDecoder(response.Body).Decode(&result); err != nil { - return attachment{}, err - } - return result, nil -} - -func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error { - return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{ - "text": "", "attachment_ids": []string{attachmentID}, - }, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString()) -} - -func (c *apiClient) jsonRequest( - ctx context.Context, - method string, - path string, - payload any, - result any, - authorization string, - extraHeader ...string, -) error { - var body io.Reader - if payload != nil { - encoded, err := json.Marshal(payload) - if err != nil { - return err - } - body = bytes.NewReader(encoded) - } - request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) - if err != nil { - return err - } - if payload != nil { - request.Header.Set("Content-Type", "application/json") - } - if authorization != "" { - request.Header.Set("Authorization", authorization) - } - for index := 0; index+1 < len(extraHeader); index += 2 { - request.Header.Set(extraHeader[index], extraHeader[index+1]) - } - response, err := c.client.Do(request) - if err != nil { - return err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode < 200 || response.StatusCode >= 300 { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message) - } - if result == nil { - return nil - } - return json.NewDecoder(response.Body).Decode(result) -} - -func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err == nil && count == int64(expected) { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err()) - case <-ticker.C: - } - } -} - -func activeReplicasQuery() string { - return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))` -} - -func freshMetricQuery(metric string) string { - return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric) -} - -func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil) - if err != nil { - return 0, err - } - values := request.URL.Query() - values.Set("query", query) - request.URL.RawQuery = values.Encode() - response, err := http.DefaultClient.Do(request) - if err != nil { - return 0, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusOK { - return 0, fmt.Errorf("prometheus status %d", response.StatusCode) - } - var payload struct { - Status string `json:"status"` - Data struct { - Result []struct { - Value []json.RawMessage `json:"value"` - } `json:"result"` - } `json:"data"` - } - if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { - return 0, err - } - if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 { - return 0, errors.New("prometheus query returned no scalar value") - } - var raw string - if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil { - return 0, err - } - value, err := strconv.ParseFloat(raw, 64) - return int64(value), err -} - -func jpegFixture(width, height int) ([]byte, error) { - pixels := image.NewRGBA(image.Rect(0, 0, width, height)) - state := uint64(0x9e3779b97f4a7c15) - for index := range pixels.Pix { - state ^= state << 13 - state ^= state >> 7 - state ^= state << 17 - pixels.Pix[index] = byte(state >> 24) - } - var encoded bytes.Buffer - if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil { - return nil, err - } - return encoded.Bytes(), nil -} - -type rateReader struct { - reader *bytes.Reader - rate int64 - started time.Time - read int64 -} - -func newRateReader(data []byte, rate int64) *rateReader { - return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()} -} - -func (r *rateReader) Read(buffer []byte) (int, error) { - for { - allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read - if allowed > 0 { - if int64(len(buffer)) > allowed { - buffer = buffer[:allowed] - } - count, err := r.reader.Read(buffer) - r.read += int64(count) - return count, err - } - time.Sleep(10 * time.Millisecond) - } -} - -func writeJSON(output io.Writer, value any) error { - encoder := json.NewEncoder(output) - encoder.SetIndent("", " ") - return encoder.Encode(value) -} diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go new file mode 100644 index 00000000..5dc30720 --- /dev/null +++ b/cmd/lab5experiment/api_scenario.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type apiState struct { + BaseURL string `json:"base_url"` + Token string `json:"token"` + AttachmentID string `json:"attachment_id"` +} + +type apiUser struct { + ID string `json:"id"` +} + +type apiSession struct { + Token string `json:"token"` +} + +type apiChat struct { + ID string `json:"id"` +} + +func runAPIProduce(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-produce [-base-url URL]") + } + client := &http.Client{Timeout: 10 * time.Second} + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + password := "Lab5-" + uuid.NewString() + owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password) + if err != nil { + return err + } + token, err := createAPISession(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + var attachment model.Attachment + response, err := apiRequest( + client, http.MethodPost, + *baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png", + token, "image/png", fixture, &attachment, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued { + return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status) + } + messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}}) + if err != nil { + return err + } + response, err = apiRequestWithHeaders( + client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages", + token, "application/json", messageBody, nil, + map[string]string{"Idempotency-Key": uuid.NewString()}, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("link queued attachment returned %d", response.StatusCode) + } + response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil) + if err != nil { + return err + } + if response.StatusCode != http.StatusConflict { + return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) + } + return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) +} + +func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { + flags := flag.NewFlagSet("api-verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]") + } + var state apiState + if err := json.NewDecoder(input).Decode(&state); err != nil { + return fmt.Errorf("decode API scenario state: %w", err) + } + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) + defer cancel() + var attachment model.Attachment + for { + response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("attachment metadata returned %d", response.StatusCode) + } + if attachment.Status == model.AttachmentReady { + break + } + if attachment.Status == model.AttachmentFailed { + return fmt.Errorf("attachment failed with %v", attachment.ErrorCode) + } + if err := wait(ctx, 250*time.Millisecond); err != nil { + return err + } + } + if len(attachment.Variants) != 3 { + return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants)) + } + response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) { + return errors.New("original changed after worker and broker restart") + } + response, thumbnail, err := apiBytes( + client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" { + return errors.New("thumbnail response is invalid") + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail)) + if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 { + return errors.New("thumbnail dimensions are invalid") + } + return json.NewEncoder(output).Encode(map[string]any{ + "accepted_while_worker_stopped": true, + "accepted_before_outbox_publish": true, + "linked_while_queued": true, + "content_conflict_while_queued": true, + "ready_after_api_broker_restart": true, + "original_preserved": true, + "thumbnail_dimensions": "256x160", + }) +} + +func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": username, "password": password, + }) + if err != nil { + return apiUser{}, err + } + var user apiUser + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user) + if err != nil { + return apiUser{}, err + } + if response.StatusCode != http.StatusCreated { + return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode) + } + return user, nil +} + +func createAPISession(client *http.Client, baseURL, username, password string) (string, error) { + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Basic "+credentials) + response, err := client.Do(request) + if err != nil { + return "", err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create session returned %d", response.StatusCode) + } + var session apiSession + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return "", err + } + return session.Token, nil +} + +func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) { + body, err := json.Marshal(map[string]any{ + "title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID}, + }) + if err != nil { + return apiChat{}, err + } + var chat apiChat + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat) + if err != nil { + return apiChat{}, err + } + if response.StatusCode != http.StatusCreated { + return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode) + } + return chat, nil +} + +func apiRequest( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, +) (*http.Response, error) { + return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil) +} + +func apiRequestWithHeaders( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, + headers map[string]string, +) (*http.Response, error) { + request, err := http.NewRequest(method, requestURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + for name, value := range headers { + request.Header.Set(name, value) + } + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if result != nil && response.StatusCode < http.StatusBadRequest { + if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil { + return nil, err + } + } + return response, nil +} + +func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) { + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + return nil, nil, err + } + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + return response, body, err +} diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go new file mode 100644 index 00000000..b6532d6d --- /dev/null +++ b/cmd/lab5experiment/main.go @@ -0,0 +1,605 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +const ( + experimentLease = 2 * time.Second + maxAttempts = 5 +) + +type result struct { + StartedAt string `json:"started_at"` + DurationMillis int64 `json:"duration_ms"` + PipelineVersion int `json:"pipeline_version"` + FaultScenarios []scenario `json:"fault_scenarios"` + DuplicateDeliveries int `json:"duplicate_deliveries"` + V1ObjectCount int `json:"v1_object_count"` + V2ObjectCount int `json:"v2_object_count"` + AcceptedJobs int `json:"accepted_jobs"` + TerminalJobs int `json:"terminal_jobs"` + ReadyJobs int `json:"ready_jobs"` + FailedJobs int `json:"failed_jobs"` + PermanentErrorCode string `json:"permanent_error_code"` + ReplayAttempt int `json:"replay_attempt"` + InvariantResults map[string]bool `json:"invariant_results"` +} + +type scenario struct { + Name string `json:"name"` + AttemptAfterRecovery int `json:"attempt_after_recovery"` + ObjectsBeforeRecovery int `json:"objects_before_recovery"` + ObjectsAfterRecovery int `json:"objects_after_recovery"` + FinalStatus string `json:"final_status"` +} + +type experiment struct { + settings config.WorkerConfig + repository *postgres.Store + objects *objectstore.Store + broker *redisstream.Broker + dispatcher *outbox.Dispatcher + worker *worker.Worker + ownerID string + chatID string + fixture []byte +} + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(arguments []string, output io.Writer) error { + if len(arguments) > 0 { + switch arguments[0] { + case "api-produce": + return runAPIProduce(arguments[1:], output) + case "api-verify": + return runAPIVerify(arguments[1:], os.Stdin, output) + } + } + flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) + flags.SetOutput(io.Discard) + timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { + return errors.New("usage: lab5experiment [-timeout 2m]") + } + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + experiment, closeExperiment, err := newExperiment(ctx) + if err != nil { + return err + } + defer closeExperiment() + + report, err := experiment.run(ctx) + if err != nil { + return err + } + report.StartedAt = started.UTC().Format(time.RFC3339) + report.DurationMillis = time.Since(started).Milliseconds() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func newExperiment(ctx context.Context) (*experiment, func(), error) { + settings, err := config.LoadWorker() + if err != nil { + return nil, nil, err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return nil, nil, err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions, err := redis.ParseURL(settings.BrokerURL) + if err != nil { + pool.Close() + return nil, nil, err + } + client := redis.NewClient(redisOptions) + if err := client.Ping(ctx).Err(); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + consumer := "lab5experiment-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, + ClaimMinIdle: experimentLease, ReadCount: 32, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + if err := broker.Ensure(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + repository := postgres.New(pool) + dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ + BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, + RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processorWorker, err := worker.New( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, + worker.Config{ + WorkerID: consumer, Concurrency: 1, + LeaseDuration: experimentLease, ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + fixture, err := pngFixture(320, 200) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + value := &experiment{ + settings: settings, repository: repository, objects: objects, broker: broker, + dispatcher: dispatcher, worker: processorWorker, fixture: fixture, + } + if err := value.seed(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + closeExperiment := func() { + pool.Close() + _ = client.Close() + } + return value, closeExperiment, nil +} + +func (e *experiment) seed(ctx context.Context) error { + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") + if err != nil { + return err + } + member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") + if err != nil { + return err + } + chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) + if err != nil { + return err + } + e.ownerID = owner.ID + e.chatID = chat.ID + return nil +} + +func (e *experiment) run(ctx context.Context) (result, error) { + report := result{ + PipelineVersion: 1, + InvariantResults: map[string]bool{ + "accepted_jobs_reach_terminal_state": false, + "duplicate_effect_is_unique": false, + "old_pipeline_cannot_replace_new": false, + }, + } + for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { + observed, err := e.runLeaseRecovery(ctx, fault) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, observed) + report.AcceptedJobs++ + if observed.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + } + permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) + if err != nil { + return result{}, err + } + report.AcceptedJobs++ + report.TerminalJobs++ + report.FailedJobs++ + report.PermanentErrorCode = permanentCode + report.ReplayAttempt = replayAttempt + + commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, commitScenario) + report.AcceptedJobs++ + if commitScenario.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + + payload, err := redisstream.EncodeEnvelope(envelope) + if err != nil { + return result{}, err + } + for range 2 { + if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { + return result{}, err + } + } + duplicateDeliveries, err := e.readDeliveries(ctx, 2) + if err != nil { + return result{}, err + } + for _, delivery := range duplicateDeliveries { + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return result{}, err + } + } + report.DuplicateDeliveries = len(duplicateDeliveries) + report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V1ObjectCount != 4 { + return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) + } + report.InvariantResults["duplicate_effect_is_unique"] = true + + oldPayload := payload + v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} + if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { + return result{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return result{}, err + } + v2Delivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { + return result{}, err + } + if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { + return result{}, err + } + staleDelivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { + return result{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return result{}, err + } + if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { + return result{}, errors.New("new pipeline version was not preserved after stale delivery") + } + report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V2ObjectCount != 8 { + return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) + } + report.InvariantResults["old_pipeline_cannot_replace_new"] = true + report.AcceptedJobs++ + report.ReadyJobs++ + report.TerminalJobs++ + report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs + return report, nil +} + +func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { + attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") + if err != nil { + return "", 0, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { + return "", 0, errors.New("corrupted staging object did not reach failed") + } + firstCode := *metadata.ErrorCode + replayed, err := e.repository.ReplayFailedAttachment( + ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", + ) + if err != nil { + return "", 0, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", 0, err + } + replayDelivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { + return "", 0, errors.New("replayed permanent failure changed terminal classification") + } + return firstCode, replayed.Attempt, nil +} + +func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + if fault == worker.FaultBeforeSideEffect && before != 0 { + return scenario{}, errors.New("objects were written before the injected side effect") + } + if fault == worker.FaultAfterSideEffect && before != 4 { + return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, err + } + recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) + if err != nil { + return scenario{}, err + } + if recovered.Requeued != 1 { + return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return scenario{}, err + } + retry, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { + return scenario{}, err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) + if err != nil { + return scenario{}, err + } + return scenario{ + Name: string(fault), AttemptAfterRecovery: job.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, nil +} + +func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + return scenario{ + Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, delivery.Envelope, attachmentID, nil +} + +func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { + attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") + return attachmentID, job, err +} + +func (e *experiment) queueAttachmentData( + ctx context.Context, + data []byte, + contentType string, +) (string, string, model.AttachmentJob, error) { + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/experiment" + digest := sha256.Sum256(data) + _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, + }) + if err != nil { + return "", "", model.AttachmentJob{}, err + } + if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { + return "", "", model.AttachmentJob{}, err + } + job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} + if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { + return "", "", model.AttachmentJob{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", "", model.AttachmentJob{}, err + } + return attachmentID, stagingKey, job, nil +} + +func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { + deliveries, err := e.readDeliveries(ctx, 1) + if err != nil { + return redisstream.Delivery{}, err + } + return deliveries[0], nil +} + +func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { + var result []redisstream.Delivery + for len(result) < expected { + deliveries, err := e.broker.Read(ctx) + if err != nil { + return nil, err + } + result = append(result, deliveries...) + } + if len(result) != expected { + return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) + } + return result, nil +} + +func (e *experiment) ackReclaimed(ctx context.Context) error { + deliveries, _, err := e.broker.Reclaim(ctx, "0-0") + if err != nil { + return err + } + if len(deliveries) != 1 { + return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) + } + return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) +} + +func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { + objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) + if err != nil { + return 0, err + } + if next != "" { + return 0, errors.New("unexpected object pagination") + } + return len(objects), nil +} + +func pngFixture(width, height int) ([]byte, error) { + value := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, value); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func wait(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/evidence/lab5-idempotency.json b/evidence/lab5-idempotency.json new file mode 100644 index 00000000..c6e9050c --- /dev/null +++ b/evidence/lab5-idempotency.json @@ -0,0 +1,42 @@ +{ + "started_at": "2026-10-22T14:19:25Z", + "duration_ms": 6981, + "pipeline_version": 1, + "fault_scenarios": [ + { + "name": "before_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 0, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_commit", + "attempt_after_recovery": 1, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + } + ], + "duplicate_deliveries": 2, + "v1_object_count": 4, + "v2_object_count": 8, + "accepted_jobs": 5, + "terminal_jobs": 5, + "ready_jobs": 4, + "failed_jobs": 1, + "permanent_error_code": "invalid_image", + "replay_attempt": 2, + "invariant_results": { + "accepted_jobs_reach_terminal_state": true, + "duplicate_effect_is_unique": true, + "old_pipeline_cannot_replace_new": true + } +} diff --git a/evidence/lab5-resilience.json b/evidence/lab5-resilience.json new file mode 100644 index 00000000..c82d1860 --- /dev/null +++ b/evidence/lab5-resilience.json @@ -0,0 +1 @@ +{"accepted_before_outbox_publish":true,"accepted_while_worker_stopped":true,"content_conflict_while_queued":true,"linked_while_queued":true,"original_preserved":true,"ready_after_api_broker_restart":true,"thumbnail_dimensions":"256x160"} diff --git a/scripts/test-lab5-acceptance.sh b/scripts/test-lab5-acceptance.sh new file mode 100755 index 00000000..f62397cc --- /dev/null +++ b/scripts/test-lab5-acceptance.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + +state_file=$(mktemp) + +restore_pipeline() { + rm -f "$state_file" + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM + +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file" +docker compose restart api redis-broker >/dev/null +docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null +docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file" + diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 00000000..9bf4511c --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -eu + +restore_pipeline() { + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment + -- 2.54.0 From 487015dd2f603ad2941499a62856a5d790d28147 Mon Sep 17 00:00:00 2001 From: dizi Date: Thu, 22 Oct 2026 17:22:44 +0300 Subject: [PATCH 04/11] ops: run and monitor Redis Streams workers --- Dockerfile | 4 +- compose.yaml | 191 +++++ deploy/compose.images.yaml | 6 + lab5.env.example | 32 + monitoring/grafana/dashboards/messenger.json | 719 ++++++++++++++++--- monitoring/prometheus.yml | 15 + scripts/restart.sh | 4 +- scripts/rolling-update.sh | 4 +- 8 files changed, 857 insertions(+), 118 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff27469a..efd6b0d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . RUN go build -p=1 -o /out/messenger ./cmd/messenger -RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment +RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment FROM build AS test @@ -35,7 +35,7 @@ FROM alpine:3.24.2 RUN addgroup -S app && adduser -S -G app -u 10001 app COPY --from=build --chown=app:app /out/messenger /app/messenger -COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment +COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment USER app WORKDIR /app diff --git a/compose.yaml b/compose.yaml index f1c370aa..b2137b4f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -34,6 +34,32 @@ services: restart: unless-stopped networks: [data] + redis-broker: + image: redis:8.2.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - always + - --maxmemory-policy + - noeviction + - --requirepass + - ${REDIS_BROKER_PASSWORD} + environment: + REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD} + volumes: + - redis_broker_data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${REDIS_BROKER_CPUS} + mem_limit: ${REDIS_BROKER_MEMORY} + restart: unless-stopped + networks: [data] + s3: build: context: . @@ -147,6 +173,7 @@ services: ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES} ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} + PIPELINE_VERSION: ${PIPELINE_VERSION} expose: - "8080" depends_on: @@ -170,6 +197,103 @@ services: restart: unless-stopped networks: [application, data] + dispatcher: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/messenger", "dispatch"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + BROKER_URL: ${BROKER_URL} + BROKER_STREAM: ${BROKER_STREAM} + BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} + BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} + BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} + OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE} + OUTBOX_LEASE_SECONDS: ${OUTBOX_LEASE_SECONDS} + OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS} + OUTBOX_RECOVERY_SECONDS: ${OUTBOX_RECOVERY_SECONDS} + OUTBOX_RETRY_SECONDS: ${OUTBOX_RETRY_SECONDS} + PIPELINE_METRICS_HOST: 0.0.0.0 + PIPELINE_METRICS_PORT: 9091 + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${DISPATCHER_CPUS} + mem_limit: ${DISPATCHER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/messenger", "worker"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_USE_TLS: ${S3_USE_TLS} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} + BROKER_URL: ${BROKER_URL} + BROKER_STREAM: ${BROKER_STREAM} + BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} + BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} + BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY} + WORKER_PREFETCH: ${WORKER_PREFETCH} + WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS} + WORKER_PROCESSING_TIMEOUT_SECONDS: ${WORKER_PROCESSING_TIMEOUT_SECONDS} + WORKER_READ_BLOCK_SECONDS: ${WORKER_READ_BLOCK_SECONDS} + WORKER_RECLAIM_SECONDS: ${WORKER_RECLAIM_SECONDS} + WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} + WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS} + WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS} + ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} + ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} + WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} + PIPELINE_METRICS_HOST: 0.0.0.0 + PIPELINE_METRICS_PORT: 9091 + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${WORKER_CPUS} + mem_limit: ${WORKER_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [data, monitoring] + nginx: build: context: . @@ -222,6 +346,55 @@ services: mem_limit: ${TEST_MEMORY} networks: [data] + lab5-experiment: + profiles: ["experiments"] + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/lab5experiment"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_USE_TLS: ${S3_USE_TLS} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} + BROKER_URL: ${BROKER_URL} + BROKER_STREAM: ${BROKER_STREAM} + BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} + BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} + BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} + WORKER_CONCURRENCY: 1 + WORKER_PREFETCH: 32 + WORKER_LEASE_SECONDS: 2 + WORKER_PROCESSING_TIMEOUT_SECONDS: 1 + WORKER_READ_BLOCK_SECONDS: 1 + WORKER_RECLAIM_SECONDS: 2 + WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} + WORKER_RETRY_BASE_SECONDS: 1 + WORKER_RETRY_MAX_SECONDS: 4 + ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} + ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} + WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} + PIPELINE_METRICS_HOST: 0.0.0.0 + PIPELINE_METRICS_PORT: 9091 + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [application, data] + postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 environment: @@ -251,6 +424,20 @@ services: restart: unless-stopped networks: [data, monitoring] + redis-broker-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://redis-broker:6379 + REDIS_USER: default + REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + depends_on: + redis-broker: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + nginx-exporter: image: nginx/nginx-prometheus-exporter:1.5.1 command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] @@ -285,6 +472,9 @@ services: - nginx-exporter - postgres-exporter - valkey-exporter + - redis-broker-exporter + - dispatcher + - worker - s3 cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} @@ -314,6 +504,7 @@ services: volumes: postgres_data: valkey_data: + redis_broker_data: prometheus_data: grafana_data: s3_data: diff --git a/deploy/compose.images.yaml b/deploy/compose.images.yaml index 4251b210..7d983198 100644 --- a/deploy/compose.images.yaml +++ b/deploy/compose.images.yaml @@ -5,6 +5,12 @@ services: migrate: image: ${API_IMAGE:?API_IMAGE is required} + dispatcher: + image: ${API_IMAGE:?API_IMAGE is required} + + worker: + image: ${API_IMAGE:?API_IMAGE is required} + nginx: image: ${NGINX_IMAGE:?NGINX_IMAGE is required} diff --git a/lab5.env.example b/lab5.env.example index 5ad34dcb..c9d7b28f 100644 --- a/lab5.env.example +++ b/lab5.env.example @@ -26,6 +26,32 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge VALKEY_PASSWORD=local-valkey-password CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 +# Redis Streams используется только как устойчивый брокер заданий. +REDIS_BROKER_PASSWORD=local-redis-broker-password +BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 +BROKER_STREAM=attachment-jobs +BROKER_CONSUMER_GROUP=attachment-workers +BROKER_FAILED_STREAM=attachment-jobs-failed +BROKER_FAILED_MAX_LENGTH=10000 + +# Версия pipeline, outbox и ограниченные повторы worker. +PIPELINE_VERSION=1 +OUTBOX_BATCH_SIZE=32 +OUTBOX_LEASE_SECONDS=15 +OUTBOX_POLL_SECONDS=1 +OUTBOX_RECOVERY_SECONDS=10 +OUTBOX_RETRY_SECONDS=2 +WORKER_CONCURRENCY=2 +WORKER_PREFETCH=4 +WORKER_LEASE_SECONDS=30 +WORKER_PROCESSING_TIMEOUT_SECONDS=20 +WORKER_READ_BLOCK_SECONDS=2 +WORKER_RECLAIM_SECONDS=5 +WORKER_MAX_ATTEMPTS=5 +WORKER_RETRY_BASE_SECONDS=1 +WORKER_RETRY_MAX_SECONDS=30 +WORKER_MEMORY_BYTES=738197504 + # Административные данные MinIO используются только при bootstrap. MINIO_ROOT_USER=local-minio-admin MINIO_ROOT_PASSWORD=local-minio-admin-password @@ -91,6 +117,12 @@ POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 VALKEY_MEMORY=256m +REDIS_BROKER_CPUS=1 +REDIS_BROKER_MEMORY=256m +DISPATCHER_CPUS=0.5 +DISPATCHER_MEMORY=128m +WORKER_CPUS=2 +WORKER_MEMORY=1g S3_CPUS=1 S3_MEMORY=512m S3_BOOTSTRAP_CPUS=0.25 diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json index 0478e84b..a4d0a5b0 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger.json @@ -1,143 +1,638 @@ { - "annotations": {"list": []}, + "annotations": { + "list": [] + }, "editable": false, "graphTooltip": 1, "panels": [ { - "type": "timeseries", "title": "API requests per second", "id": 1, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, - "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API RPS by replica", "id": 2, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, - "targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API responses by status class", "id": 3, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, - "targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Healthy API replicas", "id": 4, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, - "targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API latency p95 by replica and route", "id": 5, - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, - "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, - "targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API in flight", "id": 6, - "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, - "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Go heap by replica", "id": 7, - "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Idempotency replays", "id": 8, - "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, - "targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Nginx connections and requests", "id": 9, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, + "type": "timeseries", + "title": "API requests per second", + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, "targets": [ - {"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"}, - {"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"} + { + "expr": "sum(rate(messenger_http_requests_total[1m]))", + "legendFormat": "total", + "refId": "A" + } ] }, { - "type": "timeseries", "title": "PostgreSQL connections", "id": 10, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, - "targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}] - }, - { - "type": "timeseries", "title": "PostgreSQL transactions", "id": 11, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, + "type": "timeseries", + "title": "API RPS by replica", + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, "targets": [ - {"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"}, - {"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"} + { + "expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } ] }, { - "type": "timeseries", "title": "PostgreSQL database size", "id": 12, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Valkey memory", "id": 13, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Valkey keys and commands", "id": 14, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, + "type": "timeseries", + "title": "API responses by status class", + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, "targets": [ - {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, - {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} + { + "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "A" + } ] }, { - "type": "timeseries", "title": "Attachment uploads by result", "id": 15, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 56}, - "targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Attachment upload latency p95", "id": 16, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56}, - "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, - "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Attachment upload traffic", "id": 17, - "gridPos": {"h": 8, "w": 8, "x": 0, "y": 64}, - "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}, - "targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}] - }, - { - "type": "timeseries", "title": "MinIO objects", "id": 18, - "gridPos": {"h": 8, "w": 8, "x": 8, "y": 64}, - "targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}] - }, - { - "type": "timeseries", "title": "MinIO requests and errors", "id": 19, - "gridPos": {"h": 8, "w": 8, "x": 16, "y": 64}, + "type": "timeseries", + "title": "Healthy API replicas", + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, "targets": [ - {"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"}, - {"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"} + { + "expr": "sum(up{job=\"messenger\"})", + "legendFormat": "replicas", + "refId": "A" + } ] }, { - "type": "timeseries", "title": "MinIO availability", "id": 20, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 72}, - "targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}] + "type": "timeseries", + "title": "API latency p95 by replica and route", + "id": 5, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "{{instance}} {{route}}", + "refId": "A" + } + ] }, { - "type": "timeseries", "title": "MinIO storage", "id": 21, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 72}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "type": "timeseries", + "title": "API in flight", + "id": 6, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, "targets": [ - {"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"}, - {"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"} + { + "expr": "messenger_http_requests_in_flight", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Go heap by replica", + "id": 7, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Idempotency replays", + "id": 8, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "targets": [ + { + "expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", + "legendFormat": "replays/s", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Nginx connections and requests", + "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "expr": "nginx_connections_active", + "legendFormat": "active connections", + "refId": "A" + }, + { + "expr": "rate(nginx_http_requests_total[1m])", + "legendFormat": "requests/s", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL connections", + "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "targets": [ + { + "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", + "legendFormat": "connections", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL transactions", + "id": 11, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "targets": [ + { + "expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", + "legendFormat": "commit/s", + "refId": "A" + }, + { + "expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", + "legendFormat": "rollback/s", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL database size", + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname=\"messenger\"}", + "legendFormat": "database", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Valkey memory", + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"valkey\"}", + "legendFormat": "used", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Valkey keys and commands", + "id": 14, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "expr": "sum(redis_db_keys{job=\"valkey\"})", + "legendFormat": "keys", + "refId": "A" + }, + { + "expr": "rate(redis_commands_processed_total{job=\"valkey\"}[1m])", + "legendFormat": "commands/s", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment uploads by result", + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment upload latency p95", + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment upload traffic", + "id": 17, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", + "legendFormat": "bytes/s", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "MinIO objects", + "id": 18, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 64 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_object_total)", + "legendFormat": "objects", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "MinIO requests and errors", + "id": 19, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 64 + }, + "targets": [ + { + "expr": "sum(rate(minio_s3_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "A" + }, + { + "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", + "legendFormat": "errors/s", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "MinIO availability", + "id": 20, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "targets": [ + { + "expr": "up{job=\"minio\"}", + "legendFormat": "up", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "MinIO storage", + "id": 21, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_total_bytes)", + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(minio_cluster_capacity_usable_free_bytes)", + "legendFormat": "free", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment pipeline throughput", + "id": 22, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment worker latency and concurrency", + "id": 23, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "in flight", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Attachment retries and reclaimed deliveries", + "id": 24, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", + "legendFormat": "retries", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", + "legendFormat": "reclaimed", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Transactional outbox", + "id": 25, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(messenger_outbox_published_total[1m]))", + "legendFormat": "published", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", + "legendFormat": "publish failures", + "refId": "B" + }, + { + "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", + "legendFormat": "recovered leases", + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "title": "Redis broker memory", + "id": 26, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", + "legendFormat": "used", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Redis broker commands and keys", + "id": 27, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "targets": [ + { + "expr": "rate(redis_commands_processed_total{job=\"redis-broker\"}[1m])", + "legendFormat": "commands", + "refId": "A" + }, + { + "expr": "sum(redis_db_keys{job=\"redis-broker\"})", + "legendFormat": "keys", + "refId": "B" + } ] } ], "refresh": "5s", "schemaVersion": 41, - "tags": ["highload", "lab4", "nginx", "failover"], - "templating": {"list": []}, - "time": {"from": "now-15m", "to": "now"}, + "tags": [ + "highload", + "lab5", + "nginx", + "failover" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, "timezone": "browser", - "title": "Messenger lab4 overview", - "uid": "messenger-overview", - "version": 3 + "title": "Messenger lab5 overview", + "uid": "messenger-lab5", + "version": 2 } diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 83721c48..1869184e 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -22,6 +22,21 @@ scrape_configs: static_configs: - targets: ["valkey-exporter:9121"] + - job_name: redis-broker + static_configs: + - targets: ["redis-broker-exporter:9121"] + + - job_name: attachment-worker + dns_sd_configs: + - names: ["worker"] + type: A + port: 9091 + refresh_interval: 2s + + - job_name: outbox-dispatcher + static_configs: + - targets: ["dispatcher:9091"] + - job_name: minio metrics_path: /minio/v2/metrics/cluster static_configs: diff --git a/scripts/restart.sh b/scripts/restart.sh index a724583d..d5872516 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,8 +2,8 @@ set -eu -docker compose stop nginx api valkey postgres s3 -docker compose start postgres valkey s3 +docker compose stop nginx api worker dispatcher redis-broker valkey postgres s3 +docker compose start postgres valkey redis-broker s3 docker compose run --rm s3-bootstrap docker compose run --rm migrate docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index d02dc02d..695e3f8f 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -150,7 +150,7 @@ fi cp "$upstream_file" "$upstream_backup" compose pull api nginx swagger prometheus grafana s3 s3-bootstrap -compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3 +compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 compose run --rm --no-deps s3-bootstrap compose run --rm --no-deps migrate @@ -208,7 +208,7 @@ for container in $candidate_pool; do done smoke_check -compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana +compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana printf '%s\n' "$candidate_pool" > "$active_containers" cp "$manifest" "$current_manifest" -- 2.54.0 From 38351061258a82331d3d599bb55dcd32ef1bb0f9 Mon Sep 17 00:00:00 2001 From: dizi Date: Thu, 22 Oct 2026 17:22:48 +0300 Subject: [PATCH 05/11] ci: verify and deploy lab5 --- .gitea/workflows/ci.yml | 4 +++- .gitea/workflows/deploy.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8e0d5d56..f2d5cad4 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh - name: Lint uses: golangci/golangci-lint-action@v7 @@ -80,6 +80,8 @@ jobs: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 docker compose up -d --scale api=2 --wait --wait-timeout 90 + make test-lab5-acceptance + make test-lab5 - name: Container logs if: failure() diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 36d9be7e..54da2be0 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -60,7 +60,7 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh - name: Lint uses: golangci/golangci-lint-action@v7 @@ -93,6 +93,8 @@ jobs: go run ./cmd/dependencycheck -base-url=http://nginx:8080 make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + make test-lab5-acceptance + make test-lab5 - name: Publish verified images id: publish -- 2.54.0 From aa5d6a6e163a0940d4e6c40482c18d6f998eb2a3 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 13:45:49 +0300 Subject: [PATCH 06/11] fix: enforce pipeline lifecycle limits --- cmd/lab5experiment/main.go | 8 +- compose.yaml | 124 +++-------------- config/messenger.yaml | 76 +++++++++++ internal/bootstrap/pipeline.go | 7 +- internal/config/config.go | 112 +++++++++------- internal/config/config_test.go | 126 ++++++++++++++++-- internal/config/loader.go | 47 +++++++ internal/config/pipeline.go | 110 +++++++++------ .../messenger/attachment/transform/exif.go | 37 ++++- .../attachment/transform/processor.go | 59 ++++++-- .../attachment/transform/processor_test.go | 27 +++- .../messenger/attachment/transform/resize.go | 37 ++++- .../messenger/attachment/worker/worker.go | 64 ++++++++- .../attachment/worker/worker_test.go | 57 +++++++- internal/outbox/dispatcher.go | 29 ++++ internal/outbox/dispatcher_test.go | 42 ++++-- internal/postgres/attachment_lease.go | 32 +++++ internal/postgres/outbox.go | 23 ++++ lab5.env.example | 58 +------- migrations/0007_outbox_retention.sql | 3 + 20 files changed, 770 insertions(+), 308 deletions(-) create mode 100644 config/messenger.yaml create mode 100644 internal/config/loader.go create mode 100644 internal/postgres/attachment_lease.go create mode 100644 migrations/0007_outbox_retention.sql diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go index b6532d6d..3749235f 100644 --- a/cmd/lab5experiment/main.go +++ b/cmd/lab5experiment/main.go @@ -169,6 +169,7 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, }) if err != nil { pool.Close() @@ -185,9 +186,10 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, worker.Config{ WorkerID: consumer, Concurrency: 1, - LeaseDuration: experimentLease, ProcessingTimeout: time.Second, - ReclaimInterval: experimentLease, - MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, + ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, MemoryBytes: settings.MemoryBytes, }, diff --git a/compose.yaml b/compose.yaml index b2137b4f..48ad2db0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,9 @@ +x-messenger-config-volume: &messenger-config-volume + type: bind + source: ./config/messenger.yaml + target: /etc/messenger/messenger.yaml + read_only: true + services: postgres: image: postgres:17.6-alpine3.22 @@ -115,14 +121,10 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "migrate"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} + volumes: + - *messenger-config-volume depends_on: postgres: condition: service_healthy @@ -137,43 +139,19 @@ services: build: . image: ${API_IMAGE:-mtusi-messenger:lab5} environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} DATABASE_URL: ${DATABASE_URL} CACHE_URL: ${CACHE_URL} - SESSION_STORE: ${SESSION_STORE} - SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} - IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} - HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} - HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} - HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} - HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} - HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} - PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} - PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} - PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} - PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} - PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} SWAGGER_PORT: ${SWAGGER_PORT} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES} - ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} - PIPELINE_VERSION: ${PIPELINE_VERSION} + volumes: + - *messenger-config-volume expose: - "8080" depends_on: @@ -201,25 +179,11 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "dispatch"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE} - OUTBOX_LEASE_SECONDS: ${OUTBOX_LEASE_SECONDS} - OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS} - OUTBOX_RECOVERY_SECONDS: ${OUTBOX_RECOVERY_SECONDS} - OUTBOX_RETRY_SECONDS: ${OUTBOX_RETRY_SECONDS} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume expose: - "9091" depends_on: @@ -241,39 +205,16 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "worker"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - WORKER_CONCURRENCY: ${WORKER_CONCURRENCY} - WORKER_PREFETCH: ${WORKER_PREFETCH} - WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS} - WORKER_PROCESSING_TIMEOUT_SECONDS: ${WORKER_PROCESSING_TIMEOUT_SECONDS} - WORKER_READ_BLOCK_SECONDS: ${WORKER_READ_BLOCK_SECONDS} - WORKER_RECLAIM_SECONDS: ${WORKER_RECLAIM_SECONDS} - WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} - WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS} - WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS} - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume expose: - "9091" depends_on: @@ -351,39 +292,16 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/lab5experiment"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - WORKER_CONCURRENCY: 1 - WORKER_PREFETCH: 32 - WORKER_LEASE_SECONDS: 2 - WORKER_PROCESSING_TIMEOUT_SECONDS: 1 - WORKER_READ_BLOCK_SECONDS: 1 - WORKER_RECLAIM_SECONDS: 2 - WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} - WORKER_RETRY_BASE_SECONDS: 1 - WORKER_RETRY_MAX_SECONDS: 4 - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume depends_on: migrate: condition: service_completed_successfully diff --git a/config/messenger.yaml b/config/messenger.yaml new file mode 100644 index 00000000..031e1522 --- /dev/null +++ b/config/messenger.yaml @@ -0,0 +1,76 @@ +version: 1 + +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 + +database: + min_connections: 1 + max_connections: 10 + max_connection_age: 30m + max_connection_idle: 5m + health_check_period: 30s + dependency_timeout: 5s + +migration: + timeout: 30s + +object_storage: + use_tls: false + timeout: 5s + +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 + +pipeline: + version: 1 + +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 + +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 + +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 + +monitoring: + metrics_address: 0.0.0.0:9091 diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index 2f794c50..97ce3ec6 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -55,7 +55,8 @@ func RunDispatcher(logger *slog.Logger) error { dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, - RetryDelay: settings.RetryDelay, + RetryDelay: settings.RetryDelay, Retention: settings.Retention, + CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize, }) if err != nil { return err @@ -112,8 +113,8 @@ func RunWorker(logger *slog.Logger) error { logger, postgres.New(pool), broker, objects, processor, metrics, worker.Config{ WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, - ProcessingTimeout: settings.ProcessingTimeout, - ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, MemoryBytes: settings.MemoryBytes, diff --git a/internal/config/config.go b/internal/config/config.go index b27025c5..977abc54 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -98,14 +98,16 @@ type objectStorageConfig struct { } func LoadServer() (ServerConfig, error) { - values := viper.New() - values.AutomaticEnv() - - host, err := requiredString(values, "APP_HOST") + values, err := loadValues() if err != nil { return ServerConfig{}, err } - port, err := requiredPort(values, "APP_PORT") + + host, err := requiredString(values, "server.host") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "server.port") if err != nil { return ServerConfig{}, err } @@ -113,71 +115,71 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - cacheURL, err := requiredString(values, "CACHE_URL") + cacheURL, err := requiredString(values, "cache.url") if err != nil { return ServerConfig{}, err } - store, err := requiredString(values, "SESSION_STORE") + store, err := requiredString(values, "server.session_store") if err != nil { return ServerConfig{}, err } if store != "valkey" { - return ServerConfig{}, errors.New("SESSION_STORE must be valkey") + return ServerConfig{}, errors.New("server.session_store must be valkey") } - sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30) + sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) + idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300) + requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300) + readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300) + readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300) + writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600) + idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour) if err != nil { return ServerConfig{}, err } - shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300) + shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024) + passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024) if err != nil { return ServerConfig{}, err } - passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20) + passwordIterations, err := integer(values, "server.password.iterations", 1, 20) if err != nil { return ServerConfig{}, err } - passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255) + passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255) if err != nil { return ServerConfig{}, err } - passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024) + passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024) if err != nil { return ServerConfig{}, err } - passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024) + passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024) if err != nil { return ServerConfig{}, err } - swaggerPort, err := requiredPort(values, "SWAGGER_PORT") + swaggerPort, err := requiredPort(values, "server.swagger_port") if err != nil { return ServerConfig{}, err } @@ -185,30 +187,30 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) if err != nil { return ServerConfig{}, err } - attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) if err != nil { return ServerConfig{}, err } - attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes) + return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes) } - attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } minimumDecodeMemory := int64(attachmentMaxPixels) * 8 if int64(attachmentDecodeMemory) < minimumDecodeMemory { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory) } - pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000) + pipelineVersion, err := integer(values, "pipeline.version", 1, 1000) if err != nil { return ServerConfig{}, err } @@ -236,13 +238,15 @@ func LoadServer() (ServerConfig, error) { } func LoadMigration() (MigrationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return MigrationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return MigrationConfig{}, err } - migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute) if err != nil { return MigrationConfig{}, err } @@ -255,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) { } func LoadReconciliation() (ReconciliationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return ReconciliationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return ReconciliationConfig{}, err @@ -276,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) { } func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { - endpoint, err := requiredString(values, "S3_ENDPOINT") + endpoint, err := requiredString(values, "object_storage.endpoint") if err != nil { return objectStorageConfig{}, err } - region, err := requiredString(values, "S3_REGION") + region, err := requiredString(values, "object_storage.region") if err != nil { return objectStorageConfig{}, err } - bucket, err := requiredString(values, "S3_BUCKET") + bucket, err := requiredString(values, "object_storage.bucket") if err != nil { return objectStorageConfig{}, err } - accessKey, err := requiredString(values, "S3_ACCESS_KEY") + accessKey, err := requiredString(values, "object_storage.access_key") if err != nil { return objectStorageConfig{}, err } - secretKey, err := requiredString(values, "S3_SECRET_KEY") + secretKey, err := requiredString(values, "object_storage.secret_key") if err != nil { return objectStorageConfig{}, err } - useTLS, err := boolean(values, "S3_USE_TLS") + useTLS, err := boolean(values, "object_storage.use_tls") if err != nil { return objectStorageConfig{}, err } - timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) if err != nil { return objectStorageConfig{}, err } @@ -311,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { } func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { - databaseURL, err := requiredString(values, "DATABASE_URL") + databaseURL, err := requiredString(values, "database.url") if err != nil { return databaseConfig{}, err } - maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) + maxConns, err := integer(values, "database.max_connections", 1, 100) if err != nil { return databaseConfig{}, err } - minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) + minConns, err := integer(values, "database.min_connections", 0, maxConns) if err != nil { return databaseConfig{}, err } - maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) + maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) + maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) + healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour) if err != nil { return databaseConfig{}, err } - dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) + dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute) if err != nil { return databaseConfig{}, err } @@ -365,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) { return value, nil } -func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) { - value, err := integer(values, name, minimum, maximum) +func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { + value, err := requiredString(values, name) if err != nil { return 0, err } - return time.Duration(value) * time.Second, nil + parsed, err := time.ParseDuration(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum) + } + return parsed, nil } func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 863746ce..80d22161 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,24 +1,17 @@ package config import ( + "os" + "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { - values := map[string]string{ - "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", - "DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4", - "DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60", - "DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5", - "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", - "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", - "S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5", - } - for name, value := range values { - t.Setenv(name, value) - } + setConfigEnvironment(t, validTestConfig) t.Setenv("APP_HOST", "") t.Setenv("CACHE_URL", "") t.Setenv("SWAGGER_PORT", "") @@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { require.NoError(t, err) require.Equal(t, "attachments", settings.S3Bucket) } + +func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("WORKER_CONCURRENCY", "64") + + settings, err := LoadWorker() + require.NoError(t, err) + require.Equal(t, 2, settings.Concurrency) + require.Equal(t, 5*time.Second, settings.HeartbeatInterval) +} + +func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) { + content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1) + setConfigEnvironment(t, content) + + _, err := LoadWorker() + require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease") +} + +func TestLoadValuesRejectsUnknownVersion(t *testing.T) { + setConfigEnvironment(t, "version: 2\n") + + _, err := loadValues() + require.ErrorContains(t, err, "version: must be 1") +} + +func setConfigEnvironment(t *testing.T, content string) { + t.Helper() + path := filepath.Join(t.TempDir(), "messenger.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + t.Setenv("MESSENGER_CONFIG_PATH", path) + for name, value := range map[string]string{ + "APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081", + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + } { + t.Setenv(name, value) + } +} + +const validTestConfig = `version: 1 +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 +database: + min_connections: 0 + max_connections: 4 + max_connection_age: 5m + max_connection_idle: 1m + health_check_period: 30s + dependency_timeout: 5s +migration: + timeout: 30s +object_storage: + use_tls: false + timeout: 5s +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 +pipeline: + version: 1 +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 +monitoring: + metrics_address: 0.0.0.0:9091 +` diff --git a/internal/config/loader.go b/internal/config/loader.go new file mode 100644 index 00000000..6cd115c1 --- /dev/null +++ b/internal/config/loader.go @@ -0,0 +1,47 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/viper" +) + +const supportedConfigVersion = 1 + +var environmentKeys = map[string]string{ + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +func loadValues() (*viper.Viper, error) { + path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH")) + if path == "" { + return nil, errors.New("MESSENGER_CONFIG_PATH is required") + } + values := viper.New() + values.SetConfigFile(path) + if err := values.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read messenger config: %w", err) + } + if version := values.GetInt("version"); version != supportedConfigVersion { + return nil, fmt.Errorf("version: must be %d", supportedConfigVersion) + } + for key, name := range environmentKeys { + if err := values.BindEnv(key, name); err != nil { + return nil, fmt.Errorf("bind %s: %w", name, err) + } + } + return values, nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go index 572d59bf..76b4f5d8 100644 --- a/internal/config/pipeline.go +++ b/internal/config/pipeline.go @@ -2,6 +2,7 @@ package config import ( "errors" + "fmt" "net" "time" @@ -26,6 +27,9 @@ type DispatcherConfig struct { PollInterval time.Duration RecoveryInterval time.Duration RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int MetricsAddress string } @@ -52,6 +56,7 @@ type WorkerConfig struct { Concurrency int Prefetch int LeaseDuration time.Duration + HeartbeatInterval time.Duration ProcessingTimeout time.Duration ReadBlock time.Duration ReclaimInterval time.Duration @@ -83,8 +88,10 @@ type brokerConfig struct { } func LoadJobAdmin() (JobAdminConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return JobAdminConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return JobAdminConfig{}, err @@ -97,8 +104,10 @@ func LoadJobAdmin() (JobAdminConfig, error) { } func LoadDispatcher() (DispatcherConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return DispatcherConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return DispatcherConfig{}, err @@ -107,23 +116,35 @@ func LoadDispatcher() (DispatcherConfig, error) { if err != nil { return DispatcherConfig{}, err } - batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000) + batchSize, err := integer(values, "outbox.batch_size", 1, 1000) if err != nil { return DispatcherConfig{}, err } - leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600) + leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute) if err != nil { return DispatcherConfig{}, err } - pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60) + pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute) if err != nil { return DispatcherConfig{}, err } - recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600) + recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour) if err != nil { return DispatcherConfig{}, err } - retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600) + retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000) if err != nil { return DispatcherConfig{}, err } @@ -138,13 +159,17 @@ func LoadDispatcher() (DispatcherConfig, error) { BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, - RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, + Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, + MetricsAddress: metricsAddress, }, nil } func LoadWorker() (WorkerConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return WorkerConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return WorkerConfig{}, err @@ -157,57 +182,64 @@ func LoadWorker() (WorkerConfig, error) { if err != nil { return WorkerConfig{}, err } - concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64) + concurrency, err := integer(values, "worker.concurrency", 1, 64) if err != nil { return WorkerConfig{}, err } - prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024) + prefetch, err := integer(values, "worker.prefetch", concurrency, 1024) if err != nil { return WorkerConfig{}, err } - leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600) + leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600) + heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if heartbeatInterval >= leaseDuration { + return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease") + } + processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } if processingTimeout >= leaseDuration { - return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS") + return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease") } - readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60) + readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute) if err != nil { return WorkerConfig{}, err } - reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600) + reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100) + maxAttempts, err := integer(values, "worker.max_attempts", 1, 100) if err != nil { return WorkerConfig{}, err } - retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600) + retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400) + retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour) if err != nil { return WorkerConfig{}, err } if retryMax < retryBase { - return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS") + return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base") } - attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) if err != nil { return WorkerConfig{}, err } - attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) if err != nil { return WorkerConfig{}, err } - memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024) + memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024) if err != nil { return WorkerConfig{}, err } @@ -225,44 +257,46 @@ func LoadWorker() (WorkerConfig, error) { ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, - ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock, ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), - MemoryBytes: int64(memoryBytes), - MetricsAddress: metricsAddress, + MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, }, nil } func metricsAddress(values *viper.Viper) (string, error) { - host, err := requiredString(values, "PIPELINE_METRICS_HOST") + address, err := requiredString(values, "monitoring.metrics_address") if err != nil { return "", err } - port, err := requiredPort(values, "PIPELINE_METRICS_PORT") + _, port, err := net.SplitHostPort(address) if err != nil { - return "", err + return "", fmt.Errorf("monitoring.metrics_address: %w", err) } - return net.JoinHostPort(host, port), nil + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + return address, nil } func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { - url, err := requiredString(values, "BROKER_URL") + url, err := requiredString(values, "broker.url") if err != nil { return brokerConfig{}, err } - stream, err := requiredString(values, "BROKER_STREAM") + stream, err := requiredString(values, "broker.stream") if err != nil { return brokerConfig{}, err } - consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP") + consumerGroup, err := requiredString(values, "broker.consumer_group") if err != nil { return brokerConfig{}, err } - failedStream, err := requiredString(values, "BROKER_FAILED_STREAM") + failedStream, err := requiredString(values, "broker.failed_stream") if err != nil { return brokerConfig{}, err } - failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000) + failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000) if err != nil { return brokerConfig{}, err } diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go index 35a5c664..53e532be 100644 --- a/internal/messenger/attachment/transform/exif.go +++ b/internal/messenger/attachment/transform/exif.go @@ -1,6 +1,7 @@ package transform import ( + "context" "encoding/binary" "errors" "image" @@ -9,11 +10,14 @@ import ( var errInvalidEXIF = errors.New("invalid EXIF metadata") -func jpegOrientation(data []byte) (int, error) { +func jpegOrientation(ctx context.Context, data []byte) (int, error) { if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { return 0, errInvalidEXIF } for offset := 2; offset < len(data); { + if err := contextError(ctx); err != nil { + return 0, err + } if data[offset] != 0xff { return 0, errInvalidEXIF } @@ -43,12 +47,12 @@ func jpegOrientation(data []byte) (int, error) { if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { continue } - return tiffOrientation(payload[6:]) + return tiffOrientation(ctx, payload[6:]) } return 1, nil } -func tiffOrientation(data []byte) (int, error) { +func tiffOrientation(ctx context.Context, data []byte) (int, error) { if len(data) < 8 { return 0, errInvalidEXIF } @@ -74,6 +78,9 @@ func tiffOrientation(data []byte) (int, error) { return 0, errInvalidEXIF } for index := uint64(0); index < count; index++ { + if err := contextError(ctx); err != nil { + return 0, err + } entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] if order.Uint16(entry[:2]) != 0x0112 { continue @@ -90,7 +97,7 @@ func tiffOrientation(data []byte) (int, error) { return 1, nil } -func applyOrientation(source image.Image, orientation int) *image.NRGBA { +func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) { bounds := source.Bounds() width, height := bounds.Dx(), bounds.Dy() outputWidth, outputHeight := width, height @@ -99,12 +106,20 @@ func applyOrientation(source image.Image, orientation int) *image.NRGBA { } output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) for y := 0; y < outputHeight; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < outputWidth; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) } } - return output + return output, nil } func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { @@ -128,11 +143,19 @@ func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { } } -func flattenWhite(source image.Image) *image.NRGBA { +func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) { bounds := source.Bounds() output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() output.SetNRGBA(x, y, color.NRGBA{ R: uint8((r + 0xffff - a) / 0x101), @@ -142,5 +165,5 @@ func flattenWhite(source image.Image) *image.NRGBA { }) } } - return output + return output, nil } diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go index a8dde17d..8a6dd08f 100644 --- a/internal/messenger/attachment/transform/processor.go +++ b/internal/messenger/attachment/transform/processor.go @@ -9,6 +9,7 @@ import ( "image" "image/jpeg" _ "image/png" + "io" "math" "net/http" @@ -78,7 +79,7 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp if err := contextError(ctx); err != nil { return Result{}, err } - img, format, width, height, orientation, err := p.decode(data, declaredContentType) + img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType) if err != nil { return Result{}, err } @@ -86,12 +87,12 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return Result{}, err } - oriented := applyOrientation(img, orientation) - if err := contextError(ctx); err != nil { + oriented, err := applyOrientation(ctx, img, orientation) + if err != nil { return Result{}, err } - opaque := flattenWhite(oriented) - if err := contextError(ctx); err != nil { + opaque, err := flattenWhite(ctx, oriented) + if err != nil { return Result{}, err } result := Result{ @@ -99,7 +100,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp Orientation: orientation, } - thumbnail := fit(opaque, 256, 256, false) + thumbnail, err := fitContext(ctx, opaque, 256, 256, false) + if err != nil { + return Result{}, err + } result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) if err != nil { return Result{}, err @@ -109,7 +113,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp } crop := centeredSquare(opaque) - crop = resize(crop, 128, 128) + crop, err = resizeContext(ctx, crop, 128, 128) + if err != nil { + return Result{}, err + } result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) if err != nil { return Result{}, err @@ -118,7 +125,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return Result{}, err } - optimized := fit(opaque, 1280, 1280, false) + optimized, err := fitContext(ctx, opaque, 1280, 1280, false) + if err != nil { + return Result{}, err + } result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) if err != nil { return Result{}, err @@ -129,7 +139,11 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return result, nil } -func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) { +func (p *Processor) decode( + ctx context.Context, + data []byte, + declaredContentType string, +) (image.Image, string, int, int, int, error) { if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { return nil, "", 0, 0, 0, ErrUnsupportedMedia } @@ -142,27 +156,48 @@ func (p *Processor) decode(data []byte, declaredContentType string) (image.Image if http.DetectContentType(data) != declaredContentType { return nil, "", 0, 0, 0, ErrUnsupportedMedia } - config, format, err := image.DecodeConfig(bytes.NewReader(data)) + config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { return nil, "", 0, 0, 0, ErrInvalidImage } if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { return nil, "", 0, 0, 0, ErrPayloadTooLarge } - img, decodedFormat, err := image.Decode(bytes.NewReader(data)) + img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } if err != nil || decodedFormat != format { return nil, "", 0, 0, 0, ErrInvalidImage } orientation := 1 if format == "jpeg" { - orientation, err = jpegOrientation(data) + orientation, err = jpegOrientation(ctx, data) if err != nil { + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } return nil, "", 0, 0, 0, ErrInvalidImage } } return img, declaredContentType, config.Width, config.Height, orientation, nil } +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + if err := contextError(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + func contextError(ctx context.Context) error { select { case <-ctx.Done(): diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go index 972cb6d6..7b595dd9 100644 --- a/internal/messenger/attachment/transform/processor_test.go +++ b/internal/messenger/attachment/transform/processor_test.go @@ -64,7 +64,10 @@ func TestFlattenWhiteCompositesAlpha(t *testing.T) { source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) - result := flattenWhite(source) + result, err := flattenWhite(context.Background(), source) + if err != nil { + t.Fatalf("flatten image: %v", err) + } if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { t.Fatalf("transparent pixel: got %#v", got) } @@ -154,6 +157,28 @@ func TestProcessHonorsCanceledContext(t *testing.T) { } } +func TestResizeStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := cancelingImage{ + Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)), + cancel: cancel, + } + _, err := resizeContext(ctx, source, 128, 128) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +type cancelingImage struct { + image.Image + cancel context.CancelFunc +} + +func (i cancelingImage) At(x, y int) color.Color { + i.cancel() + return i.Image.At(x, y) +} + func TestEstimateMemoryWeight(t *testing.T) { weight, err := EstimateMemoryWeight(1000, 320, 200) if err != nil { diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go index 4526b5ef..2d86bb83 100644 --- a/internal/messenger/attachment/transform/resize.go +++ b/internal/messenger/attachment/transform/resize.go @@ -1,14 +1,21 @@ package transform import ( + "context" "image" "image/color" ) -func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA { +func fitContext( + ctx context.Context, + source image.Image, + maxWidth int, + maxHeight int, + upscale bool, +) (*image.NRGBA, error) { bounds := source.Bounds() width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) - return resize(source, width, height) + return resizeContext(ctx, source, width, height) } func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { @@ -43,15 +50,23 @@ func (i croppedImage) At(x, y int) color.Color { return i.source.At(i.offset.X+x, i.offset.Y+y) } -func resize(source image.Image, width, height int) *image.NRGBA { +func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) { bounds := source.Bounds() if bounds.Dx() == width && bounds.Dy() == height { - return copyNRGBA(source) + return copyNRGBAContext(ctx, source) } output := image.NewNRGBA(image.Rect(0, 0, width, height)) for y := 0; y < height; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) for x := 0; x < width; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) @@ -60,7 +75,7 @@ func resize(source image.Image, width, height int) *image.NRGBA { output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) } } - return output + return output, nil } func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { @@ -93,13 +108,21 @@ func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { return uint8((value + 1<<31) >> 32) } -func copyNRGBA(source image.Image) *image.NRGBA { +func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) { bounds := source.Bounds() output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) } } - return output + return output, nil } diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index 9fa0082a..f94f525d 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -27,6 +27,7 @@ import ( type Repository interface { AttachmentJob(context.Context, string) (model.AttachmentJob, error) ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) @@ -59,6 +60,7 @@ type Config struct { WorkerID string Concurrency int LeaseDuration time.Duration + HeartbeatInterval time.Duration ProcessingTimeout time.Duration ReclaimInterval time.Duration MaxAttempts int @@ -119,6 +121,7 @@ func New( func validateConfig(config Config) error { if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration || config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { @@ -218,9 +221,17 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return ErrInjectedFailure } - processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout) + processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout) + heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx) + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }() variants, err := w.process(processingCtx, lease) - cancel() + cancelProcessing() + cancelHeartbeat() + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + return heartbeatErr + } if err != nil { result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) return err @@ -252,6 +263,34 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return nil } +func (w *Worker) maintainLease( + ctx context.Context, + cancelProcessing context.CancelFunc, + lease model.AttachmentJobLease, +) error { + ticker := time.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration) + if err != nil { + if ctx.Err() != nil { + return nil + } + cancelProcessing() + return fmt.Errorf("renew attachment job lease: %w", err) + } + if !renewed { + cancelProcessing() + return model.ErrStaleLease + } + } + } +} + func (w *Worker) resolveClaim( ctx context.Context, delivery redisstream.Delivery, @@ -295,15 +334,18 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([ return nil, err } defer func() { _ = body.Close() }() - data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1)) + data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1)) if err != nil { return nil, err } if int64(len(data)) > w.config.MaxBytes { return nil, transform.ErrPayloadTooLarge } - imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data)) + imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, transform.ErrInvalidImage } weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) @@ -346,6 +388,20 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([ return result, nil } +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.reader.Read(buffer) + } +} + var errWorkerCapacity = errors.New("image exceeds worker memory budget") func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go index 77a1d5f5..7a573f1d 100644 --- a/internal/messenger/attachment/worker/worker_test.go +++ b/internal/messenger/attachment/worker/worker_test.go @@ -1,13 +1,16 @@ package worker import ( + "context" "errors" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" ) @@ -46,11 +49,61 @@ func TestProcessingErrorClassification(t *testing.T) { } } +func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) { + var renewals atomic.Int32 + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + renewals.Add(1) + return true, nil + }} + config := validConfig() + config.HeartbeatInterval = 5 * time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }() + require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) +} + +func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) { + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + return false, nil + }} + config := validConfig() + config.HeartbeatInterval = time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + err := worker.maintainLease(ctx, cancel, testLease()) + require.ErrorIs(t, err, model.ErrStaleLease) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +type repositoryStub struct { + Repository + renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) +} + +func (s *repositoryStub) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + return s.renew(ctx, lease, duration) +} + +func testLease() model.AttachmentJobLease { + return model.AttachmentJobLease{Job: model.AttachmentJob{ + ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1", + }} +} + func validConfig() Config { return Config{ WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, - ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second, - MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second, + ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, } } diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go index 17778e3c..22aa75de 100644 --- a/internal/outbox/dispatcher.go +++ b/internal/outbox/dispatcher.go @@ -16,6 +16,7 @@ type Store interface { MarkOutboxPublished(context.Context, string, string) (bool, error) ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) + DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error) } type Publisher interface { @@ -34,6 +35,9 @@ type Config struct { PollInterval time.Duration RecoveryInterval time.Duration RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int } type Dispatcher struct { @@ -72,6 +76,12 @@ func (c Config) validate() error { return errors.New("outbox recovery interval must be positive") case c.RetryDelay <= 0: return errors.New("outbox retry delay must be positive") + case c.Retention <= 0: + return errors.New("outbox retention must be positive") + case c.CleanupInterval <= 0: + return errors.New("outbox cleanup interval must be positive") + case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000: + return errors.New("outbox cleanup batch size must be between 1 and 1000") default: return nil } @@ -84,8 +94,10 @@ func (d *Dispatcher) Run(ctx context.Context) error { pollTicker := time.NewTicker(d.config.PollInterval) recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + cleanupTicker := time.NewTicker(d.config.CleanupInterval) defer pollTicker.Stop() defer recoveryTicker.Stop() + defer cleanupTicker.Stop() if err := d.RecoverExpired(ctx); err != nil { if ctx.Err() != nil { @@ -112,10 +124,27 @@ func (d *Dispatcher) Run(ctx context.Context) error { } return err } + case <-cleanupTicker.C: + if _, err := d.CleanupPublished(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } } } } +func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) { + deleted, err := d.store.DeletePublishedOutboxEvents( + ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize, + ) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return deleted, nil +} + func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) if err != nil { diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go index 036741e8..206790d2 100644 --- a/internal/outbox/dispatcher_test.go +++ b/internal/outbox/dispatcher_test.go @@ -70,6 +70,18 @@ func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { require.Zero(t, published) } +func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) { + store := &storeStub{deleted: 2} + dispatcher := newTestDispatcher(t, store, &publisherStub{}) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + deleted, err := dispatcher.CleanupPublished(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, deleted) + require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore) + require.Equal(t, 100, store.cleanupLimit) +} + func TestDispatcherStopsAfterCancellation(t *testing.T) { store := &storeStub{} publisher := &publisherStub{} @@ -96,6 +108,9 @@ func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatch PollInterval: time.Millisecond, RecoveryInterval: time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, + CleanupInterval: time.Hour, + CleanupBatchSize: 100, }) require.NoError(t, err) return dispatcher @@ -114,14 +129,17 @@ func testPayload() json.RawMessage { } type storeStub struct { - mu sync.Mutex - events []model.OutboxEvent - markErr error - markLost bool - marked []string - released string - nextAttemptAt time.Time - recoveredCalls int + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int + deleted int + publishedBefore time.Time + cleanupLimit int } func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { @@ -164,6 +182,14 @@ func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, return 0, nil } +func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishedBefore = before + s.cleanupLimit = limit + return s.deleted, nil +} + type publisherStub struct { mu sync.Mutex messageID string diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go new file mode 100644 index 00000000..b6883eb0 --- /dev/null +++ b/internal/postgres/attachment_lease.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE attachment_jobs + SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'), + updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_until > clock_timestamp()` + result, err := s.pool.Exec( + ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(), + ) + if err != nil { + return false, fmt.Errorf("renew attachment job lease: %w", err) + } + return result.RowsAffected() == 1, nil +} diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go index 6bfcc107..9eda8656 100644 --- a/internal/postgres/outbox.go +++ b/internal/postgres/outbox.go @@ -111,3 +111,26 @@ func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) ( } return int(result.RowsAffected()), nil } + +func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) { + if publishedBefore.IsZero() || limit < 1 || limit > 1000 { + return 0, model.ErrInvalidInput + } + const query = ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at < $1 + ORDER BY published_at, id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM outbox_events event + USING candidates + WHERE event.id = candidates.id` + result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/lab5.env.example b/lab5.env.example index c9d7b28f..5d3809a1 100644 --- a/lab5.env.example +++ b/lab5.env.example @@ -1,6 +1,7 @@ # Адрес и внутренний порт HTTP-сервера. APP_HOST=0.0.0.0 APP_PORT=8080 +MESSENGER_CONFIG_PATH=config/messenger.yaml # Адреса публикации сервисов на локальной машине. API_BIND_ADDRESS=127.0.0.1:8080 @@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 -# Время жизни сессии, идемпотентности и используемое хранилище. -SESSION_TTL_SECONDS=3600 -IDEMPOTENCY_TTL_SECONDS=86400 -SESSION_STORE=valkey - # Число API-реплик за Nginx. API_REPLICAS=2 @@ -29,28 +25,6 @@ CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 # Redis Streams используется только как устойчивый брокер заданий. REDIS_BROKER_PASSWORD=local-redis-broker-password BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 -BROKER_STREAM=attachment-jobs -BROKER_CONSUMER_GROUP=attachment-workers -BROKER_FAILED_STREAM=attachment-jobs-failed -BROKER_FAILED_MAX_LENGTH=10000 - -# Версия pipeline, outbox и ограниченные повторы worker. -PIPELINE_VERSION=1 -OUTBOX_BATCH_SIZE=32 -OUTBOX_LEASE_SECONDS=15 -OUTBOX_POLL_SECONDS=1 -OUTBOX_RECOVERY_SECONDS=10 -OUTBOX_RETRY_SECONDS=2 -WORKER_CONCURRENCY=2 -WORKER_PREFETCH=4 -WORKER_LEASE_SECONDS=30 -WORKER_PROCESSING_TIMEOUT_SECONDS=20 -WORKER_READ_BLOCK_SECONDS=2 -WORKER_RECLAIM_SECONDS=5 -WORKER_MAX_ATTEMPTS=5 -WORKER_RETRY_BASE_SECONDS=1 -WORKER_RETRY_MAX_SECONDS=30 -WORKER_MEMORY_BYTES=738197504 # Административные данные MinIO используются только при bootstrap. MINIO_ROOT_USER=local-minio-admin @@ -62,36 +36,6 @@ S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY=messenger-app S3_SECRET_KEY=local-messenger-s3-password -S3_USE_TLS=false -S3_TIMEOUT_SECONDS=5 -ATTACHMENT_MAX_BYTES=10485760 -ATTACHMENT_MAX_PIXELS=40000000 -ATTACHMENT_BODY_MEMORY_BYTES=67108864 -ATTACHMENT_DECODE_MEMORY_BYTES=402653184 - -# Размер пула и обслуживание соединений PostgreSQL. -DB_MIN_CONNS=1 -DB_MAX_CONNS=10 -DB_MAX_CONNECTION_AGE_SECONDS=1800 -DB_MAX_CONNECTION_IDLE_SECONDS=300 -DB_HEALTH_CHECK_SECONDS=30 - -# Таймауты зависимостей, миграций и HTTP-запросов. -DEPENDENCY_TIMEOUT_SECONDS=5 -MIGRATION_TIMEOUT_SECONDS=30 -REQUEST_TIMEOUT_SECONDS=5 -HTTP_READ_HEADER_TIMEOUT_SECONDS=5 -HTTP_READ_TIMEOUT_SECONDS=10 -HTTP_WRITE_TIMEOUT_SECONDS=10 -HTTP_IDLE_TIMEOUT_SECONDS=60 -HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 - -# Параметры Argon2id для новых хешей паролей. -PASSWORD_MEMORY_KIB=65536 -PASSWORD_ITERATIONS=3 -PASSWORD_PARALLELISM=2 -PASSWORD_SALT_LENGTH=16 -PASSWORD_KEY_LENGTH=32 # Порт Swagger внутри конфигурации CORS API. SWAGGER_PORT=8081 diff --git a/migrations/0007_outbox_retention.sql b/migrations/0007_outbox_retention.sql new file mode 100644 index 00000000..cc936f62 --- /dev/null +++ b/migrations/0007_outbox_retention.sql @@ -0,0 +1,3 @@ +CREATE INDEX outbox_events_published_at_idx + ON outbox_events (published_at, id) + WHERE published_at IS NOT NULL; -- 2.54.0 From 25c80a00729f3abf62b496ef8372e41e93c7621f Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 13:45:58 +0300 Subject: [PATCH 07/11] fix: preserve attachment pipeline invariants --- internal/broker/redisstream/broker.go | 13 +- internal/broker/redisstream/broker_test.go | 21 +++- internal/postgres/attachment_jobs.go | 31 ++--- internal/postgres/attachments.go | 66 +++++++++- internal/postgres/store_integration_test.go | 133 ++++++++++++++++---- internal/reconcile/reconcile.go | 36 ++++-- internal/reconcile/reconcile_test.go | 113 ++++++++++++++++- 7 files changed, 353 insertions(+), 60 deletions(-) diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go index 74a53745..35d46f09 100644 --- a/internal/broker/redisstream/broker.go +++ b/internal/broker/redisstream/broker.go @@ -14,7 +14,10 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" ) -const payloadField = "payload" +const ( + payloadField = "payload" + failedPayloadMaxBytes = 64 << 10 +) var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) @@ -167,6 +170,7 @@ func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { deliveries = append(deliveries, delivery) continue } + delivery.payload = boundedPayload(payload) envelope, err := DecodeEnvelope(payload) if err != nil { delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) @@ -184,6 +188,13 @@ func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { return deliveries, nil } +func boundedPayload(payload []byte) []byte { + if len(payload) > failedPayloadMaxBytes { + payload = payload[:failedPayloadMaxBytes] + } + return append([]byte(nil), payload...) +} + func messagePayload(message redis.XMessage) ([]byte, error) { value, ok := message.Values[payloadField] if !ok { diff --git a/internal/broker/redisstream/broker_test.go b/internal/broker/redisstream/broker_test.go index 5e2f4c64..5d96c429 100644 --- a/internal/broker/redisstream/broker_test.go +++ b/internal/broker/redisstream/broker_test.go @@ -47,7 +47,26 @@ func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, deliveries, 1) require.Error(t, deliveries[0].DecodeError) - require.Empty(t, deliveries[0].payload) + require.Contains(t, string(deliveries[0].payload), `"storage_key":"private/object"`) +} + +func TestBrokerBoundsMalformedPayloadForFailedStream(t *testing.T) { + payload := make([]byte, failedPayloadMaxBytes+1) + for index := range payload { + payload[index] = 'x' + } + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: payload}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Len(t, deliveries[0].payload, failedPayloadMaxBytes) } func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) { diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 34f914e7..3bcfb391 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -361,7 +361,7 @@ func (s *Store) CompleteAttachmentJob( lease model.AttachmentJobLease, variants []model.Variant, ) (model.JobMutationResult, error) { - if !validCompletedVariants(variants) { + if !validCompletedVariants(variants, lease.Job.PipelineVersion) { return "", model.ErrInvalidInput } tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) @@ -423,20 +423,23 @@ func (s *Store) CompleteAttachmentJob( return model.JobMutationApplied, nil } -func validCompletedVariants(variants []model.Variant) bool { - seen := make(map[string]struct{}, len(variants)) - for _, variant := range variants { - if variant.Name == "" || variant.ObjectKey == "" { - return false - } - if _, duplicate := seen[variant.Name]; duplicate { - return false - } - seen[variant.Name] = struct{}{} +func validCompletedVariants(variants []model.Variant, pipelineVersion int) bool { + if len(variants) != 4 || pipelineVersion < 1 { + return false } - _, hasOriginal := seen["original"] - _, hasThumbnail := seen["thumbnail"] - return hasOriginal && hasThumbnail + required := map[string]struct{}{ + "original": {}, "thumbnail": {}, "crop": {}, "optimized": {}, + } + for _, variant := range variants { + if variant.ObjectKey == "" || variant.PipelineVersion != pipelineVersion { + return false + } + if _, exists := required[variant.Name]; !exists { + return false + } + delete(required, variant.Name) + } + return len(required) == 0 } func (s *Store) RetryAttachmentJob( diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 44d60dff..e7c25318 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -174,16 +174,51 @@ func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey strin return exists, nil } +func (s *Store) AttachmentObjectKeys(ctx context.Context, attachmentID string) ([]string, error) { + const query = ` + SELECT object_key + FROM ( + SELECT object_key + FROM attachments + WHERE id = $1 AND object_key <> '' + UNION + SELECT object_key + FROM attachment_variants + WHERE attachment_id = $1 + ) objects + ORDER BY object_key` + rows, err := s.pool.Query(ctx, query, attachmentID) + if err != nil { + return nil, fmt.Errorf("list attachment object keys: %w", err) + } + defer rows.Close() + + keys, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("scan attachment object keys: %w", err) + } + return keys, nil +} + func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { if limit < 1 || limit > 1000 { return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") } query := ` SELECT ` + attachmentColumnsWithAlias("a") + `, - EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) - FROM attachments a - WHERE a.object_key > $1 - ORDER BY a.object_key + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id), + objects.object_key + FROM ( + SELECT id AS attachment_id, object_key + FROM attachments + WHERE object_key <> '' + UNION + SELECT attachment_id, object_key + FROM attachment_variants + ) objects + JOIN attachments a ON a.id = objects.attachment_id + WHERE objects.object_key > $1 + ORDER BY objects.object_key LIMIT $2` rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) if err != nil { @@ -193,7 +228,9 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { var attachment model.Attachment - err := scanAttachmentWithReference(row, &attachment) + var objectKey string + err := scanAttachmentWithReferenceAndObjectKey(row, &attachment, &objectKey) + attachment.ObjectKey = objectKey return publicAttachment(attachment), err })) if err != nil { @@ -207,6 +244,25 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi return attachments, next, nil } +func scanAttachmentWithReferenceAndObjectKey( + row rowScanner, + attachment *model.Attachment, + objectKey *string, +) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, objectKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { var stagingKey *string err := row.Scan( diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 89ce5882..e478d9be 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "strings" "sync" @@ -204,18 +205,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { s.Equal(model.AttachmentProcessing, lease.Attachment.Status) s.NotEmpty(lease.Attachment.StagingKey) - variants := []model.Variant{ - { - Name: "original", ContentType: "image/png", SizeBytes: 128, - SHA256: strings.Repeat("a", 64), Width: 320, Height: 200, - ObjectKey: "attachments/" + attachment.ID + "/v1/original.png", - }, - { - Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64, - SHA256: strings.Repeat("b", 64), Width: 256, Height: 160, - ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg", - }, - } + variants := completedVariants(attachment.ID, 1) result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) s.Require().NoError(err) s.Equal(model.JobMutationApplied, result) @@ -228,8 +218,26 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { ) s.Require().NoError(err) s.Equal(model.AttachmentReady, metadata.Status) - s.Len(metadata.Variants, 1) + s.Len(metadata.Variants, 3) s.Equal(variants[1].ObjectKey, variant.ObjectKey) + listedKeys := make(map[string]struct{}) + var after string + for { + listed, next, err := s.store.ListAttachments(context.Background(), after, 1000) + s.Require().NoError(err) + for _, candidate := range listed { + if candidate.ID == attachment.ID { + listedKeys[candidate.ObjectKey] = struct{}{} + } + } + if next == "" { + break + } + after = next + } + for _, candidate := range variants { + s.Contains(listedKeys, candidate.ObjectKey) + } v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2} queuedV2, err := s.store.QueueAttachmentPipelineVersion( @@ -243,6 +251,46 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + _, secondJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + events, err := s.store.ClaimOutboxEvents(context.Background(), 1000, time.Minute) + s.Require().NoError(err) + + eventIDs := make([]string, 0, 2) + for _, event := range events { + if event.JobID != firstJob.ID && event.JobID != secondJob.ID { + continue + } + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + eventIDs = append(eventIDs, event.ID) + } + s.Require().Len(eventIDs, 2) + _, err = s.pool.Exec( + context.Background(), + `UPDATE outbox_events SET published_at = clock_timestamp() - INTERVAL '2 days' WHERE id IN ($1, $2)`, + eventIDs[0], eventIDs[1], + ) + s.Require().NoError(err) + + deleted, err := s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + deleted, err = s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + + var remaining int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM outbox_events WHERE id IN ($1, $2)`, eventIDs[0], eventIDs[1], + ).Scan(&remaining) + s.Require().NoError(err) + s.Zero(remaining) +} + func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { owner, chatID := createOwnerAndChat(s.T(), s.store) attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) @@ -303,22 +351,34 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { s.Equal(2, current.Attempt) s.Equal(model.AttachmentJobQueued, current.Status) - result, err := s.store.CompleteAttachmentJob(context.Background(), lease, []model.Variant{ - { - Name: "original", ContentType: "image/png", SizeBytes: 128, - SHA256: strings.Repeat("a", 64), Width: 320, Height: 200, - ObjectKey: "attachments/" + attachment.ID + "/v1/original.png", - }, - { - Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64, - SHA256: strings.Repeat("b", 64), Width: 256, Height: 160, - ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg", - }, - }) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) s.Require().NoError(err) s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + + complete := completedVariants(attachment.ID, job.PipelineVersion) + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, complete[:3]) + s.ErrorIs(err, model.ErrInvalidInput) + + duplicate := append([]model.Variant(nil), complete...) + duplicate[3].Name = duplicate[2].Name + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, duplicate) + s.ErrorIs(err, model.ErrInvalidInput) + + mismatched := append([]model.Variant(nil), complete...) + mismatched[3].PipelineVersion++ + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, mismatched) + s.ErrorIs(err, model.ErrInvalidInput) +} + func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { owner, chatID := createOwnerAndChat(s.T(), s.store) pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ @@ -533,6 +593,29 @@ func createQueuedAttachment( return queued, job } +func completedVariants(attachmentID string, pipelineVersion int) []model.Variant { + types := []struct { + name string + contentType string + extension string + }{ + {name: "original", contentType: "image/png", extension: "png"}, + {name: "thumbnail", contentType: "image/jpeg", extension: "jpg"}, + {name: "crop", contentType: "image/jpeg", extension: "jpg"}, + {name: "optimized", contentType: "image/webp", extension: "webp"}, + } + variants := make([]model.Variant, 0, len(types)) + for index, variant := range types { + variants = append(variants, model.Variant{ + Name: variant.name, ContentType: variant.contentType, SizeBytes: int64(128 - index*16), + SHA256: strings.Repeat(string(rune('a'+index)), 64), Width: 320 - index*32, Height: 200 - index*20, + ObjectKey: fmt.Sprintf("attachments/%s/v%d/%s.%s", attachmentID, pipelineVersion, variant.name, variant.extension), + PipelineVersion: pipelineVersion, + }) + } + return variants +} + func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { t.Helper() var count int diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 6f84b585..beb843c9 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -11,6 +11,7 @@ import ( type MetadataStore interface { ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentObjectKeys(context.Context, string) ([]string, error) AttachmentExistsByObjectKey(context.Context, string) (bool, error) ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) DeleteAttachmentIfDeleting(context.Context, string) (bool, error) @@ -158,6 +159,9 @@ func (r *Reconciler) handlePair( report.MetadataScanned++ report.ObjectsScanned++ switch metadata.Status { + case model.AttachmentQueued, model.AttachmentProcessing, model.AttachmentFailed: + report.Consistent++ + return nil case model.AttachmentPending: if metadata.CreatedAt.After(anomalyCutoff) { report.FreshPendingAttachments++ @@ -208,10 +212,16 @@ func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Att if !r.config.Apply { return nil } - if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil { - return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err) + objectKeys, err := r.metadata.AttachmentObjectKeys(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("list attachment %q object keys: %w", attachment.ID, err) + } + for _, objectKey := range objectKeys { + if err := r.objects.Delete(ctx, objectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", objectKey, err) + } + report.DeletedAttachmentObjects++ } - report.DeletedAttachmentObjects++ deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) if err != nil { return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) @@ -229,6 +239,14 @@ func (r *Reconciler) handleOrphanObject( report *Report, ) error { report.ObjectsScanned++ + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + report.Consistent++ + return nil + } if object.LastModified.After(cutoff) { report.FreshOrphanObjects++ return nil @@ -237,13 +255,6 @@ func (r *Reconciler) handleOrphanObject( if !r.config.Apply { return nil } - exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) - if err != nil { - return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) - } - if exists { - return nil - } if err := r.objects.Delete(ctx, object.Key); err != nil { return fmt.Errorf("delete orphan object %q: %w", object.Key, err) } @@ -258,6 +269,11 @@ func (r *Reconciler) handleMissingObject( report *Report, ) error { report.MetadataScanned++ + if metadata.Status == model.AttachmentQueued || + metadata.Status == model.AttachmentProcessing || + metadata.Status == model.AttachmentFailed { + return nil + } if metadata.Status == model.AttachmentPending { if metadata.CreatedAt.After(cutoff) { report.FreshPendingAttachments++ diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index a468274a..8e1c1b3f 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -82,6 +82,98 @@ func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { require.Contains(t, objects.items, "attachments/a/original") } +func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) { + now := time.Now().UTC() + key := "staging/a/original" + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentQueued, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + key: {Key: key, LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", OrphanPrefixes: []string{"staging/"}, PageSize: 1, + GraceTime: time.Minute, UnreferencedRetention: time.Hour, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.OldOrphanObjects) + require.Zero(t, report.FreshOrphanObjects) + require.Contains(t, objects.items, key) +} + +func TestReconcilerPreservesAttachmentDuringReprocessing(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, status := range []string{ + model.AttachmentQueued, + model.AttachmentProcessing, + model.AttachmentFailed, + } { + t.Run(status, func(t *testing.T) { + original := attachment("a", "attachments/a/v1/original.png", status, now.Add(-48*time.Hour), false) + original.PipelineVersion = 2 + missing := attachment("a", "attachments/a/v1/optimized.webp", status, now.Add(-48*time.Hour), false) + missing.PipelineVersion = 2 + metadata := &metadataStub{items: map[string]model.Attachment{ + original.ObjectKey: original, + missing.ObjectKey: missing, + }} + objects := &objectStub{items: map[string]model.StoredObject{ + original.ObjectKey: {Key: original.ObjectKey, LastModified: now.Add(-48 * time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedAttachmentObjects) + require.Zero(t, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, original.ObjectKey) + require.Contains(t, metadata.items, missing.ObjectKey) + require.Contains(t, objects.items, original.ObjectKey) + }) + } +} + +func TestReconcilerDeletesAllAttachmentVariantsOnce(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + keys := []string{ + "attachments/a/v1/crop.jpg", + "attachments/a/v1/optimized.webp", + "attachments/a/v1/original.png", + "attachments/a/v1/thumbnail.jpg", + } + metadata := &metadataStub{items: make(map[string]model.Attachment, len(keys))} + objects := &objectStub{items: make(map[string]model.StoredObject, len(keys))} + for _, key := range keys { + metadata.items[key] = attachment("a", key, model.AttachmentReady, old, false) + objects.items[key] = model.StoredObject{Key: key, LastModified: old} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, len(keys), report.DeletedAttachmentObjects) + require.Equal(t, 1, report.DeletedUnreferencedRows) + require.Empty(t, metadata.items) + require.Empty(t, objects.items) + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + func TestNewRejectsInvalidRetention(t *testing.T) { _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) require.Error(t, err) @@ -100,6 +192,17 @@ type metadataStub struct { appeared map[string]bool } +func (s *metadataStub) AttachmentObjectKeys(_ context.Context, attachmentID string) ([]string, error) { + var keys []string + for key, item := range s.items { + if item.ID == attachmentID { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys, nil +} + func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { keys := make([]string, 0, len(s.items)) for key := range s.items { @@ -129,6 +232,7 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string } func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { + claim := false for key, item := range s.items { if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { continue @@ -142,19 +246,20 @@ func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate m } item.Status = model.AttachmentDeleting s.items[key] = item - return true, nil + claim = true } - return false, nil + return claim, nil } func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + deleted := false for key, item := range s.items { if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { delete(s.items, key) - return true, nil + deleted = true } } - return false, nil + return deleted, nil } type objectStub struct { -- 2.54.0 From c5b804426e343b05ac307257189ffe495929c967 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 13:46:06 +0300 Subject: [PATCH 08/11] fix: expose pipeline health metrics --- REPORT.md | 10 +- compose.yaml | 4 + .../messenger/attachment/pipeline/metrics.go | 4 +- internal/messenger/httpapi/metrics.go | 9 +- internal/messenger/httpapi/server_test.go | 4 +- monitoring/grafana/dashboards/messenger.json | 1378 ++++++++++------- monitoring/postgres-queries.yml | 39 + 7 files changed, 889 insertions(+), 559 deletions(-) create mode 100644 monitoring/postgres-queries.yml diff --git a/REPORT.md b/REPORT.md index a084953a..a3422fd1 100644 --- a/REPORT.md +++ b/REPORT.md @@ -7,7 +7,7 @@ ## Запуск из чистого clone -Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Локальные пароли из `lab5.env.example` предназначены только для изолированного учебного стенда. +Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Обычные настройки приложения находятся в `config/messenger.yaml`; `.env` содержит адреса окружения, локальные credentials и параметры контейнеров. ```sh cp lab5.env.example .env @@ -97,11 +97,11 @@ Worker повторно проверяет доверенный staging объе EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments//pipelines///.`, поэтому повтор той же операции записывает тот же объект. -Concurrency и prefetch ограничены значениями `WORKER_CONCURRENCY` и `WORKER_PREFETCH`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `WORKER_MEMORY_BYTES` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. +Concurrency и prefetch задаются через `worker.concurrency` и `worker.prefetch`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `worker.memory_bytes` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. Worker продлевает lease каждые 5 секунд; потеря lease отменяет обработку. Decode и собственные циклы преобразования проверяют context во время работы. ## Доставка, retry и восстановление -Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. +Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. Опубликованные события хранятся семь суток и удаляются раз в минуту пакетами до 1000 строк, не задерживая первый dispatch после запуска. Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями. @@ -153,7 +153,7 @@ Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidenc ## Мониторинг и CI -Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard содержит RPS, status classes, p95 API latency, Go heap, PostgreSQL, Valkey, MinIO, throughput pipeline по результатам, p95 обработки, in-flight jobs, retry/reclaim, outbox publish failures и Redis broker memory/commands. +Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard разделён на блоки API, PostgreSQL, Valkey/Redis, object storage и attachment pipeline. В нём показаны timing основных ручек, CPU, соединения и транзакции PostgreSQL, cache hit/miss, память, доступность worker/dispatcher, throughput, p95 обработки, in-flight jobs, retry/reclaim, outbox backlog, Redis Streams lag, PEL, DLQ и idle consumers. CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment. @@ -170,7 +170,7 @@ CI для pull request и `main` выполняет contract check, OpenAPI vali ## Ограничения -Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Lease должен быть больше максимального времени обработки; для больших production изображений нужен heartbeat продления lease. Deadline проверяется между стадиями, но библиотечный вызов WebP не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy. +Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Decode и собственные циклы обработки прерываются по deadline; библиотечный вызов WebP обрабатывает ограниченный вариант до `1280×1280` и не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy. ## Источники и библиотеки diff --git a/compose.yaml b/compose.yaml index 48ad2db0..291eaec1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -320,6 +320,9 @@ services: DATA_SOURCE_USER: ${POSTGRES_USER} DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} PG_EXPORTER_COLLECTION_TIMEOUT: 5s + PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml + volumes: + - ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro depends_on: postgres: condition: service_healthy @@ -348,6 +351,7 @@ services: REDIS_ADDR: redis://redis-broker:6379 REDIS_USER: default REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed depends_on: redis-broker: condition: service_healthy diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go index ad76eca2..8e8d4324 100644 --- a/internal/messenger/attachment/pipeline/metrics.go +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -9,6 +9,8 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + type Metrics struct { workerInFlight prometheus.Gauge workerJobs *prometheus.CounterVec @@ -34,7 +36,7 @@ func NewMetrics() *Metrics { }, []string{"result"}), workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds", - Help: "Attachment job delivery processing duration.", Buckets: prometheus.DefBuckets, + Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets, }, []string{"result"}), workerRetries: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total", diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go index 1dae064e..f1b058e2 100644 --- a/internal/messenger/httpapi/metrics.go +++ b/internal/messenger/httpapi/metrics.go @@ -11,6 +11,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +var ( + requestDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30} + uploadDurationBuckets = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} +) + type metrics struct { requests *prometheus.CounterVec duration *prometheus.HistogramVec @@ -36,7 +41,7 @@ func newMetrics() *metrics { Subsystem: "http", Name: "request_duration_seconds", Help: "HTTP request duration in seconds.", - Buckets: prometheus.DefBuckets, + Buckets: requestDurationBuckets, }, []string{"method", "route"}), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "messenger", @@ -59,7 +64,7 @@ func newMetrics() *metrics { Namespace: "messenger", Name: "attachment_upload_duration_seconds", Help: "Attachment upload duration in seconds.", - Buckets: prometheus.DefBuckets, + Buckets: uploadDurationBuckets, }), attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "messenger", diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index 00467d34..044121d1 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -90,8 +90,8 @@ func (s attachmentServiceStub) Metadata(context.Context, string, string) (model. func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) { return attachment.Content{ Attachment: s.uploaded, - Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, - Body: io.NopCloser(strings.NewReader("content")), + Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, + Body: io.NopCloser(strings.NewReader("content")), }, s.err } diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json index a4d0a5b0..de0ef2d0 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger.json @@ -6,612 +6,893 @@ "graphTooltip": 1, "panels": [ { - "type": "timeseries", - "title": "API requests per second", - "id": 1, + "type": "row", + "title": "API and edge", + "id": 100, + "collapsed": true, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "targets": [ - { - "expr": "sum(rate(messenger_http_requests_total[1m]))", - "legendFormat": "total", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "API RPS by replica", - "id": 2, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "targets": [ - { - "expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", - "legendFormat": "{{instance}}", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "API responses by status class", - "id": 3, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "targets": [ - { - "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", - "legendFormat": "{{status_class}}", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Healthy API replicas", - "id": 4, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "targets": [ - { - "expr": "sum(up{job=\"messenger\"})", - "legendFormat": "replicas", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "API latency p95 by replica and route", - "id": 5, - "gridPos": { - "h": 8, + "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 0 }, - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "targets": [ + "panels": [ { - "expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", - "legendFormat": "{{instance}} {{route}}", - "refId": "A" + "type": "timeseries", + "title": "API requests by route", + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API responses by status class", + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API latency p95 by route", + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API replicas and requests in flight", + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "expr": "sum(up{job=\"messenger\"})", + "legendFormat": "healthy replicas", + "refId": "A" + }, + { + "expr": "sum(messenger_http_requests_in_flight)", + "legendFormat": "in flight", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API CPU usage", + "id": 5, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{job=\"messenger\"}[1m]))", + "legendFormat": "API", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Nginx requests and connections", + "id": 6, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(nginx_http_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "A" + }, + { + "expr": "sum(nginx_connections_active)", + "legendFormat": "active connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } } ] }, { - "type": "timeseries", - "title": "API in flight", - "id": 6, + "type": "row", + "title": "PostgreSQL", + "id": 101, + "collapsed": true, "gridPos": { - "h": 8, - "w": 8, + "h": 1, + "w": 24, "x": 0, - "y": 24 + "y": 25 }, - "targets": [ + "panels": [ { - "expr": "messenger_http_requests_in_flight", - "legendFormat": "{{instance}}", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Go heap by replica", - "id": 7, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 24 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" + "type": "timeseries", + "title": "PostgreSQL availability and connections", + "id": 7, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "expr": "up{job=\"postgres\"}", + "legendFormat": "up", + "refId": "A" + }, + { + "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", + "legendFormat": "connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, - "overrides": [] - }, - "targets": [ { - "expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", - "legendFormat": "{{instance}}", - "refId": "A" + "type": "timeseries", + "title": "PostgreSQL transactions", + "id": 8, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "targets": [ + { + "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m]))", + "legendFormat": "commit/s", + "refId": "A" + }, + { + "expr": "sum(rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m]))", + "legendFormat": "rollback/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL database size", + "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname=\"messenger\"}", + "legendFormat": "messenger", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment jobs by status", + "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_count", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Oldest attachment job", + "id": 11, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_oldest_age_seconds", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Pending outbox", + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_outbox_pending", + "legendFormat": "events", + "refId": "A" + }, + { + "expr": "messenger_outbox_oldest_pending_age_seconds", + "legendFormat": "oldest age, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } } ] }, { - "type": "timeseries", - "title": "Idempotency replays", - "id": 8, + "type": "row", + "title": "Valkey and Redis", + "id": 102, + "collapsed": true, "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 24 - }, - "targets": [ - { - "expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", - "legendFormat": "replays/s", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Nginx connections and requests", - "id": 9, - "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 32 + "y": 50 }, - "targets": [ + "panels": [ { - "expr": "nginx_connections_active", - "legendFormat": "active connections", - "refId": "A" + "type": "timeseries", + "title": "Valkey and Redis availability", + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "targets": [ + { + "expr": "up{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "up{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { - "expr": "rate(nginx_http_requests_total[1m])", - "legendFormat": "requests/s", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "PostgreSQL connections", - "id": 10, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "targets": [ + "type": "timeseries", + "title": "Valkey and Redis memory", + "id": 14, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, { - "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", - "legendFormat": "connections", - "refId": "A" + "type": "timeseries", + "title": "Valkey cache hits and misses", + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_keyspace_hits_total{job=\"valkey\"}[1m]))", + "legendFormat": "hits/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_keyspace_misses_total{job=\"valkey\"}[1m]))", + "legendFormat": "misses/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis commands", + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_commands_processed_total{job=\"valkey\"}[1m]))", + "legendFormat": "Valkey commands/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_commands_processed_total{job=\"redis-broker\"}[1m]))", + "legendFormat": "Redis commands/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } } ] }, { - "type": "timeseries", - "title": "PostgreSQL transactions", - "id": 11, + "type": "row", + "title": "Object storage", + "id": 103, + "collapsed": true, "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 40 + "y": 67 }, - "targets": [ + "panels": [ { - "expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", - "legendFormat": "commit/s", - "refId": "A" + "type": "timeseries", + "title": "MinIO availability", + "id": 17, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 68 + }, + "targets": [ + { + "expr": "up{job=\"minio\"}", + "legendFormat": "up", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { - "expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", - "legendFormat": "rollback/s", - "refId": "B" + "type": "timeseries", + "title": "MinIO objects and requests", + "id": 18, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_object_total)", + "legendFormat": "objects", + "refId": "A" + }, + { + "expr": "sum(rate(minio_s3_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "B" + }, + { + "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", + "legendFormat": "errors/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO storage", + "id": 19, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_total_bytes)", + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(minio_cluster_capacity_usable_free_bytes)", + "legendFormat": "free", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } } ] }, { - "type": "timeseries", - "title": "PostgreSQL database size", - "id": 12, + "type": "row", + "title": "Attachment pipeline", + "id": 104, + "collapsed": true, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "pg_database_size_bytes{datname=\"messenger\"}", - "legendFormat": "database", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Valkey memory", - "id": 13, - "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 48 + "y": 76 }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "targets": [ + "panels": [ { - "expr": "redis_memory_used_bytes{job=\"valkey\"}", - "legendFormat": "used", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Valkey keys and commands", - "id": 14, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "targets": [ - { - "expr": "sum(redis_db_keys{job=\"valkey\"})", - "legendFormat": "keys", - "refId": "A" + "type": "timeseries", + "title": "Pipeline availability and consumers", + "id": 20, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "targets": [ + { + "expr": "sum(up{job=\"attachment-worker\"})", + "legendFormat": "workers up", + "refId": "A" + }, + { + "expr": "sum(up{job=\"outbox-dispatcher\"})", + "legendFormat": "dispatcher up", + "refId": "B" + }, + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "active consumers", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { - "expr": "rate(redis_commands_processed_total{job=\"valkey\"}[1m])", - "legendFormat": "commands/s", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment uploads by result", - "id": 15, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "targets": [ - { - "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", - "legendFormat": "{{result}}", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment upload latency p95", - "id": 16, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", - "legendFormat": "p95", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment upload traffic", - "id": 17, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 64 - }, - "fieldConfig": { - "defaults": { - "unit": "Bps" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", - "legendFormat": "bytes/s", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "MinIO objects", - "id": 18, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 64 - }, - "targets": [ - { - "expr": "sum(minio_cluster_usage_object_total)", - "legendFormat": "objects", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "MinIO requests and errors", - "id": 19, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 64 - }, - "targets": [ - { - "expr": "sum(rate(minio_s3_requests_total[1m]))", - "legendFormat": "requests/s", - "refId": "A" + "type": "timeseries", + "title": "Attachment upload results", + "id": 21, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", + "legendFormat": "bytes/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + } }, { - "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", - "legendFormat": "errors/s", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "MinIO availability", - "id": 20, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 72 - }, - "targets": [ - { - "expr": "up{job=\"minio\"}", - "legendFormat": "up", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "MinIO storage", - "id": 21, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 72 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "sum(minio_cluster_usage_total_bytes)", - "legendFormat": "used", - "refId": "A" + "type": "timeseries", + "title": "Worker throughput by result", + "id": 22, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } }, { - "expr": "sum(minio_cluster_capacity_usable_free_bytes)", - "legendFormat": "free", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment pipeline throughput", - "id": 22, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 80 - }, - "fieldConfig": { - "defaults": { - "unit": "ops" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", - "legendFormat": "{{result}}", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment worker latency and concurrency", - "id": 23, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 80 - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", - "legendFormat": "p95", - "refId": "A" + "type": "timeseries", + "title": "Worker processing latency p95", + "id": 23, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", + "legendFormat": "upload p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } }, { - "expr": "sum(messenger_attachment_worker_jobs_in_flight)", - "legendFormat": "in flight", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "Attachment retries and reclaimed deliveries", - "id": 24, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 88 - }, - "fieldConfig": { - "defaults": { - "unit": "ops" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", - "legendFormat": "retries", - "refId": "A" + "type": "timeseries", + "title": "Worker jobs in flight", + "id": 24, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 93 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { - "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", - "legendFormat": "reclaimed", - "refId": "B" - } - ] - }, - { - "type": "timeseries", - "title": "Transactional outbox", - "id": 25, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 88 - }, - "fieldConfig": { - "defaults": { - "unit": "ops" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "sum(rate(messenger_outbox_published_total[1m]))", - "legendFormat": "published", - "refId": "A" + "type": "timeseries", + "title": "Retries and reclaimed deliveries", + "id": 25, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", + "legendFormat": "retries/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", + "legendFormat": "reclaimed/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } }, { - "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", - "legendFormat": "publish failures", - "refId": "B" + "type": "timeseries", + "title": "Outbox dispatcher", + "id": 26, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_outbox_published_total[1m]))", + "legendFormat": "published/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", + "legendFormat": "failures/s", + "refId": "B" + }, + { + "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", + "legendFormat": "recovered/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } }, { - "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", - "legendFormat": "recovered leases", - "refId": "C" - } - ] - }, - { - "type": "timeseries", - "title": "Redis broker memory", - "id": 26, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 96 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", - "legendFormat": "used", - "refId": "A" - } - ] - }, - { - "type": "timeseries", - "title": "Redis broker commands and keys", - "id": 27, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 96 - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "targets": [ - { - "expr": "rate(redis_commands_processed_total{job=\"redis-broker\"}[1m])", - "legendFormat": "commands", - "refId": "A" + "type": "timeseries", + "title": "Redis Stream lag, PEL and DLQ", + "id": 27, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_lag{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "lag", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_messages_pending{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "PEL", + "refId": "B" + }, + { + "expr": "max(redis_stream_length{job=\"redis-broker\",stream=\"attachment-jobs-failed\"})", + "legendFormat": "DLQ", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { - "expr": "sum(redis_db_keys{job=\"redis-broker\"})", - "legendFormat": "keys", - "refId": "B" + "type": "timeseries", + "title": "Redis Stream consumers", + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "consumers", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_consumer_idle_seconds{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "max idle, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } } ] } @@ -621,8 +902,7 @@ "tags": [ "highload", "lab5", - "nginx", - "failover" + "pipeline" ], "templating": { "list": [] @@ -634,5 +914,5 @@ "timezone": "browser", "title": "Messenger lab5 overview", "uid": "messenger-lab5", - "version": 2 + "version": 3 } diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml new file mode 100644 index 00000000..7666281c --- /dev/null +++ b/monitoring/postgres-queries.yml @@ -0,0 +1,39 @@ +messenger_attachment_jobs: + query: | + WITH statuses(status) AS ( + VALUES ('queued'), ('processing'), ('failed') + ) + SELECT + statuses.status, + COUNT(j.id)::bigint AS count, + COALESCE(EXTRACT(EPOCH FROM (clock_timestamp() - MIN(j.created_at))), 0)::double precision AS oldest_age_seconds + FROM statuses + LEFT JOIN attachment_jobs j ON j.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: Attachment job status. + - count: + usage: GAUGE + description: Attachment jobs in this status. + - oldest_age_seconds: + usage: GAUGE + description: Age of the oldest attachment job in this status. + +messenger_outbox: + query: | + SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::bigint AS pending, + COALESCE( + EXTRACT(EPOCH FROM (clock_timestamp() - MIN(created_at) FILTER (WHERE published_at IS NULL))), + 0 + )::double precision AS oldest_pending_age_seconds + FROM outbox_events + metrics: + - pending: + usage: GAUGE + description: Unpublished outbox events. + - oldest_pending_age_seconds: + usage: GAUGE + description: Age of the oldest unpublished outbox event. -- 2.54.0 From 9ccae0c0c3a2753176cd9e376865494e8c45a10d Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 14:01:01 +0300 Subject: [PATCH 09/11] fix: preserve reconciliation dry runs --- internal/reconcile/reconcile.go | 4 +++ internal/reconcile/reconcile_test.go | 48 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index beb843c9..9c4ee8c5 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -283,6 +283,7 @@ func (r *Reconciler) handleMissingObject( return r.deletePair(ctx, metadata, cutoff, report) } if metadata.Status == model.AttachmentDeleting { + report.DeletingAttachments++ return r.deleteClaimedMetadata(ctx, metadata, report) } if metadata.CreatedAt.After(cutoff) { @@ -308,6 +309,9 @@ func (r *Reconciler) handleMissingObject( } func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) if err != nil { return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 8e1c1b3f..8cbef1f6 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -82,6 +82,54 @@ func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { require.Contains(t, objects.items, "attachments/a/original") } +func TestReconcilerDeletingAttachmentRespectsApplyMode(t *testing.T) { + now := time.Now().UTC() + key := "attachments/a/original" + tests := []struct { + name string + apply bool + objectPresent bool + }{ + {name: "dry run with object", objectPresent: true}, + {name: "dry run without object"}, + {name: "apply with object", apply: true, objectPresent: true}, + {name: "apply without object", apply: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentDeleting, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: make(map[string]model.StoredObject)} + if test.objectPresent { + objects.items[key] = model.StoredObject{Key: key, LastModified: now.Add(-time.Hour)} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: test.apply, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, report.DeletingAttachments) + var expectedDeletedRows, expectedDeletedObjects int + if test.apply { + expectedDeletedRows = 1 + } + if test.apply && test.objectPresent { + expectedDeletedObjects = 1 + } + require.Equal(t, expectedDeletedRows, report.DeletedUnreferencedRows) + require.Equal(t, expectedDeletedObjects, report.DeletedAttachmentObjects) + _, metadataExists := metadata.items[key] + require.Equal(t, !test.apply, metadataExists) + _, objectExists := objects.items[key] + require.Equal(t, test.objectPresent && !test.apply, objectExists) + }) + } +} + func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) { now := time.Now().UTC() key := "staging/a/original" -- 2.54.0 From f8a624a40bf29c95718e3545b3c82bd8e36c3cd0 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 14:01:03 +0300 Subject: [PATCH 10/11] fix: validate message request fields --- internal/messenger/httpapi/handlers.go | 10 +++--- internal/messenger/httpapi/request.go | 20 +++++++++++ internal/messenger/httpapi/server_test.go | 41 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go index b1e618fc..1b41574d 100644 --- a/internal/messenger/httpapi/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -208,18 +208,16 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { if !valid { return invalidRequest() } - var input struct { - Text string `json:"text"` - AttachmentIDs []string `json:"attachment_ids"` - } + var input createMessageRequest if err := readJSON(w, r, &input); err != nil { return err } - if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) { + text, attachmentIDs, valid := input.values() + if !valid || !validMessage(text, attachmentIDs) || !canonicalAttachmentIDs(attachmentIDs) { return invalidRequest() } message, replay, err := s.messageRepository.CreateMessage( - r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL, + r.Context(), chatID, user.ID, text, attachmentIDs, key, s.idempotencyTTL, ) if errors.Is(err, model.ErrNotFound) { return notFound() diff --git a/internal/messenger/httpapi/request.go b/internal/messenger/httpapi/request.go index c0a51591..63cb6944 100644 --- a/internal/messenger/httpapi/request.go +++ b/internal/messenger/httpapi/request.go @@ -20,6 +20,26 @@ const maxRequestBytes = 1 << 20 var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) +type createMessageRequest struct { + Text json.RawMessage `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` +} + +func (r createMessageRequest) values() (string, []string, bool) { + var text *string + if len(r.Text) == 0 || json.Unmarshal(r.Text, &text) != nil || text == nil { + return "", nil, false + } + if len(r.AttachmentIDs) == 0 { + return *text, nil, true + } + var attachmentIDs *[]string + if json.Unmarshal(r.AttachmentIDs, &attachmentIDs) != nil || attachmentIDs == nil { + return "", nil, false + } + return *text, *attachmentIDs, true +} + func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError { mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index 044121d1..d2e0a217 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -263,6 +263,47 @@ func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() { } } +func (s *MessengerSuite) TestMessageCreationValidatesFieldPresenceAndTypes() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + attachmentID := "00000000-0000-0000-0000-000000000002" + invalidBodies := []string{ + `{"attachment_ids":["` + attachmentID + `"]}`, + `{"text":null,"attachment_ids":["` + attachmentID + `"]}`, + `{"text":"x","attachment_ids":null}`, + `{"text":1}`, + `{"text":"x","attachment_ids":{}}`, + } + for _, body := range invalidBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusBadRequest, response.Code, body) + } + + validBodies := []string{ + `{"text":"message"}`, + `{"text":"message","attachment_ids":[]}`, + `{"text":"","attachment_ids":["` + attachmentID + `"]}`, + } + for _, body := range validBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusCreated, response.Code, body) + } +} + +func createMessageRequestResponse(server *Server, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", + strings.NewReader(body), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "message-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} + func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() { t := s.T() store := messengerStoreStub{ -- 2.54.0 From 4e1a74d9fff03b66ac30b0da9d4a6e9004a2b2a0 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 14:08:14 +0300 Subject: [PATCH 11/11] test: cover reconciliation dry runs --- internal/postgres/store_integration_test.go | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index e478d9be..48675230 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -18,6 +18,7 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" ) type PostgresSuite struct { @@ -412,6 +413,43 @@ func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { s.Equal(model.AttachmentReady, visible.Status) } +func (s *PostgresSuite) TestReconciliationDryRunPreservesDeletingAttachment() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "deleting.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("e", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + + reconciler, err := reconcile.New(s.store, emptyObjectStore{}, reconcile.Config{ + Prefix: "attachments/", PageSize: 1000, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, + }) + s.Require().NoError(err) + report, err := reconciler.Run(context.Background()) + s.Require().NoError(err) + s.Zero(report.DeletedUnreferencedRows) + + var status string + err = s.pool.QueryRow(context.Background(), "SELECT status FROM attachments WHERE id = $1", pending.ID).Scan(&status) + s.Require().NoError(err) + s.Equal(model.AttachmentDeleting, status) +} + +type emptyObjectStore struct{} + +func (emptyObjectStore) List(context.Context, string, string, int) ([]model.StoredObject, string, error) { + return nil, "", nil +} + +func (emptyObjectStore) Delete(context.Context, string) error { + return errors.New("unexpected object deletion") +} + func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { owner, firstChatID := createOwnerAndChat(s.T(), s.store) secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) -- 2.54.0