From d761861def80096fe0897a3b693eda6ff8d0c0c7 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:36:59 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D1=8C=D1=82?= =?UTF-8?q?=D0=B5=20S3-=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D1=81=20=D0=B4=D0=B5=D0=B4=D1=83=D0=BF=D0=BB=D0=B8=D0=BA=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D0=B5=D0=B9=20=D0=B8=20=D0=B1=D0=B5=D0=B7=D0=BE?= =?UTF-8?q?=D0=BF=D0=B0=D1=81=D0=BD=D0=BE=D0=B9=20=D1=81=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=BE=D0=B9=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8?= =?UTF-8?q?=D1=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- Dockerfile | 4 +- cmd/reconcile/main.go | 99 +++ cmd/s3init/main.go | 30 + cmd/server/main.go | 46 ++ compose.yaml | 137 +++- go.mod | 25 +- go.sum | 36 +- infra/haproxy/haproxy.cfg | 2 + infra/s3/README.md | 77 ++ infra/s3/seaweedfs.options | 31 + internal/app/app.go | 8 + internal/app/app_test.go | 5 +- internal/app/attachment_test.go | 171 +++++ internal/attachment/attachment.go | 22 + internal/attachment/blob.go | 25 + internal/attachment/handler.go | 121 +++ internal/attachment/lock.go | 23 + internal/attachment/reconcile.go | 479 ++++++++++++ internal/attachment/reconcile_test.go | 708 ++++++++++++++++++ internal/attachment/schema_test.go | 27 + internal/attachment/service.go | 172 +++++ internal/attachment/service_test.go | 270 +++++++ internal/attachment/store/postgres.go | 79 ++ internal/attachment/validation.go | 51 ++ internal/attachment/validation_test.go | 93 +++ internal/chat/attachment_handler_test.go | 72 ++ internal/chat/attachment_http_test.go | 125 ++++ internal/chat/attachment_message_test.go | 66 ++ internal/chat/handler.go | 68 +- internal/chat/message.go | 49 +- internal/chat/service.go | 5 +- internal/chat/service_test.go | 5 +- internal/chat/store/attachments_test.go | 312 ++++++++ internal/chat/store/postgres.go | 128 +++- internal/chat/store/postgres_test.go | 3 +- internal/platform/config/config.go | 11 +- internal/platform/config/config_test.go | 15 +- internal/platform/config/s3.go | 55 ++ internal/platform/config/s3_test.go | 27 + internal/platform/httpx/middleware.go | 5 + internal/platform/s3store/bootstrap_test.go | 142 ++++ internal/platform/s3store/integration_test.go | 270 +++++++ internal/platform/s3store/readiness.go | 114 +++ internal/platform/s3store/readiness_test.go | 48 ++ internal/platform/s3store/store.go | 198 +++++ internal/platform/s3store/store_test.go | 171 +++++ internal/server/server.go | 4 +- internal/server/server_test.go | 177 +++++ internal/testenv/s3.go | 206 +++++ migrations/00003_attachments.sql | 35 + scripts/init-secrets.sh | 80 +- scripts/restart.sh | 5 + scripts/wait-ready.sh | 22 + 54 files changed, 5061 insertions(+), 102 deletions(-) create mode 100644 cmd/reconcile/main.go create mode 100644 cmd/s3init/main.go create mode 100644 infra/s3/README.md create mode 100644 infra/s3/seaweedfs.options create mode 100644 internal/app/attachment_test.go create mode 100644 internal/attachment/attachment.go create mode 100644 internal/attachment/blob.go create mode 100644 internal/attachment/handler.go create mode 100644 internal/attachment/lock.go create mode 100644 internal/attachment/reconcile.go create mode 100644 internal/attachment/reconcile_test.go create mode 100644 internal/attachment/schema_test.go create mode 100644 internal/attachment/service.go create mode 100644 internal/attachment/service_test.go create mode 100644 internal/attachment/store/postgres.go create mode 100644 internal/attachment/validation.go create mode 100644 internal/attachment/validation_test.go create mode 100644 internal/chat/attachment_handler_test.go create mode 100644 internal/chat/attachment_http_test.go create mode 100644 internal/chat/attachment_message_test.go create mode 100644 internal/chat/store/attachments_test.go create mode 100644 internal/platform/config/s3.go create mode 100644 internal/platform/config/s3_test.go create mode 100644 internal/platform/s3store/bootstrap_test.go create mode 100644 internal/platform/s3store/integration_test.go create mode 100644 internal/platform/s3store/readiness.go create mode 100644 internal/platform/s3store/readiness_test.go create mode 100644 internal/platform/s3store/store.go create mode 100644 internal/platform/s3store/store_test.go create mode 100644 internal/testenv/s3.go create mode 100644 migrations/00003_attachments.sql create mode 100644 scripts/wait-ready.sh diff --git a/.env.example b/.env.example index d99d26a2..85dc5bfe 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,12 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_RECONCILE_ACCESS_KEY_ID_FILE=/run/secrets/s3_reconcile_access_key_id +S3_RECONCILE_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_reconcile_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/Dockerfile b/Dockerfile index 2a0b1902..a3e94d88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY cmd ./cmd COPY internal ./internal COPY migrations ./migrations RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init # The load generator gets an image of its own; the API image does not carry it. FROM build AS loadgen-build @@ -25,7 +25,7 @@ ENTRYPOINT ["/usr/local/bin/loadgen"] # Last, so that `build: .` builds it; it also runs the migrate service. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app -COPY --from=build /out/server /out/migrate /usr/local/bin/ +COPY --from=build /out/server /out/migrate /out/reconcile /out/s3init /usr/local/bin/ USER app ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 00000000..b7090a83 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,99 @@ +// Command reconcile reports attachment inconsistencies as JSON lines. Mutations +// require --apply; --deep also streams referenced objects to verify SHA256. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "messenger/internal/attachment" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, out, errOut io.Writer) int { + flags := flag.NewFlagSet("reconcile", flag.ContinueOnError) + flags.SetOutput(errOut) + apply := flags.Bool("apply", false, "apply repairs (default: read-only dry run)") + deep := flags.Bool("deep", false, "stream referenced objects and verify SHA256 and size") + grace := flags.Duration("grace", time.Hour, "minimum file/object age; must be at least 2m") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() != 0 || *grace < 2*time.Minute { + fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m") + return 2 + } + dbConfig, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + // Compose mounts the reconciler identity at the standard credential paths. + s3Config, err := config.LoadS3(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 1 + } + defer db.Close() + r, err := attachment.NewReconciler(db, blobs, *grace) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 2 + } + // A broken output stream cancels the scan so repairs cannot continue after + // the caller loses the audit trail. Print partial statistics on scan errors. + scanCtx, cancel := context.WithCancel(ctx) + defer cancel() + encoder := json.NewEncoder(out) + var outputErr error + stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) { + if outputErr == nil { + outputErr = encoder.Encode(f) + if outputErr != nil { + cancel() + } + } + }) + if outputErr == nil { + outputErr = encoder.Encode(stats) + } + if outputErr != nil { + fmt.Fprintln(errOut, "reconcile output:", outputErr) + return 1 + } + if runErr != nil { + fmt.Fprintln(errOut, "reconcile:", runErr) + return 1 + } + return 0 +} diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go new file mode 100644 index 00000000..db308002 --- /dev/null +++ b/cmd/s3init/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "fmt" + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" + "os" + "time" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "bucket bootstrap failed:", err) + os.Exit(1) + } +} +func run() error { + cfg, err := config.LoadS3(os.LookupEnv) + if err != nil { + return err + } + s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return s.Bootstrap(ctx) +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 9d4f4b94..cb4aec4a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,10 +17,12 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "messenger/internal/app" + "messenger/internal/attachment" "messenger/internal/chat" "messenger/internal/platform/config" "messenger/internal/platform/ident" "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" "messenger/internal/platform/valkey" "messenger/internal/server" ) @@ -59,6 +61,26 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } defer sessions.Close() + var blobs attachment.Blobs + var reconciler *attachment.Reconciler + if cfg.S3.Endpoint != "" { + blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey) + if err != nil { + return err + } + reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv)) + if e != nil { + return e + } + reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey) + if e != nil { + return e + } + reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour) + if err != nil { + return err + } + } ln, err := net.Listen("tcp", cfg.Addr()) if err != nil { @@ -77,6 +99,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) MaxBodyBytes: cfg.MaxBodyBytes, InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, + Blobs: blobs, + MaxUploadBytes: cfg.S3.MaxBytes, + MaxImagePixels: cfg.S3.MaxPixels, }) var debugLn net.Listener if cfg.DebugAddr != "" { @@ -89,6 +114,27 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) ctx, cancel := context.WithCancel(ctx) var background sync.WaitGroup background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if reconciler != nil { + background.Go(func() { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + scan, cancel := context.WithTimeout(ctx, 2*time.Minute) + stats, err := reconciler.Run(scan, true, false, nil) + cancel() + if err != nil { + log.Warn("attachment cleanup incomplete", "error", err) + } else { + log.Info("attachment cleanup", "scanned_files", stats.ScannedFiles, "repaired", stats.Repaired) + } + } + } + }) + } if debugLn != nil { background.Go(func() { if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { diff --git a/compose.yaml b/compose.yaml index 5b3e7b7e..bc08ab33 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,5 @@ # Local stand: HAProxy in front of interchangeable API replicas over shared -# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data +# PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data # network has no way out. The load-test knobs are in compose.bench.yaml. name: lab4 @@ -38,11 +38,24 @@ services: SESSION_STORE_URL_FILE: /run/secrets/session_store_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - # Go does not read the cgroup memory limit. - GOMEMLIMIT: 230MiB + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + # The background reconciler uses a separate identity with List rights. + S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id + S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key + # One decoder per replica. Progressive JPEG retains coefficient buffers; + # a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer. + GOMEMLIMIT: 1536MiB secrets: - database_url - session_store_url + - s3_access_key_id + - s3_secret_access_key + - s3_reconcile_access_key_id + - s3_reconcile_secret_access_key networks: - edge - data @@ -53,14 +66,16 @@ services: condition: service_healthy valkey: condition: service_healthy - # Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL. - stop_grace_period: 15s + s3-init: + condition: service_completed_successfully + # Binary deadline 30s plus compensation, shutdown and drain margin. + stop_grace_period: 45s deploy: replicas: ${API_REPLICAS:-2} resources: limits: cpus: "1.0" - memory: 256M + memory: 2G migrate: image: mtusi-messenger:lab4 @@ -138,9 +153,105 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M + s3: + image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0 + command: ["server", "-options=/etc/seaweedfs/lab.options"] + environment: + GOMEMLIMIT: 460MiB + secrets: + - s3_identities + volumes: + - ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro + # Both volume data and the image's /data/filerldb2 metadata live here. + - s3data:/data + networks: + - data + healthcheck: + # A reachable anonymous request must be refused, including before init. + test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + start_interval: 1s + stop_grace_period: 30s + pids_limit: 256 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + pids: 256 + + s3-init: + build: . + image: mtusi-messenger:lab4 + entrypoint: ["/usr/local/bin/s3init"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - source: s3_admin_access_key_id + target: s3_access_key_id + - source: s3_admin_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + s3: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + reconcile: + profiles: ["recon"] + build: . + image: mtusi-messenger:lab4 + entrypoint: ["/usr/local/bin/reconcile"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - database_url + - source: s3_reconcile_access_key_id + target: s3_access_key_id + - source: s3_reconcile_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + volumes: pgdata: valkeydata: + s3data: networks: edge: {} @@ -156,3 +267,17 @@ secrets: file: ./secrets/postgres_password valkey_auth: file: ./secrets/valkey_auth.conf + s3_identities: + file: ./secrets/s3_identities.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_reconcile_access_key_id: + file: ./secrets/s3_reconcile_access_key_id + s3_reconcile_secret_access_key: + file: ./secrets/s3_reconcile_secret_access_key diff --git a/go.mod b/go.mod index 9c6fb13e..5439a0c6 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,20 @@ module messenger go 1.27.1 require ( + github.com/docker/docker v28.3.3+incompatible + github.com/docker/go-connections v0.6.0 github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 github.com/pressly/goose/v3 v3.27.0 github.com/redis/go-redis/v9 v9.23.0 github.com/testcontainers/testcontainers-go v0.39.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 golang.org/x/crypto v0.57.0 + google.golang.org/grpc v1.84.0 + google.golang.org/protobuf v1.36.12 ) require ( @@ -31,9 +36,8 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/creack/pty v1.1.24 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.3.3+incompatible // indirect - github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.1 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect @@ -54,12 +58,15 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect github.com/mfridman/interpolate v0.0.2 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect @@ -72,20 +79,24 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pelletier/go-toml/v2 v2.4.3 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sethvargo/go-retry v0.4.0 // indirect github.com/shirou/gopsutil/v4 v4.26.6 // indirect - github.com/sirupsen/logrus v1.9.4 // indirect + github.com/sirupsen/logrus v1.10.2 // indirect github.com/stretchr/testify v1.12.1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect @@ -97,11 +108,11 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.59.0 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/grpc v1.84.0 // indirect - google.golang.org/protobuf v1.36.12 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index fb12767b..9807f1b1 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -100,8 +102,11 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -116,6 +121,12 @@ github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5L github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -145,8 +156,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -162,12 +175,14 @@ github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBP github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -180,6 +195,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= @@ -188,6 +204,8 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwN github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g= github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= @@ -202,6 +220,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= @@ -249,8 +269,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -281,6 +301,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8= @@ -290,6 +312,8 @@ google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2Q google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 129badc3..50795f51 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -55,6 +55,8 @@ frontend entry default_backend replicas backend replicas + # Binary handlers have a 30s budget, ordinary API handlers retain 5s. + http-request set-timeout server 35s if { path_reg ^/api/v1/chats/[^/]+/attachments$|^/api/v1/attachments/[^/]+/content$ } balance roundrobin option httpchk http-check send meth GET uri /health/ready diff --git a/infra/s3/README.md b/infra/s3/README.md new file mode 100644 index 00000000..cc9c0bc5 --- /dev/null +++ b/infra/s3/README.md @@ -0,0 +1,77 @@ +# Private object storage + +`s3` runs SeaweedFS 4.53 in one process. The image is pinned by digest in +`compose.yaml` and `internal/testenv/s3.go`. `seaweedfs.options` is shared by +the stand and the disposable integration-test server. + +S3 HTTP (8333) and SeaweedFS's companion gRPC (18333) listen on the private +container network. This image has no flag to disable the companion listener; +its identity-cache mutations reject remote peers, which the integration test +checks. Master, volume and filer HTTP/gRPC endpoints bind to loopback. +No host port is published. `/data` +contains both object volumes and the image's LevelDB filer metadata; the +named `s3data` volume must be retained together with PostgreSQL data. + +Run `sh scripts/init-secrets.sh` before starting the stand. It preserves the +existing PostgreSQL and Valkey credentials and creates three independent S3 +identities plus the static server configuration. Credential values are never +printed. Repeated initialization keeps all existing keys. Set `S3_BUCKET` in +the environment or the local `.env` before the first initialization; changing +it later requires a planned migration of identities and retained data. + +The server loads its static identity secret before serving requests. There is +no anonymous identity. The bootstrap identity has administrative rights and +its individual key files are mounted only in `s3-init`. The API identity has +`Read:` and `Write:`; it cannot list objects or administer +buckets. The reconciler identity adds `List:`. All three identities +are necessarily present in the S3 server's own identity configuration secret. + +`s3-init` creates the bucket idempotently and waits for storage before API +replicas start. An anonymous 403 and bucket metadata alone do not prove that +retained volume locations have registered with the elected master. Bootstrap +first reads a persistent `.health/storage-v1` marker (creating it only when +absent), then verifies a fresh write/read/delete. This startup-only barrier +uses the caller's deadline, capped at 60 seconds; ordinary S3 operations and +API readiness do not retry through it. The marker is outside the reconciler's +`staging/` and `blobs/` namespaces. `scripts/restart.sh` already runs this +bootstrap before restoring API replicas. + +The bootstrap uses `/usr/local/bin/s3init`. The optional `recon` profile runs +`/usr/local/bin/reconcile`, with reconciler credentials mapped to the standard +`S3_ACCESS_KEY_ID_FILE` and `S3_SECRET_ACCESS_KEY_FILE` paths: + +```sh +docker compose --profile recon run --rm reconcile +docker compose --profile recon run --rm reconcile --apply +``` + +The API receives its own standard S3 key files. Its background reconciliation +worker must construct a separate store from +`S3_RECONCILE_ACCESS_KEY_ID_FILE` and `S3_RECONCILE_SECRET_ACCESS_KEY_FILE`; +listing with the API identity is denied. S3 is intentionally absent from API +readiness so text traffic can remain available during an object-store outage. + +The S3 cgroup is limited to one CPU, 512 MiB, and 256 processes. Upload limits +are four concurrent requests and 64 MiB in flight; reader buffers have a +32 MiB budget and the chunk cache is disabled. Disk allocation is capped at +64 volumes of approximately 128 MiB each (with normal volume rollover +overhead); this is a lab capacity limit, not a filesystem quota. + +Each API replica has 2 GiB and `GOMEMLIMIT=1536MiB`, with one image decoder. +Pixels alone are not the full budget: progressive JPEG retains coefficients, +and a wide RGBA64 PNG keeps two large scanlines beside its pixel buffer. +The 40 MP RGB/CMYK progressive JPEG and wide-PNG measurements are recorded in +`evidence/decoder-memory-*.json`; this does not change the S3 service's 512 MiB limit. + +Verify the adapter and real server with Docker running: + +```sh +go test -race -count=1 ./internal/platform/s3store +``` + +`testenv.StartS3(ctx)` returns `Store` with reconciler rights, `APIStore`, +`Admin`, endpoint and credentials. `Recreate(ctx)` replaces only its own +container while keeping its named volume and does not recreate the bucket. +It waits for the same retained-marker and write/read/delete barrier before +returning. `Start(ctx)` does so as well. Always call `Close()` to remove the +fixture's container and volume. diff --git a/infra/s3/seaweedfs.options b/infra/s3/seaweedfs.options new file mode 100644 index 00000000..db8c1c1f --- /dev/null +++ b/infra/s3/seaweedfs.options @@ -0,0 +1,31 @@ +dir=/data +ip=127.0.0.1 +ip.bind=127.0.0.1 +master.telemetry=false +master.volumeSizeLimitMB=128 +volume.port=9340 +volume.max=64 +volume.index=leveldb +volume.fileSizeLimitMB=10 +volume.concurrentUploadLimitMB=64 +volume.concurrentDownloadLimitMB=64 +volume.readBufferSizeMB=1 +volume.preStopSeconds=1 +filer=true +filer.exposeDirectoryData=false +filer.disableDirListing=true +filer.concurrentFileUploadLimit=4 +filer.concurrentUploadLimitMB=64 +s3=true +s3.ip.bind=0.0.0.0 +s3.port=8333 +s3.config=/run/secrets/s3_identities +s3.autoCreateBucket=false +s3.iam=false +s3.port.iceberg=0 +s3.port.lance=0 +s3.concurrentFileUploadLimit=4 +s3.concurrentUploadLimitMB=64 +s3.readerCacheSizeMB=32 +s3.cacheCapacityMB=0 +s3.idleTimeout=35 diff --git a/internal/app/app.go b/internal/app/app.go index 720a9ae5..3949add5 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -12,6 +12,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + "messenger/internal/attachment" + attachmentstore "messenger/internal/attachment/store" "messenger/internal/auth" authstore "messenger/internal/auth/store" "messenger/internal/chat" @@ -42,6 +44,9 @@ type Deps struct { InstanceID string // IdempotencyTTL zero means the contract's 24 hours. IdempotencyTTL time.Duration + Blobs attachment.Blobs + MaxUploadBytes int64 + MaxImagePixels int64 } // defaultRequestTimeout stays below the server's 10s write timeout, so a @@ -76,6 +81,9 @@ func New(d Deps) *App { auth.NewHandler(sessions).Register(public, protected) user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) + attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels) + binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions)) + attachment.NewHandler(attachments).Register(protected, binary) return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 02f8a33b..a9741ecc 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "net/url" "os" + "reflect" "sort" "strings" "sync" @@ -415,11 +416,11 @@ func TestMessagesEndToEnd(t *testing.T) { } page1 := ts.history(bob.token, room.ID, "?limit=1") - if len(page1.Items) != 1 || page1.NextCursor == nil || page1.Items[0] != first { + if len(page1.Items) != 1 || page1.NextCursor == nil || !reflect.DeepEqual(page1.Items[0], first) { t.Fatalf("page1 = %+v", page1) } page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor)) - if len(page2.Items) != 1 || page2.NextCursor != nil || page2.Items[0] != second { + if len(page2.Items) != 1 || page2.NextCursor != nil || !reflect.DeepEqual(page2.Items[0], second) { t.Fatalf("page2 = %+v", page2) } diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go new file mode 100644 index 00000000..2041d5e6 --- /dev/null +++ b/internal/app/attachment_test.go @@ -0,0 +1,171 @@ +package app_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "image" + "image/jpeg" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/app" + "messenger/internal/attachment" + "messenger/internal/chat" + "messenger/internal/testenv" +) + +func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized") +} + +func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) { + ts := newTester(t) + actor := ts.user("deadline_user", "User") + ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond}) + var held []*pgxpool.Conn + for range db.Pool.Config().MaxConns { + conn, err := db.Pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + held = append(held, conn) + } + defer func() { + for _, conn := range held { + conn.Release() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx) + req.Header.Set("Authorization", "Bearer "+actor.token) + rec := httptest.NewRecorder() + started := time.Now() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { + t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + } +} + +func (ts *tester) binary(method, path, token, mime string, data []byte, length int64) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(data)) + req.ContentLength = length + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if mime != "" { + req.Header.Set("Content-Type", mime) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + res := response{status: rec.Code, requestID: rec.Header().Get("X-Request-Id"), challenge: rec.Header().Get("WWW-Authenticate"), raw: rec.Body.Bytes()} + if res.requestID == "" || rec.Header().Get("X-Instance-Id") == "" { + ts.t.Fatal("binary response missing correlation headers") + } + return res +} + +func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) { + ctx := context.Background() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + ts := newTester(t) + ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, Blobs: s3.APIStore}) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "files", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/attachments?filename=" + url.QueryEscape("фото.png") + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))) + ts.expect(res, 201, "") + a := decode[attachment.Response](t, res) + digest := sha256.Sum256(data) + if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "ready" || a.ErrorCode != nil || len(a.Variants) != 0 { + t.Fatalf("attachment: %+v", a) + } + base := "/api/v1/attachments/" + a.ID + for _, suffix := range []string{"", "/content"} { + ts.expect(ts.do("GET", base+suffix, eve.token, nil, nil), 404, "not_found") + } + ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "") + got := ts.binary("GET", base+"/content", bob.token, "", nil, 0) + ts.expect(got, 200, "") + if !bytes.Equal(got.raw, data) { + t.Fatal("download bytes changed") + } + ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 400, "invalid_request") + msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()) + ts.expect(msg, 201, "") + if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID { + t.Fatal("lost attachment on message") + } + other := ts.chat(alice.token, "other", alice.ID) + ts.expect(ts.do("POST", "/api/v1/chats/"+other.ID+"/messages", alice.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()), 404, "not_found") + for _, tc := range []struct { + name, mime string + data []byte + length int64 + want int + }{ + {"MIME spoof", "image/jpeg", data, int64(len(data)), 415}, + {"broken body", "image/png", data[:len(data)/2], int64(len(data) / 2), 415}, + {"unsupported", "image/gif", data, int64(len(data)), 415}, + {"oversize", "image/png", nil, 10485761, 413}, + {"unknown length", "image/png", data, -1, 400}, + {"empty", "image/png", nil, 0, 400}, + } { + t.Run(tc.name, func(t *testing.T) { + r := ts.binary("POST", path, alice.token, tc.mime, tc.data, tc.length) + if r.status != tc.want { + t.Fatalf("got %d want %d: %s", r.status, tc.want, r.raw) + } + }) + } + for _, name := range []string{"../x.png", `a\b.png`, "bad\nname", strings.Repeat("я", 256)} { + ts.expect(ts.binary("POST", "/api/v1/chats/"+room.ID+"/attachments?filename="+url.QueryEscape(name), alice.token, "image/png", data, int64(len(data))), 400, "invalid_request") + } + var j bytes.Buffer + if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 201, "") + ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found") + // An interrupted read is never a 500 or a successfully published identity. + before := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&before); err != nil { + t.Fatal(err) + } + short := ts.binary("POST", path, alice.token, "image/png", data[:10], int64(len(data))) + if short.status != 400 && short.status != 503 { + t.Fatalf("interrupted body: %d %s", short.status, short.raw) + } + after := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after { + t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err) + } + if err := s3.Stop(ctx); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))), 503, "unavailable") + ts.expect(ts.do("GET", base+"/content", bob.token, nil, nil), 503, "unavailable") + ts.expect(ts.do("GET", "/health/ready", "", nil, nil), 200, "") + ts.message(alice.token, room.ID, fmt.Sprintf("text works with S3 down %d", time.Now().UnixNano())) +} diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go new file mode 100644 index 00000000..9a5cd171 --- /dev/null +++ b/internal/attachment/attachment.go @@ -0,0 +1,22 @@ +package attachment + +import ( + "context" + "github.com/google/uuid" + "time" +) + +type Attachment struct { + ID, ChatID, OwnerID uuid.UUID + Filename string + FileHash []byte + SizeBytes int64 + MIMEType, Status string + CreatedAt time.Time +} + +type Repository interface { + IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error) + Publish(context.Context, Attachment, string) error + GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error) +} diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go new file mode 100644 index 00000000..c211b36e --- /dev/null +++ b/internal/attachment/blob.go @@ -0,0 +1,25 @@ +package attachment + +import ( + "context" + "errors" + "io" + "time" +) + +var ErrObjectMissing = errors.New("object missing") + +type Object struct { + Key string + Size int64 + Modified time.Time +} + +type Blobs interface { + Put(context.Context, string, io.Reader, int64, string) error + Open(context.Context, string) (io.ReadCloser, error) + Stat(context.Context, string) (Object, error) + Copy(context.Context, string, string) error + Delete(context.Context, string) error + List(context.Context, string, string, int) ([]Object, error) +} diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go new file mode 100644 index 00000000..6420519f --- /dev/null +++ b/internal/attachment/handler.go @@ -0,0 +1,121 @@ +package attachment + +import ( + "encoding/hex" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" +) + +type Handler struct{ s *Service } + +func NewHandler(s *Service) *Handler { return &Handler{s} } +func (h *Handler) Register(protected, binary gin.IRouter) { + binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) + protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata)) + binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) +} + +type Response struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func toResponse(a Attachment) Response { + return Response{ + ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt), + } +} + +func (h *Handler) upload(c *gin.Context, actor uuid.UUID) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + httpx.Error(c, apperr.Invalid("chat_id must be a UUID")) + return + } + names := c.Request.URL.Query()["filename"] + if len(names) != 1 { + httpx.Error(c, apperr.Invalid("one filename is required")) + return + } + if err := ValidateFilename(names[0]); err != nil { + httpx.Error(c, err) + return + } + media, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || len(params) > 0 { + httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes) + a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toResponse(a)) +} +func attachmentID(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("attachment_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("attachment_id must be a UUID") + } + return id, nil +} +func (h *Handler) metadata(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + a, err := h.s.Get(c.Request.Context(), actor, id) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toResponse(a)) +} +func (h *Handler) content(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + variant, present := c.GetQuery("variant") + if present && variant != "original" { + httpx.Error(c, apperr.Invalid("variant must be original")) + return + } + a, r, err := h.s.Content(c.Request.Context(), actor, id) + if err != nil { + httpx.Error(c, err) + return + } + defer r.Close() + c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Type", a.MIMEType) + c.Header("Content-Length", strconv.FormatInt(a.SizeBytes, 10)) + c.Status(http.StatusOK) + if _, err := io.CopyN(c.Writer, r, a.SizeBytes); err != nil { + slog.Error("download interrupted", "request_id", httpx.RequestID(c), "error", err) + } +} diff --git a/internal/attachment/lock.go b/internal/attachment/lock.go new file mode 100644 index 00000000..31b73342 --- /dev/null +++ b/internal/attachment/lock.go @@ -0,0 +1,23 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + + "github.com/jackc/pgx/v5" +) + +// LockHash serializes CAS publication and collection until the transaction ends. +// A collision in the signed first eight digest bytes only adds serialization. +func LockHash(ctx context.Context, tx pgx.Tx, hash []byte) error { + if len(hash) != sha256.Size { + return errors.New("attachment: lock requires a SHA256 hash") + } + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(hash[:8]))) + return err +} + +func BlobKey(hash []byte) string { return "blobs/" + hex.EncodeToString(hash) } diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 00000000..729e42f6 --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,479 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + reconcilePageSize = 500 + reconcileOperationTimeout = 30 * time.Second +) + +type Finding struct { + Kind string `json:"kind"` + Key string `json:"key"` + Action string `json:"action"` + Applied bool `json:"applied"` +} + +// SQLQueries counts SQL statements, including transaction control and locks. +// S3Calls counts calls to the Blobs port, including each list page and deep GET. +type Stats struct { + ElapsedMS int64 `json:"elapsed_ms"` + SQLQueries int64 `json:"sql_queries"` + S3Calls int64 `json:"s3_calls"` + ScannedFiles int64 `json:"scanned_files"` + ScannedObjects int64 `json:"scanned_objects"` + Findings int64 `json:"findings"` + Repaired int64 `json:"repaired"` +} + +type Reconciler struct { + db *pgxpool.Pool + blobs Blobs + grace time.Duration +} + +func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) { + if db == nil || blobs == nil { + return nil, errors.New("reconcile: database and blobs are required") + } + if grace == 0 { + grace = time.Hour + } + if grace < 2*time.Minute { + return nil, errors.New("reconcile: grace must be at least two minutes") + } + return &Reconciler{db: db, blobs: blobs, grace: grace}, nil +} + +// Run streams findings and keeps at most one page of metadata in memory. +// Dry runs only read storage. Each apply candidate is checked again under the +// publisher's transaction lock; no scan snapshot authorizes a later mutation. +func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) { + started := time.Now() + defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }() + run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)} + for _, prefix := range []string{"staging/", "blobs/"} { + if err = run.objects(prefix); err != nil { + return stats, err + } + } + err = run.files() + return stats, err +} + +type reconcileRun struct { + r *Reconciler + ctx context.Context + apply, deep bool + emit func(Finding) + stats *Stats + cutoff time.Time +} + +func (s *reconcileRun) note(f *Finding) { + if f == nil { + return + } + s.stats.Findings++ + if f.Applied { + s.stats.Repaired++ + } + if s.emit != nil { + s.emit(*f) + } +} + +func (s *reconcileRun) old(modified time.Time) bool { + // Unknown timestamps must never authorize collection. + return !modified.IsZero() && modified.Before(s.cutoff) +} + +func (s *reconcileRun) objects(prefix string) error { + after := "" + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + page, err := s.r.blobs.List(opCtx, prefix, after, reconcilePageSize) + cancel() + if err != nil { + return fmt.Errorf("reconcile list %s: %w", prefix, err) + } + if len(page) == 0 { + return nil + } + var known map[string]bool + if prefix == "blobs/" { + known, err = s.knownFiles(page) + if err != nil { + return err + } + } + for _, obj := range page { + if obj.Key <= after { + return errors.New("reconcile: object page did not advance") + } + after = obj.Key + s.stats.ScannedObjects++ + if !s.old(obj.Modified) { + continue + } + var finding *Finding + if prefix == "staging/" { + finding, err = s.staging(obj) + } else { + finding, err = s.orphan(obj, known[obj.Key]) + } + if err != nil { + return fmt.Errorf("reconcile %s: %w", obj.Key, err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) stat(key string) (Object, error) { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + return s.r.blobs.Stat(opCtx, key) +} + +func (s *reconcileRun) delete(key string) error { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + err := s.r.blobs.Delete(opCtx, key) + if errors.Is(err, ErrObjectMissing) { + return nil + } + return err +} + +func (s *reconcileRun) staging(obj Object) (*Finding, error) { + f := &Finding{Kind: "stale_staging", Key: obj.Key, Action: "delete_object"} + if !s.apply { + return f, nil + } + current, err := s.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err = s.delete(obj.Key); err != nil { + return nil, err + } + f.Applied = true + return f, nil +} + +func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { + hashes := make([][]byte, 0, len(page)) + for _, obj := range page { + if !s.old(obj.Modified) { + continue + } + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err == nil && len(hash) == sha256.Size && BlobKey(hash) == obj.Key { + hashes = append(hashes, hash) + } + } + known := make(map[string]bool, len(hashes)) + if len(hashes) == 0 { + return known, nil + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var hash []byte + if err := rows.Scan(&hash); err != nil { + return nil, err + } + known[BlobKey(hash)] = true + } + return known, rows.Err() +} + +func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) { + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key { + return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil + } + if known { + return nil, nil + } + f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"} + if !s.apply { + return f, nil + } + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var exists bool + s.stats.SQLQueries++ + if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil { + return nil, err + } + if exists { + return nil, nil + } + current, err := op.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err = op.delete(obj.Key); err != nil { + return nil, err + } + f.Applied = true + return f, nil + }) +} + +// locked also covers hashes without a files row: a publisher and an orphan +// collector therefore cannot pass each other's existence check concurrently. +func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Finding, error)) (*Finding, error) { + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + s.stats.SQLQueries++ + tx, err := s.r.db.Begin(opCtx) + if err != nil { + return nil, err + } + committed := false + defer func() { + if !committed { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s.stats.SQLQueries++ + _ = tx.Rollback(cleanup) + } + }() + // Pool defaults cover ordinary requests; object I/O can outlive those + // idle limits. The operation deadline bounds the entire locked candidate. + s.stats.SQLQueries++ + if _, err = tx.Exec(opCtx, `SET LOCAL idle_in_transaction_session_timeout='35s'`); err != nil { + return nil, err + } + s.stats.SQLQueries++ + if err = LockHash(opCtx, tx, hash); err != nil { + return nil, err + } + f, err := fn(opCtx, tx) + if err != nil { + return nil, err + } + s.stats.SQLQueries++ + err = tx.Commit(opCtx) + committed = true // pgx closes the transaction even when COMMIT fails. + if err != nil { + return nil, err + } + return f, nil +} + +type reconcileFile struct { + hash []byte + size int64 + status string + created time.Time + referenced bool +} + +func (s *reconcileRun) files() error { + after := []byte{} + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, + EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) + FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after) + if err != nil { + return err + } + page := make([]reconcileFile, 0, reconcilePageSize) + for rows.Next() { + var f reconcileFile + if err = rows.Scan(&f.hash, &f.size, &f.status, &f.created, &f.referenced); err != nil { + rows.Close() + return err + } + page = append(page, f) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + // Close the page's SQL rows before acquiring a transaction connection. + for _, f := range page { + after = f.hash + s.stats.ScannedFiles++ + if !s.candidate(f) { + continue + } + var finding *Finding + if s.apply { + finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var current reconcileFile + s.stats.SQLQueries++ + err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, + EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) + FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash). + Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.candidate(current) { + return nil, nil + } + return op.inspect(current, tx) + }) + } else { + finding, err = s.inspect(f, nil) + } + if err != nil { + return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) candidate(f reconcileFile) bool { + if !s.old(f.created) { + return false + } + if !f.referenced { + return f.status == "pending" || f.status == "ready" + } + return f.status == "ready" || f.status == "failed" +} + +func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { + key := BlobKey(f.hash) + if !f.referenced { + // A publication that rolled back can leave fresh bytes behind an old + // pending row. Both the metadata and the current object must be old. + obj, err := s.stat(key) + if err != nil && !errors.Is(err, ErrObjectMissing) { + return nil, err + } + if err == nil && !s.old(obj.Modified) { + return nil, nil + } + finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"} + if !s.apply { + return finding, nil + } + // S3 must succeed first. A failed SQL commit then leaves a missing, + // unreferenced row that the next run can safely finish collecting. + if err := s.delete(key); err != nil { + return nil, err + } + s.stats.SQLQueries++ + if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + } + obj, err := s.stat(key) + if errors.Is(err, ErrObjectMissing) { + if f.status == "failed" { + return nil, nil + } + finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"} + if s.apply { + s.stats.SQLQueries++ + if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil { + return nil, err + } + finding.Applied = true + } + return finding, nil + } + if err != nil { + return nil, err + } + if obj.Size != f.size { + return &Finding{Kind: "size_mismatch", Key: key, Action: "report"}, nil + } + if s.deep { + match, err := s.verify(f) + if err != nil { + return nil, err + } + if !match { + return &Finding{Kind: "hash_mismatch", Key: key, Action: "report"}, nil + } + } + // A failed file stays failed even when the key reappears. Recovery requires + // an explicit publisher; a shallow HEAD cannot prove content integrity. + if f.status == "failed" { + return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil + } + return nil, nil +} + +func (s *reconcileRun) verify(f reconcileFile) (bool, error) { + if f.size < 0 || f.size == math.MaxInt64 { + return false, errors.New("reconcile: invalid file size") + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + body, err := s.r.blobs.Open(opCtx, BlobKey(f.hash)) + if err != nil { + return false, err + } + hash := sha256.New() + n, readErr := io.Copy(hash, io.LimitReader(body, f.size+1)) + closeErr := body.Close() + if readErr != nil { + return false, readErr + } + if closeErr != nil { + return false, closeErr + } + return n == f.size && bytes.Equal(hash.Sum(nil), f.hash), nil +} diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go new file mode 100644 index 00000000..39fd6dcd --- /dev/null +++ b/internal/attachment/reconcile_test.go @@ -0,0 +1,708 @@ +package attachment_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func TestReconcileBlobKey(t *testing.T) { + hash, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + if got := attachment.BlobKey(hash); got != "blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Fatalf("blob key: %q", got) + } +} + +// Only object storage is doubled: SQL, transactions, FKs and advisory locks are real. +type reconcileBlobs struct { + mu sync.Mutex + objects map[string]attachment.Object + bodies map[string][]byte + statErr, deleteErr, openErr error + listed func(string) + stating func(string) + opening func(string) io.ReadCloser + writes, readBytes int + limits []int +} + +func newReconcileBlobs() *reconcileBlobs { + return &reconcileBlobs{objects: make(map[string]attachment.Object), bodies: make(map[string][]byte)} +} +func (b *reconcileBlobs) add(key string, body []byte, modified time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + b.objects[key] = attachment.Object{Key: key, Size: int64(len(body)), Modified: modified} + b.bodies[key] = bytes.Clone(body) +} +func (b *reconcileBlobs) has(key string) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.objects[key] + return ok +} +func (b *reconcileBlobs) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error { + body, err := io.ReadAll(io.LimitReader(r, size+1)) + if err != nil { + return err + } + if int64(len(body)) != size { + return io.ErrUnexpectedEOF + } + b.add(key, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Copy(_ context.Context, source, dest string) error { + b.mu.Lock() + body, ok := b.bodies[source] + b.mu.Unlock() + if !ok { + return attachment.ErrObjectMissing + } + b.add(dest, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Open(_ context.Context, key string) (io.ReadCloser, error) { + if b.openErr != nil { + return nil, b.openErr + } + if b.opening != nil { + return b.opening(key), nil + } + b.mu.Lock() + defer b.mu.Unlock() + body, ok := b.bodies[key] + if !ok { + return nil, attachment.ErrObjectMissing + } + return io.NopCloser(bytes.NewReader(body)), nil +} +func (b *reconcileBlobs) Stat(_ context.Context, key string) (attachment.Object, error) { + if b.stating != nil { + b.stating(key) + } + b.mu.Lock() + defer b.mu.Unlock() + if b.statErr != nil { + return attachment.Object{}, b.statErr + } + obj, ok := b.objects[key] + if !ok { + return attachment.Object{}, fmt.Errorf("head: %w", attachment.ErrObjectMissing) + } + return obj, nil +} +func (b *reconcileBlobs) Delete(_ context.Context, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.deleteErr != nil { + return b.deleteErr + } + delete(b.objects, key) + delete(b.bodies, key) + b.writes++ + return nil +} +func (b *reconcileBlobs) List(_ context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + b.mu.Lock() + b.limits = append(b.limits, limit) + if limit < 1 || limit > 500 { + b.mu.Unlock() + return nil, errors.New("unbounded page") + } + var page []attachment.Object + for key, obj := range b.objects { + if strings.HasPrefix(key, prefix) && key > after { + page = append(page, obj) + } + } + sort.Slice(page, func(i, j int) bool { return page[i].Key < page[j].Key }) + if len(page) > limit { + page = page[:limit] + } + b.mu.Unlock() + if b.listed != nil { + b.listed(prefix) + } + return page, nil +} + +type reconcileFixture struct { + t *testing.T + db *pgxpool.Pool + b *reconcileBlobs + user, chat, message uuid.UUID +} + +func reconcileSetup(t *testing.T, db *testenv.DB) *reconcileFixture { + t.Helper() + db.Reset(t) + f := &reconcileFixture{t: t, db: db.Pool, b: newReconcileBlobs(), user: uuid.New(), chat: uuid.New(), message: uuid.New()} + f.exec(`INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,'reconcile','Reconcile','unused',now())`, f.user) + f.exec(`INSERT INTO chats(id,title,created_at) VALUES($1,'reconcile',now())`, f.chat) + f.exec(`INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, f.message, f.user) + return f +} +func (f *reconcileFixture) exec(sql string, args ...any) { + f.t.Helper() + if _, err := f.db.Exec(context.Background(), sql, args...); err != nil { + f.t.Fatal(err) + } +} +func (f *reconcileFixture) file(body, status string, age time.Duration, referenced, present bool) []byte { + f.t.Helper() + hash := sha256.Sum256([]byte(body)) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, time.Now().Add(-age)) + if referenced { + id := uuid.New() + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'image.png',$4,now())`, id, f.chat, f.user, hash[:]) + f.exec(`INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) + VALUES($1,$2,$3,(SELECT count(*) FROM message_attachments WHERE chat_id=$1 AND message_id=$2))`, f.chat, f.message, id) + } + if present { + f.b.add(attachment.BlobKey(hash[:]), []byte(body), time.Now().Add(-age)) + } + return hash[:] +} +func (f *reconcileFixture) status(hash []byte) string { + f.t.Helper() + var s string + err := f.db.QueryRow(context.Background(), `SELECT status FROM files WHERE hash=$1`, hash).Scan(&s) + if errors.Is(err, pgx.ErrNoRows) { + return "" + } + if err != nil { + f.t.Fatal(err) + } + return s +} +func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment.Finding, error) { + f.t.Helper() + r, err := attachment.NewReconciler(f.db, f.b, 0) + if err != nil { + f.t.Fatal(err) + } + var found []attachment.Finding + stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) }) + return stats, found, err +} + +func TestReconciler(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("constructor validates and defaults grace", func(t *testing.T) { + f := reconcileSetup(t, db) + for _, grace := range []time.Duration{-time.Second, time.Minute} { + if _, err := attachment.NewReconciler(f.db, f.b, grace); err == nil { + t.Fatalf("accepted grace %v", grace) + } + } + if _, err := attachment.NewReconciler(nil, f.b, 0); err == nil { + t.Fatal("accepted nil DB") + } + if _, err := attachment.NewReconciler(f.db, nil, 0); err == nil { + t.Fatal("accepted nil blobs") + } + if _, err := attachment.NewReconciler(f.db, f.b, 2*time.Minute); err != nil { + t.Fatal(err) + } + fresh := f.file("half hour", "ready", 30*time.Minute, false, true) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(fresh) != "ready" { + t.Fatalf("default grace: %+v %v", stats, err) + } + }) + t.Run("dry run then apply then repeat preserves fresh referenced and message links", func(t *testing.T) { + f := reconcileSetup(t, db) + old := 2 * time.Hour + pending := f.file("pending", "pending", old, false, true) + unused := f.file("unused", "ready", old, false, true) + used := f.file("used", "ready", old, true, true) + fresh := f.file("fresh", "pending", time.Minute, false, true) + missing := f.file("missing", "ready", old, true, false) + f.b.add("staging/old", []byte("old"), time.Now().Add(-old)) + f.b.add("staging/fresh", []byte("new"), time.Now()) + orphan := sha256.Sum256([]byte("orphan")) + orphanKey := attachment.BlobKey(orphan[:]) + f.b.add(orphanKey, []byte("orphan"), time.Now().Add(-old)) + stats, findings, err := f.run(false, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 5 || len(findings) != 5 { + t.Fatalf("dry: %+v %v %+v", stats, err, findings) + } + if f.b.writes != 0 || f.status(pending) != "pending" || f.status(missing) != "ready" { + t.Fatal("dry run mutated storage") + } + for _, finding := range findings { + if finding.Applied { + t.Fatal("dry finding marked applied") + } + } + stats, findings, err = f.run(true, false) + if err != nil || stats.Repaired != 5 { + t.Fatalf("apply: %+v %v %+v", stats, err, findings) + } + if f.status(pending) != "" || f.status(unused) != "" || f.status(missing) != "failed" { + t.Fatal("metadata repair incorrect") + } + if f.b.has(orphanKey) || f.b.has("staging/old") || !f.b.has("staging/fresh") || !f.b.has(attachment.BlobKey(used)) || !f.b.has(attachment.BlobKey(fresh)) { + t.Fatal("incorrect object collection") + } + var links int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM message_attachments`).Scan(&links); err != nil || links != 2 { + t.Fatalf("message links lost: %d %v", links, err) + } + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 { + t.Fatalf("repeat: %+v %v", stats, err) + } + }) + t.Run("outage is not missing and failed delete keeps row", func(t *testing.T) { + f := reconcileSetup(t, db) + used := f.file("used", "ready", 2*time.Hour, true, true) + outage := errors.New("injected S3 outage") + f.b.statErr = outage + stats, _, err := f.run(true, false) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(used) != "ready" { + t.Fatalf("outage: %+v %v", stats, err) + } + f.b.statErr = nil + unused := f.file("unused", "pending", 2*time.Hour, false, true) + f.b.deleteErr = outage + _, _, err = f.run(true, false) + if !errors.Is(err, outage) || f.status(unused) != "pending" { + t.Fatalf("delete failure: %v", err) + } + }) + t.Run("deep reports corruption and never revives failed without verification", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("evil"), time.Now().Add(-2*time.Hour)) + stats, found, err := f.run(true, true) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("deep: %+v %v %+v", stats, err, found) + } + f.exec(`UPDATE files SET status='failed' WHERE hash=$1`, hash) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("shallow revived failed file") + } + stats, _, err = f.run(true, true) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("corrupt file revived") + } + }) + t.Run("wrong size reports only and fresh missing stays ready", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("sized", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("longer body"), time.Now().Add(-2*time.Hour)) + fresh := f.file("new missing", "ready", time.Minute, true, false) + stats, found, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 1 || found[0].Kind != "size_mismatch" || f.status(fresh) != "ready" { + t.Fatalf("size/fresh: %+v %v %+v", stats, err, found) + } + }) + t.Run("orphan rechecks current object age", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("orphan")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("orphan"), time.Now().Add(-2*time.Hour)) + f.b.listed = func(prefix string) { + if prefix == "blobs/" { + f.b.add(key, []byte("republished"), time.Now()) + } + } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || !f.b.has(key) { + t.Fatalf("fresh replacement deleted: %+v %v", stats, err) + } + }) + t.Run("row rechecks references after taking lock", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("race refs", "ready", 2*time.Hour, false, true) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, true, false, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'race.png',$4,now())`, uuid.New(), f.chat, f.user, hash) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatal("new reference collected") + } + }) + t.Run("concurrent upload lock prevents orphan collection", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("publishing")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("publishing"), time.Now().Add(-2*time.Hour)) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash[:]); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, true, false, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + if _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,10,'image/png','ready')`, hash[:]); err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if !f.b.has(key) || f.status(hash[:]) != "ready" { + t.Fatal("published blob collected") + } + }) + t.Run("signed big endian lock key matches publisher", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := make([]byte, 32) + for i := 0; i < 8; i++ { + hash[i] = 255 + } + hash[7] = 254 + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + other, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer other.Rollback(ctx) + var acquired bool + if err = other.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(-2::bigint)`).Scan(&acquired); err != nil || acquired { + t.Fatalf("lock mismatch: %v %v", acquired, err) + } + }) + t.Run("keyset pages tolerate deletes and report costs", func(t *testing.T) { + f := reconcileSetup(t, db) + for i := 0; i < 1003; i++ { + f.b.add(fmt.Sprintf("staging/%04d", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','pending',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + stats, _, err := f.run(true, false) + if err != nil || stats.ScannedFiles != 1003 || stats.ScannedObjects != 1003 || stats.Repaired != 2006 { + t.Fatalf("pages: %+v %v", stats, err) + } + var n int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&n); err != nil || n != 0 { + t.Fatalf("skipped rows: %d %v", n, err) + } + if len(f.b.limits) < 4 || stats.SQLQueries == 0 || stats.S3Calls == 0 { + t.Fatalf("missing paging/costs: %+v", stats) + } + raw, err := json.Marshal(stats) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"elapsed_ms", "sql_queries", "s3_calls", "scanned_files", "scanned_objects", "findings", "repaired"} { + if !bytes.Contains(raw, []byte(`"`+key+`":`)) { + t.Fatalf("missing JSON key %s: %s", key, raw) + } + } + }) + t.Run("old blob existence checks are batched by page", func(t *testing.T) { + f := reconcileSetup(t, db) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','ready',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),$1,$2,'batch.png',hash,now() FROM files`, f.chat, f.user) + for i := 1; i <= 1003; i++ { + f.b.add(fmt.Sprintf("blobs/%064x", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + stats, _, err := f.run(false, false) + if err != nil || stats.SQLQueries > 8 || stats.SQLQueries == 0 || stats.ScannedFiles != 1003 || stats.Findings != 0 { + t.Fatalf("per-object SQL lookup: %+v %v", stats, err) + } + }) +} + +func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) { + t.Helper() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + for { + var waiting bool + if err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted)`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + return + } + select { + case <-deadline.C: + t.Fatal("reconciler did not wait on publisher advisory lock") + case <-tick.C: + } + } +} + +type reconcileEndless struct { + read int64 + closed bool +} + +func (r *reconcileEndless) Read(p []byte) (int, error) { + for i := range p { + p[i] = 'x' + } + r.read += int64(len(p)) + return len(p), nil +} +func (r *reconcileEndless) Close() error { r.closed = true; return nil } + +func TestReconcilerFailureBoundaries(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("slow object call survives pool idle transaction timeout", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("slow object", "pending", 2*time.Hour, false, true) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "100ms" + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + f.db = pool + f.b.stating = func(string) { time.Sleep(250 * time.Millisecond) } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("slow S3 transaction rolled back: %+v %v", stats, err) + } + }) + t.Run("fresh object with old unused metadata is preserved", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("republished", "pending", 2*time.Hour, false, true) + f.b.add(attachment.BlobKey(hash), []byte("republished"), time.Now()) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "pending" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("fresh blob lost: %+v %v", stats, err) + } + }) + t.Run("deep read stops at size plus one and closes body", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + body := &reconcileEndless{} + f.b.opening = func(string) io.ReadCloser { return body } + stats, found, err := f.run(true, true) + if err != nil || body.read != 5 || !body.closed || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" { + t.Fatalf("unbounded deep read: read=%d closed=%v %+v %v", body.read, body.closed, stats, err) + } + }) + t.Run("deep GET outage preserves referenced state", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + outage := errors.New("injected GET outage") + f.b.openErr = outage + stats, _, err := f.run(true, true) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("GET outage: %+v %v", stats, err) + } + }) + t.Run("valid failed object remains stable even deep", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "failed", 2*time.Hour, true, true) + for _, deep := range []bool{false, true} { + stats, found, err := f.run(true, deep) + if err != nil || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "failed_present" || f.status(hash) != "failed" { + t.Fatalf("failed state unstable: %+v %v %+v", stats, err, found) + } + } + }) + t.Run("dry run works with a read only PostgreSQL session", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("missing", "ready", 2*time.Hour, true, false) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + readOnly, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer readOnly.Close() + r, err := attachment.NewReconciler(readOnly, f.b, 0) + if err != nil { + t.Fatal(err) + } + stats, err := r.Run(ctx, false, true, nil) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("read only dry run: %+v %v", stats, err) + } + }) + t.Run("failed SQL commit after blob deletion is retryable", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("unused", "pending", 2*time.Hour, false, true) + f.exec(`CREATE FUNCTION reconcile_reject_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected commit failure'; END $$`) + f.exec(`CREATE CONSTRAINT TRIGGER reconcile_reject_delete AFTER DELETE ON files DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reconcile_reject_delete()`) + t.Cleanup(func() { + f.exec(`DROP TRIGGER IF EXISTS reconcile_reject_delete ON files`) + f.exec(`DROP FUNCTION IF EXISTS reconcile_reject_delete()`) + }) + stats, _, err := f.run(true, false) + if err == nil || stats.Repaired != 0 || f.status(hash) != "pending" || f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("failed commit: %+v %v", stats, err) + } + f.exec(`DROP TRIGGER reconcile_reject_delete ON files`) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("retry: %+v %v", stats, err) + } + }) +} + +func TestReconcilerS3(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, false) + missing := f.file("missing", "ready", 2*time.Hour, true, false) + key := attachment.BlobKey(hash) + if err = s3.Store.Put(ctx, key, strings.NewReader("evil"), 4, "image/png"); err != nil { + t.Fatal(err) + } + if err = s3.Store.Put(ctx, "staging/fresh", strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, s3.Store, 0) + if err != nil { + t.Fatal(err) + } + // Exercise the command with file-based configuration and the real listing + // identity. Its default must remain read-only, including missing metadata. + secretDir := t.TempDir() + cliEnv := os.Environ() + for name, value := range map[string]string{ + "DATABASE_URL_FILE": db.DSN, + "S3_ACCESS_KEY_ID_FILE": s3.AccessKey, + "S3_SECRET_ACCESS_KEY_FILE": s3.SecretKey, + } { + path := filepath.Join(secretDir, name) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + cliEnv = append(cliEnv, name+"="+path) + } + cliEnv = append(cliEnv, "S3_ENDPOINT="+s3.Endpoint, "S3_REGION="+s3.Region, "S3_BUCKET="+s3.Bucket) + cmd := exec.CommandContext(ctx, "go", "run", "../../cmd/reconcile", "--deep") + cmd.Env = cliEnv + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("CLI dry run: %v: %s", err, stderr.String()) + } + decoder := json.NewDecoder(&stdout) + var cliStats attachment.Stats + for i := 0; i < 2; i++ { + var finding attachment.Finding + if err := decoder.Decode(&finding); err != nil || finding.Applied || finding.Kind == "" || finding.Key == "" || finding.Action == "" { + t.Fatalf("CLI finding: %+v %v", finding, err) + } + } + if err := decoder.Decode(&cliStats); err != nil || cliStats.Findings != 2 || cliStats.Repaired != 0 || f.status(missing) != "ready" { + t.Fatalf("CLI mutated by default: %+v %v", cliStats, err) + } + var found []attachment.Finding + stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) }) + if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" { + t.Fatalf("S3 dry: %+v %v %+v", stats, err, found) + } + stats, err = r.Run(ctx, true, true, nil) + if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" { + t.Fatalf("S3 apply: %+v %v", stats, err) + } + if _, err = s3.Store.Stat(ctx, key); err != nil { + t.Fatal("referenced corrupt object deleted", err) + } + if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil { + t.Fatal("fresh staging deleted", err) + } + stats, err = r.Run(ctx, true, true, nil) + if err != nil || stats.Repaired != 0 { + t.Fatalf("S3 repeat: %+v %v", stats, err) + } +} + +func TestReconcileLockRejectsMalformedHash(t *testing.T) { + if err := attachment.LockHash(context.Background(), nil, []byte("bad")); err == nil { + t.Fatal("malformed hash accepted") + } +} diff --git a/internal/attachment/schema_test.go b/internal/attachment/schema_test.go new file mode 100644 index 00000000..c5db7ee6 --- /dev/null +++ b/internal/attachment/schema_test.go @@ -0,0 +1,27 @@ +package attachment_test + +import ( + "context" + "messenger/internal/testenv" + "testing" +) + +func TestFilesSchemaRejectsMalformedHashes(t *testing.T) { + db, err := testenv.StartPostgres(context.Background()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err != nil { + t.Fatalf("valid content-addressed file: %v", err) + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ('bad',12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted a malformed SHA256") + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted duplicate content hash") + } +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 00000000..ee574176 --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,172 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "errors" + "io" + "log/slog" + "net/http" + "time" + + "github.com/google/uuid" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +type Service struct { + repo Repository + blobs Blobs + maxBytes, maxPixels int64 + decode chan struct{} +} + +func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service { + if maxBytes == 0 { + maxBytes = MaxUploadBytes + } + if maxPixels == 0 { + maxPixels = MaxImagePixels + } + return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)} +} + +func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) { + if err := ValidateFilename(filename); err != nil { + return Attachment{}, err + } + if mime != "image/png" && mime != "image/jpeg" { + return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted") + } + ok, err := s.repo.IsMember(ctx, chatID, actor) + if err != nil { + return Attachment{}, err + } + if !ok { + return Attachment{}, apperr.NotFound("chat not found") + } + if size > s.maxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if size <= 0 { + return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required") + } + if s.blobs == nil { + return Attachment{}, apperr.Unavailable("object storage is unavailable") + } + id := ident.New() + stage := "staging/" + id.String() + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := s.blobs.Delete(cleanup, stage); err != nil { + slog.Warn("staging cleanup deferred", "error", err) + } + }() + hash := sha256.New() + counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)} + err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime) + if err != nil { + _, tooLarge := errors.AsType[*http.MaxBytesError](err) + if counter.n > s.maxBytes || tooLarge { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + return Attachment{}, apperr.Invalid("incomplete upload") + } + return Attachment{}, unavailable(err) + } + var extra [1]byte + n, readErr := counter.Read(extra[:]) + if counter.n > s.maxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) { + return Attachment{}, apperr.Invalid("body length does not match Content-Length") + } + // Full decoding is bounded by dimensions and one decoder per replica. Body + // transfer itself stays streaming and does not occupy the decoder slot. + select { + case s.decode <- struct{}{}: + case <-ctx.Done(): + return Attachment{}, unavailable(ctx.Err()) + } + err = s.validate(ctx, stage, mime) + <-s.decode + if err != nil { + return Attachment{}, err + } + a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)} + if err := s.repo.Publish(ctx, a, stage); err != nil { + return Attachment{}, err + } + return a, nil +} + +func (s *Service) validate(ctx context.Context, stage, mime string) error { + r, err := s.blobs.Open(ctx, stage) + if err != nil { + return unavailable(err) + } + defer r.Close() + tracked := &storageReader{reader: r} + err = ValidateImage(tracked, mime, s.maxPixels) + if tracked.err != nil { + return unavailable(tracked.err) + } + if ctx.Err() != nil { + return unavailable(ctx.Err()) + } + return err +} + +// Image sniffing can replace a reader failure with ErrFormat. Preserve the +// storage failure independently, while ordinary EOF still means malformed bytes. +type storageReader struct { + reader io.Reader + err error +} + +func (r *storageReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil && !errors.Is(err, io.EOF) { + r.err = err + } + return n, err +} + +func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) { + a, err := s.repo.GetForMember(ctx, id, actor) + if err != nil { + return Attachment{}, err + } + if a.Status != "ready" { + return Attachment{}, apperr.Conflict("attachment is not ready") + } + return a, nil +} + +func (s *Service) Content(ctx context.Context, actor, id uuid.UUID) (Attachment, io.ReadCloser, error) { + a, err := s.Get(ctx, actor, id) + if err != nil { + return Attachment{}, nil, err + } + if s.blobs == nil { + return Attachment{}, nil, apperr.Unavailable("object storage is unavailable") + } + r, err := s.blobs.Open(ctx, BlobKey(a.FileHash)) + if err != nil { + return Attachment{}, nil, unavailable(err) + } + return a, r, nil +} + +type countReader struct { + r io.Reader + n int64 +} + +func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e } +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) +} diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go new file mode 100644 index 00000000..c207a956 --- /dev/null +++ b/internal/attachment/service_test.go @@ -0,0 +1,270 @@ +package attachment_test + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "messenger/internal/attachment" + "messenger/internal/attachment/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" +) + +type testBlobs struct { + mu sync.Mutex + data map[string][]byte + copies int + failCopy bool +} + +type blockingPublication struct { + *testBlobs + entered, resume chan struct{} +} + +func (b *blockingPublication) Copy(ctx context.Context, from, to string) error { + close(b.entered) + select { + case <-b.resume: + case <-ctx.Done(): + return ctx.Err() + } + return b.testBlobs.Copy(ctx, from, to) +} +func (b *blockingPublication) Stat(ctx context.Context, key string) (attachment.Object, error) { + o, e := b.testBlobs.Stat(ctx, key) + o.Modified = time.Now().Add(-2 * time.Hour) + return o, e +} + +func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + actor, room := ident.New(), ident.New() + if _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'race_user','User','hash',now())`, actor); err != nil { + t.Fatal(err) + } + if _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'race',now())`, room); err != nil { + t.Fatal(err) + } + if _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, room, actor); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(data) + if _, err = db.Pool.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png','pending',now()-interval '2 hours')`, hash[:], len(data)); err != nil { + t.Fatal(err) + } + b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})} + s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0) + uploaded := make(chan error, 1) + go func() { + _, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data)) + uploaded <- e + }() + select { + case <-b.entered: + case <-ctx.Done(): + t.Fatal("publisher never reached copy") + } + r, err := attachment.NewReconciler(db.Pool, b, time.Hour) + if err != nil { + t.Fatal(err) + } + collected := make(chan error, 1) + go func() { + stats, e := r.Run(ctx, true, false, nil) + if e == nil && stats.Repaired != 0 { + e = errors.New("collector removed a newly referenced file") + } + collected <- e + }() + // Observe a real waiting advisory lock, rather than guessing which goroutine won. + deadline := time.Now().Add(3 * time.Second) + waiting := false + for time.Now().Before(deadline) { + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted`).Scan(&n); err != nil { + t.Fatal(err) + } + if n > 0 { + waiting = true + break + } + time.Sleep(5 * time.Millisecond) + } + close(b.resume) + if !waiting { + t.Fatal("collector did not serialize with the publisher") + } + if err := <-uploaded; err != nil { + t.Fatal(err) + } + if err := <-collected; err != nil { + t.Fatal(err) + } + if _, err := b.Stat(ctx, attachment.BlobKey(hash[:])); err != nil { + t.Fatal("collector deleted published data", err) + } +} + +func (b *testBlobs) Put(_ context.Context, k string, r io.Reader, n int64, m string) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + b.mu.Lock() + defer b.mu.Unlock() + b.data[k] = data + return nil +} +func (b *testBlobs) Open(_ context.Context, k string) (io.ReadCloser, error) { + b.mu.Lock() + defer b.mu.Unlock() + data, ok := b.data[k] + if !ok { + return nil, attachment.ErrObjectMissing + } + return io.NopCloser(bytes.NewReader(data)), nil +} +func (b *testBlobs) Stat(_ context.Context, k string) (attachment.Object, error) { + b.mu.Lock() + defer b.mu.Unlock() + data, ok := b.data[k] + if !ok { + return attachment.Object{}, attachment.ErrObjectMissing + } + return attachment.Object{Key: k, Size: int64(len(data)), Modified: time.Now()}, nil +} +func (b *testBlobs) Copy(_ context.Context, from, to string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.failCopy { + return errors.New("copy outage") + } + b.copies++ + b.data[to] = bytes.Clone(b.data[from]) + return nil +} +func (b *testBlobs) Delete(_ context.Context, k string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.data, k) + return nil +} +func (b *testBlobs) List(context.Context, string, string, int) ([]attachment.Object, error) { + return nil, nil +} + +func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { + ctx := context.Background() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + actor, c1, c2 := ident.New(), ident.New(), ident.New() + _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'alice','Alice','hash',now());`, actor) + if err != nil { + t.Fatal(err) + } + for _, c := range []uuid.UUID{c1, c2} { + _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'room',now())`, c) + if err != nil { + t.Fatal(err) + } + _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, c, actor) + if err != nil { + t.Fatal(err) + } + } + b := &testBlobs{data: map[string][]byte{}} + s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels) + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + var results [10]attachment.Attachment + var wg sync.WaitGroup + for i := range results { + wg.Go(func() { + c := c1 + if i%2 == 1 { + c = c2 + } + var e error + results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data)) + if e != nil { + t.Error(e) + } + }) + } + wg.Wait() + var files, atts int + err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments)`).Scan(&files, &atts) + if err != nil { + t.Fatal(err) + } + if files != 1 || atts != 10 || b.copies != 1 || len(b.data) != 1 { + t.Fatalf("files=%d attachments=%d copies=%d objects=%d", files, atts, b.copies, len(b.data)) + } + if results[0].ID == results[1].ID { + t.Fatal("deduplication merged chat ACL identity") + } + if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("outsider got %v", err) + } + // A removed blob is republished from the new upload, preserving old references. + b.mu.Lock() + clear(b.data) + b.mu.Unlock() + if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil { + t.Fatal(err) + } + if b.copies != 2 { + t.Fatalf("missing ready blob was not restored: copies=%d", b.copies) + } + b.failCopy = true + other := bytes.Clone(data) + other = append(other, 0) + if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("copy outage: %v", err) + } + if len(b.data) != 1 { + t.Fatalf("failed publication left staging: %d objects", len(b.data)) + } + b.failCopy = false + // S3 publication succeeds, then PostgreSQL refuses the logical attachment. + // Compensation may remove staging but must leave shared CAS data intact. + _, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$; + CREATE TRIGGER reject_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_attachment()`) + if err != nil { + t.Fatal(err) + } + if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil { + t.Fatal("publication survived rejected metadata") + } + if len(b.data) != 2 { + t.Fatalf("expected original and recoverable orphan, got %d objects", len(b.data)) + } + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&files); err != nil || files != 1 { + t.Fatalf("failed DB insert committed file metadata: %d %v", files, err) + } +} diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go new file mode 100644 index 00000000..607932a9 --- /dev/null +++ b/internal/attachment/store/postgres.go @@ -0,0 +1,79 @@ +package store + +import ( + "context" + "errors" + "fmt" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" +) + +type Attachments struct { + pool *pgxpool.Pool + blobs attachment.Blobs +} + +func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments { + return &Attachments{pool, blobs} +} + +func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) { + var ok bool + err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok) + return ok, postgres.Classify(err) +} + +// Publication and garbage collection use the same hash lock. The transaction +// spans only publication, never client transfer or image decoding. +func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil { + return err + } + if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil { + return err + } + var status string + err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + copyNeeded := status != "ready" + if !copyNeeded { + obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash)) + if errors.Is(e, attachment.ErrObjectMissing) { + copyNeeded = true + } else if e != nil { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e) + } else if obj.Size != a.SizeBytes { + copyNeeded = true + } + } + if copyNeeded { + if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) + } + } + _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType) + if err != nil { + return fmt.Errorf("publish file: %w", err) + } + _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) + return err + })) +} + +func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) { + var a attachment.Attachment + err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at + FROM attachments a JOIN files f ON f.hash=a.file_hash + WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return attachment.Attachment{}, apperr.NotFound("attachment not found") + } + return a, postgres.Classify(err) +} diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go new file mode 100644 index 00000000..a810545f --- /dev/null +++ b/internal/attachment/validation.go @@ -0,0 +1,51 @@ +package attachment + +import ( + "bytes" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "strings" + "unicode" + "unicode/utf8" + + "messenger/internal/platform/apperr" +) + +const MaxUploadBytes int64 = 10 * 1024 * 1024 +const MaxImagePixels int64 = 40_000_000 + +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return apperr.Invalid("filename must contain 1-255 characters") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return apperr.Invalid("filename contains a forbidden character") + } + } + return nil +} + +// The small header is replayed into the decoder; pixels are allocated only +// after checking dimensions. The caller serializes decoding per replica. +func ValidateImage(r io.Reader, mime string, maxPixels int64) error { + limited := io.LimitReader(r, MaxUploadBytes+1) + var header bytes.Buffer + cfg, format, err := image.DecodeConfig(io.TeeReader(limited, &header)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + if (format != "png" && format != "jpeg") || "image/"+format != mime { + return apperr.UnsupportedMedia("image does not match Content-Type") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) { + return apperr.PayloadTooLarge("image exceeds pixel limit") + } + _, _, err = image.Decode(io.MultiReader(bytes.NewReader(header.Bytes()), limited)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + return nil +} diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go new file mode 100644 index 00000000..7ef3c509 --- /dev/null +++ b/internal/attachment/validation_test.go @@ -0,0 +1,93 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "messenger/internal/platform/apperr" + "strings" + "testing" +) + +func TestValidateFilename(t *testing.T) { + for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} { + if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%q: %v", name, err) + } + } + if err := ValidateFilename("фото 👋.png"); err != nil { + t.Fatal(err) + } +} + +type failingRead struct{ prefix []byte } + +func (r *failingRead) Read(p []byte) (int, error) { + if len(r.prefix) > 0 { + n := copy(p, r.prefix) + r.prefix = r.prefix[n:] + return n, nil + } + return 0, errors.New("injected storage read outage") +} + +type openedBlob struct { + Blobs + reader io.Reader +} + +func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(b.reader), nil +} + +func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) { + var pngData bytes.Buffer + if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil { + t.Fatal(err) + } + for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} { + s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0) + err := s.validate(context.Background(), "staging/test", "image/png") + if apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err) + } + } +} + +func TestValidateImageRejectsCorruptionAndLimitsPixelsBeforeDecode(t *testing.T) { + im := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + im.Set(1, 1, color.NRGBA{R: 200, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, im); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, mime string + data []byte + pixels int64 + kind apperr.Kind + }{ + {"valid", "image/png", b.Bytes(), 6, apperr.KindInternal}, + {"spoof", "image/jpeg", b.Bytes(), 6, apperr.KindUnsupportedMedia}, + {"truncated", "image/png", b.Bytes()[:len(b.Bytes())-20], 6, apperr.KindUnsupportedMedia}, + {"pixels", "image/png", b.Bytes(), 5, apperr.KindPayloadTooLarge}, + {"garbage", "image/png", []byte("not a picture"), 6, apperr.KindUnsupportedMedia}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateImage(bytes.NewReader(tc.data), tc.mime, tc.pixels) + if tc.name == "valid" { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil || apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want kind %v", err, tc.kind) + } + }) + } +} diff --git a/internal/chat/attachment_handler_test.go b/internal/chat/attachment_handler_test.go new file mode 100644 index 00000000..764432f5 --- /dev/null +++ b/internal/chat/attachment_handler_test.go @@ -0,0 +1,72 @@ +package chat_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/httpx" +) + +func messageRouter(f fixture, actor uuid.UUID) *gin.Engine { + r := gin.New() + r.Use(func(c *gin.Context) { httpx.SetUserID(c, actor) }) + chat.NewHandler(f.svc).Register(r) + return r +} + +func sendJSON(r http.Handler, chatID uuid.UUID, key, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/chats/"+chatID.String()+"/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", key) + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + return rec +} + +func TestMessageAttachmentJSONEmptyArrayAndOmitted(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for i, body := range []string{`{"text":"hello","attachment_ids":[]}`, `{"text":"hello"}`} { + rec := sendJSON(r, c, fmt.Sprint(i), body) + if rec.Code != http.StatusCreated { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &fields); err != nil { + t.Fatal(err) + } + if string(fields["attachment_ids"]) != "[]" { + t.Fatalf("attachment_ids = %s, want []", fields["attachment_ids"]) + } + } +} + +func TestMessageAttachmentJSONRejectsInvalidArrays(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + id := uuid.NewString() + for _, raw := range []string{ + "null", `"not an array"`, "{}", "1", "[null]", "[1]", `["bad"]`, + `["00000000-0000-0000-0000-000000000000"]`, + fmt.Sprintf("[%q]", strings.ReplaceAll(id, "-", "")), + fmt.Sprintf("[%q,%q]", id, strings.ToUpper(id)), + "[" + strings.TrimSuffix(strings.Repeat(fmt.Sprintf("%q,", id), 11), ",") + "]", + } { + t.Run(raw, func(t *testing.T) { + rec := sendJSON(r, c, "invalid", `{"text":"hello","attachment_ids":`+raw+"}") + if rec.Code != http.StatusBadRequest { + t.Fatalf("send: %d %s", rec.Code, rec.Body) + } + }) + } +} diff --git a/internal/chat/attachment_http_test.go b/internal/chat/attachment_http_test.go new file mode 100644 index 00000000..45510918 --- /dev/null +++ b/internal/chat/attachment_http_test.go @@ -0,0 +1,125 @@ +package chat_test + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], status, id, chatID, owner) + if err != nil { + t.Fatal(err) + } + return id +} + +func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + a, b := f.attachment(t, c, f.bob, "ready"), f.attachment(t, c, f.alice, "ready") + r := messageRouter(f, f.alice) + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a) + first := sendJSON(r, c, "attachments", body) + if first.Code != http.StatusCreated { + t.Fatalf("attachment-only send: %d %s", first.Code, first.Body) + } + var message chat.MessageResponse + if err := json.Unmarshal(first.Body.Bytes(), &message); err != nil { + t.Fatal(err) + } + if message.Text != "" || !reflect.DeepEqual(message.AttachmentIDs, []string{b.String(), a.String()}) { + t.Fatalf("message = %+v", message) + } + if again := sendJSON(r, c, "attachments", body); again.Code != http.StatusCreated || again.Body.String() != first.Body.String() { + t.Fatalf("replay: %d %s, want %s", again.Code, again.Body, first.Body) + } + for _, changed := range []string{ + fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, a, b), + fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, b), + fmt.Sprintf(`{"text":"different","attachment_ids":[%q,%q]}`, b, a), + } { + if rec := sendJSON(r, c, "attachments", changed); rec.Code != http.StatusConflict { + t.Fatalf("changed payload: %d %s", rec.Code, rec.Body) + } + } + rec := httptest.NewRecorder() + r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/chats/"+c.String()+"/messages", nil)) + var page pagination.PageResponse[chat.MessageResponse] + if rec.Code != http.StatusOK { + t.Fatalf("history: %d %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], message) { + t.Fatalf("history = %+v, want %+v", page, message) + } + without := sendJSON(r, c, "empty", `{"text":"text-only"}`) + empty := sendJSON(r, c, "empty", `{"attachment_ids":[],"text":"text-only"}`) + if without.Code != http.StatusCreated || empty.Code != http.StatusCreated || without.Body.String() != empty.Body.String() { + t.Fatalf("omitted vs empty replay: %d %s; %d %s", without.Code, without.Body, empty.Code, empty.Body) + } +} + +func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + other := f.newChat(t, f.alice) + ready := f.attachment(t, c, f.bob, "ready") + foreign := f.attachment(t, other, f.alice, "ready") + failed := f.attachment(t, c, f.alice, "failed") + pending := f.attachment(t, c, f.alice, "pending") + for _, tc := range []struct { + name string + actor, attachment uuid.UUID + want int + }{ + {"unknown", f.alice, ident.New(), http.StatusNotFound}, + {"foreign", f.alice, foreign, http.StatusNotFound}, + {"failed", f.alice, failed, http.StatusConflict}, + {"pending", f.alice, pending, http.StatusConflict}, + {"outsider", f.eve, ready, http.StatusNotFound}, + {"other-owner", f.alice, ready, http.StatusCreated}, + } { + t.Run(tc.name, func(t *testing.T) { + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment) + rec := sendJSON(messageRouter(f, tc.actor), c, "same", body) + if rec.Code != tc.want { + t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want) + } + }) + } +} + +func TestMessageJSONRequiresTextString(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for _, body := range []string{"null", `{}`, `{"text":null}`, `{"text":1}`, `{"text":[]}`, `{"text":"","attachment_ids":[]}`} { + if rec := sendJSON(r, c, "text", body); rec.Code != http.StatusBadRequest { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + } + if rec := sendJSON(r, c, "text", `{"text":"valid"}`); rec.Code != http.StatusCreated { + t.Fatalf("valid payload after rejected requests: %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/chat/attachment_message_test.go b/internal/chat/attachment_message_test.go new file mode 100644 index 00000000..d14678f8 --- /dev/null +++ b/internal/chat/attachment_message_test.go @@ -0,0 +1,66 @@ +package chat_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestAttachmentMessageValidatesAndOwnsOrderedIDs(t *testing.T) { + a, b := ident.New(), ident.New() + ids := []uuid.UUID{b, a} + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), "", ids...) + if err != nil || m.Text != "" || !reflect.DeepEqual(m.AttachmentIDs, ids) { + t.Fatalf("attachment-only message = %+v, %v", m, err) + } + ids[0] = a + if m.AttachmentIDs[0] != b { + t.Fatal("message aliases caller's attachment slice") + } + for _, tc := range []struct { + text string + ids []uuid.UUID + }{ + {"ok", []uuid.UUID{a, a}}, + {"ok", []uuid.UUID{uuid.Nil}}, + {"ok", make([]uuid.UUID, 11)}, + {strings.Repeat("x", 4001), []uuid.UUID{a}}, + {"nul\x00", []uuid.UUID{a}}, + } { + _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), tc.text, tc.ids...) + if err == nil || apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("invalid message accepted: %v", err) + } + } + ten := make([]uuid.UUID, 10) + for i := range ten { + ten[i] = ident.New() + } + if _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " \n", ten...); err != nil { + t.Fatalf("ten attachments: %v", err) + } +} + +func TestSameContentIncludesOrderedAttachmentsAndTreatsNilAsEmpty(t *testing.T) { + a, b := ident.New(), ident.New() + m := chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{b, a}} + for _, tc := range []struct { + ids []uuid.UUID + equal bool + }{ + {[]uuid.UUID{b, a}, true}, {[]uuid.UUID{a, b}, false}, {[]uuid.UUID{a}, false}, {nil, false}, + } { + if got := m.SameContent(chat.Message{Text: "hello", AttachmentIDs: tc.ids}); got != tc.equal { + t.Fatalf("attachments %v: equality=%v, want %v", tc.ids, got, tc.equal) + } + } + if !(chat.Message{Text: "hello"}).SameContent(chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{}}) { + t.Fatal("nil attachments must equal empty attachments") + } +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go index c913393e..95dc9089 100644 --- a/internal/chat/handler.go +++ b/internal/chat/handler.go @@ -1,6 +1,8 @@ package chat import ( + "bytes" + "encoding/json" "net/http" "regexp" @@ -36,7 +38,8 @@ type createChatRequest struct { } type createMessageRequest struct { - Text string `json:"text"` + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` } type ChatResponse struct { @@ -47,11 +50,12 @@ type ChatResponse struct { } type MessageResponse struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` } func toChatResponse(c ChatWithMembers) ChatResponse { @@ -68,12 +72,17 @@ func toChatResponse(c ChatWithMembers) ChatResponse { } func toMessageResponse(m Message) MessageResponse { + ids := make([]string, 0, len(m.AttachmentIDs)) + for _, id := range m.AttachmentIDs { + ids = append(ids, id.String()) + } return MessageResponse{ - ID: m.ID.String(), - ChatID: m.ChatID.String(), - SenderID: m.SenderID.String(), - Text: m.Text, - CreatedAt: httpx.Timestamp(m.CreatedAt), + AttachmentIDs: ids, + ID: m.ID.String(), + ChatID: m.ChatID.String(), + SenderID: m.SenderID.String(), + Text: m.Text, + CreatedAt: httpx.Timestamp(m.CreatedAt), } } @@ -144,7 +153,16 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } - message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text}) + if req.Text == nil { + httpx.Error(c, apperr.Invalid("text must be a string")) + return + } + attachments, err := decodeAttachmentIDs(req.AttachmentIDs) + if err != nil { + httpx.Error(c, err) + return + } + message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: *req.Text, AttachmentIDs: attachments}) if err != nil { httpx.Error(c, err) return @@ -207,3 +225,29 @@ func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) { } return limit, &last, nil } + +// RawMessage distinguishes an omitted optional array from an explicit null. +func decodeAttachmentIDs(raw json.RawMessage) ([]uuid.UUID, error) { + if raw == nil { + return nil, nil + } + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, apperr.Invalid("attachment_ids must be an array") + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil { + return nil, apperr.Invalid("attachment_ids must be an array of UUIDs") + } + if len(values) > MaxAttachments { + return nil, apperr.Invalid("attachment_ids must have at most 10 entries") + } + ids := make([]uuid.UUID, 0, len(values)) + for _, value := range values { + id, err := ident.Parse(value) + if err != nil { + return nil, apperr.Invalid("attachment_ids must contain valid UUIDs") + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/internal/chat/message.go b/internal/chat/message.go index 06a758bc..cda9b940 100644 --- a/internal/chat/message.go +++ b/internal/chat/message.go @@ -2,6 +2,7 @@ package chat import ( "fmt" + "slices" "strings" "time" "unicode/utf8" @@ -12,19 +13,36 @@ import ( "messenger/internal/platform/ident" ) -const MaxTextCodePoints = 4000 +const ( + MaxTextCodePoints = 4000 + MaxAttachments = 10 +) type Message struct { - ID uuid.UUID - ChatID uuid.UUID - SenderID uuid.UUID - Text string - CreatedAt time.Time + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + AttachmentIDs []uuid.UUID } -// NewMessage keeps the text verbatim; only blank or oversized text is rejected. -func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { - if strings.TrimSpace(text) == "" { +// NewMessage keeps text and attachment order verbatim; it owns its attachment slice. +func NewMessage(id, chatID, senderID uuid.UUID, text string, attachmentIDs ...uuid.UUID) (Message, error) { + if len(attachmentIDs) > MaxAttachments { + return Message{}, apperr.Invalid("attachment_ids must have at most 10 entries") + } + seen := make(map[uuid.UUID]struct{}, len(attachmentIDs)) + for _, id := range attachmentIDs { + if id == uuid.Nil { + return Message{}, apperr.Invalid("attachment_ids must contain non-nil UUIDs") + } + if _, duplicate := seen[id]; duplicate { + return Message{}, apperr.Invalid("attachment_ids must be unique") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 { return Message{}, apperr.Invalid("text must not be blank") } if utf8.RuneCountInString(text) > MaxTextCodePoints { @@ -35,16 +53,17 @@ func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { return Message{}, apperr.Invalid("text must not contain the NUL character") } return Message{ - ID: id, - ChatID: chatID, - SenderID: senderID, - Text: text, - CreatedAt: ident.TimeOf(id), + ID: id, + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: ident.TimeOf(id), + AttachmentIDs: slices.Clone(attachmentIDs), }, nil } // SameContent reports whether two messages say the same thing, whatever their // ids and times. func (m Message) SameContent(other Message) bool { - return m.Text == other.Text + return m.Text == other.Text && slices.Equal(m.AttachmentIDs, other.AttachmentIDs) } diff --git a/internal/chat/service.go b/internal/chat/service.go index ac71e96f..39a66f7c 100644 --- a/internal/chat/service.go +++ b/internal/chat/service.go @@ -78,15 +78,16 @@ type MessagePayload struct { ChatID uuid.UUID IdempotencyKey string Text string + AttachmentIDs []uuid.UUID } -// Send checks access and the text on every attempt, a replay included, as the +// Send checks access and content on every attempt, a replay included, as the // contract requires of a repeat. func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) { if err := s.requireMember(ctx, actor, p.ChatID); err != nil { return Message{}, err } - message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text) + message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text, p.AttachmentIDs...) if err != nil { return Message{}, err } diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go index 1098708c..c7a16681 100644 --- a/internal/chat/service_test.go +++ b/internal/chat/service_test.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "os" + "reflect" "strings" "sync/atomic" "testing" @@ -217,14 +218,14 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) { if err != nil { t.Fatal(err) } - if len(page.Items) != 1 || page.Items[0] != first || page.Next == nil || *page.Next != first.ID { + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], first) || page.Next == nil || *page.Next != first.ID { t.Fatalf("first page = %+v", page) } rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1) if err != nil { t.Fatal(err) } - if len(rest.Items) != 1 || rest.Items[0] != second || rest.Next != nil { + if len(rest.Items) != 1 || !reflect.DeepEqual(rest.Items[0], second) || rest.Next != nil { t.Fatalf("second page = %+v", rest) } if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) { diff --git a/internal/chat/store/attachments_test.go b/internal/chat/store/attachments_test.go new file mode 100644 index 00000000..ef4c3ac8 --- /dev/null +++ b/internal/chat/store/attachments_test.go @@ -0,0 +1,312 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, + hash[:], status, id, chatID, owner) + if err != nil { + t.Fatal(err) + } + return id +} + +func attachedMessage(t *testing.T, c, sender uuid.UUID, ids ...uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), c, sender, "", ids...) + if err != nil { + t.Fatal(err) + } + return m +} + +func assertCounts(t *testing.T, messages, keys, joins int) { + t.Helper() + var gotMessages, gotKeys, gotJoins int + err := db.Pool.QueryRow(context.Background(), ` + SELECT (SELECT count(*) FROM messages), + (SELECT count(*) FROM message_idempotency_keys), + (SELECT count(*) FROM message_attachments)`).Scan(&gotMessages, &gotKeys, &gotJoins) + if err != nil || gotMessages != messages || gotKeys != keys || gotJoins != joins { + t.Fatalf("counts = %d/%d/%d, %v; want %d/%d/%d", gotMessages, gotKeys, gotJoins, err, messages, keys, joins) + } +} + +func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + m := attachedMessage(t, c.ID, alice, ident.New()) + _, err := f.messages.AppendOnce(f.ctx, m, "unknown", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("unknown attachment: %v; want not found", err) + } + var messages, keys int + err = db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM messages), (SELECT count(*) FROM message_idempotency_keys)`).Scan(&messages, &keys) + if err != nil || messages != 0 || keys != 0 { + t.Fatalf("failed validation persisted %d messages/%d keys: %v", messages, keys, err) + } +} + +func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) { + for _, status := range []string{"foreign", "failed", "pending", "mixed"} { + t.Run(status, func(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + other := f.mustChat(t, alice) + ready := f.attachment(t, c.ID, bob, "ready") + var invalid uuid.UUID + want := apperr.KindConflict + switch status { + case "foreign": + invalid = f.attachment(t, other.ID, alice, "ready") + want = apperr.KindNotFound + case "mixed": + ready = f.attachment(t, c.ID, bob, "failed") + invalid = ident.New() + want = apperr.KindNotFound + default: + invalid = f.attachment(t, c.ID, alice, status) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ready, invalid), "bad", keyTTL) + if err == nil || apperr.KindOf(err) != want { + t.Fatalf("validation: %v; want %v", err, want) + } + assertCounts(t, 0, 0, 0) + good := f.attachment(t, c.ID, bob, "ready") + f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad") + assertCounts(t, 1, 1, 1) + }) + } +} + +func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + a, b := f.attachment(t, c.ID, bob, "ready"), f.attachment(t, c.ID, bob, "ready") + first := attachedMessage(t, c.ID, alice, b, a) + got := f.appendOnce(t, first, "k") + if !reflect.DeepEqual(got, first) { + t.Fatalf("append = %+v, want %+v", got, first) + } + replayed := f.appendOnce(t, attachedMessage(t, c.ID, alice, a, b), "k") + if !reflect.DeepEqual(replayed, first) { + t.Fatalf("replay must return original ordered attachments: %+v", replayed) + } + assertCounts(t, 1, 1, 2) + expire(t) + second := attachedMessage(t, c.ID, bob, a) + f.appendOnce(t, second, "k2") + replacement := attachedMessage(t, c.ID, alice, a, b) + f.appendOnce(t, replacement, "k") + assertCounts(t, 3, 2, 5) + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 1) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("first page = %+v, %v", page, err) + } + page, err = f.messages.ListByChat(f.ctx, c.ID, &first.ID, 10) + if err != nil || !reflect.DeepEqual(page, []chat.Message{second, replacement}) { + t.Fatalf("remaining page = %+v, %v", page, err) + } +} + +func TestAppendOnceRevalidatesAttachmentOnReplay(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "ready") + first := attachedMessage(t, c.ID, alice, a) + f.appendOnce(t, first, "k") + if _, err := db.Pool.Exec(f.ctx, `UPDATE files SET status = 'failed'`); err != nil { + t.Fatal(err) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("failed replay attachment = %v, want conflict", err) + } + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("failed blob must preserve history attachment ids: %+v, %v", page, err) + } + assertCounts(t, 1, 1, 1) +} + +func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "ready") + // A database fault at the join boundary must not leave a key or message. + _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments DROP CONSTRAINT test_reject_join`); err != nil { + t.Error(err) + } + }() + if _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL); err == nil { + t.Fatal("join fault must fail append") + } + assertCounts(t, 0, 0, 0) +} + +func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a, b := f.attachment(t, c.ID, alice, "ready"), f.attachment(t, c.ID, alice, "ready") + const requests = 20 + results := make(chan chat.Message, requests) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range requests { + ids := []uuid.UUID{a, b} + if i%2 != 0 { + ids = []uuid.UUID{b, a} + } + m := attachedMessage(t, c.ID, alice, ids...) + wg.Go(func() { + <-start + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + results <- got + }) + } + close(start) + wg.Wait() + close(results) + var winner *chat.Message + n := 0 + for got := range results { + n++ + if winner == nil { + winner = &got + } else if !reflect.DeepEqual(got, *winner) { + t.Fatalf("concurrent replay differs: %+v, want %+v", got, *winner) + } + } + if n != requests { + t.Fatalf("got %d successful replies, want %d", n, requests) + } + assertCounts(t, 1, 1, 2) +} + +type queryCounter struct{ count atomic.Int32 } + +func (c *queryCounter) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + c.count.Add(1) + return ctx +} +func (*queryCounter) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} + +func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + var ids []uuid.UUID + for range 10 { + ids = append(ids, f.attachment(t, c.ID, alice, "ready")) + } + cfg := db.Pool.Config() + counter := &queryCounter{} + cfg.ConnConfig.Tracer = counter + pool, err := pgxpool.NewWithConfig(f.ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := pool.Ping(f.ctx); err != nil { + t.Fatal(err) + } + repo := store.NewMessages(pool) + counter.count.Store(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids[0]), "one", keyTTL); err != nil { + t.Fatal(err) + } + one := counter.count.Swap(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids...), "ten", keyTTL); err != nil { + t.Fatal(err) + } + ten := counter.count.Swap(0) + if one != ten { + t.Fatalf("queries grew from %d to %d for 1 vs 10 attachments", one, ten) + } + page, err := repo.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 2 || !reflect.DeepEqual(page[1].AttachmentIDs, ids) { + t.Fatalf("history = %+v, %v", page, err) + } + if n := counter.count.Load(); n != 1 { + t.Fatalf("history used %d queries, want one", n) + } +} + +func TestAppendOnceRetriesWhenBoundKeyDisappears(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + f.appendOnce(t, first, "vanish") + next := f.newMessage(t, c.ID, alice) + // Simulate the missing-key boundary after an unsuccessful claim using a + // statement trigger. Its second invocation sees no key and is a no-op. + _, err := db.Pool.Exec(f.ctx, fmt.Sprintf(` + CREATE FUNCTION test_vanish_key() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM messages WHERE id = '%s'::uuid) THEN + DELETE FROM message_idempotency_keys WHERE key = 'vanish'; + END IF; + RETURN NULL; + END $$; + CREATE TRIGGER test_vanish_key AFTER INSERT ON messages + FOR EACH STATEMENT EXECUTE FUNCTION test_vanish_key()`, next.ID.String())) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `DROP TRIGGER test_vanish_key ON messages; DROP FUNCTION test_vanish_key()`); err != nil { + t.Error(err) + } + }() + ctx, cancel := context.WithTimeout(f.ctx, 3*time.Second) + defer cancel() + got, err := f.messages.AppendOnce(ctx, next, "vanish", keyTTL) + if err != nil || got.ID != next.ID { + t.Fatalf("missing bound key = %+v, %v; want new message", got, err) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index 6554bf9f..cd6df783 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -19,6 +19,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "messenger/internal/chat" + "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" ) @@ -118,9 +119,8 @@ func NewMessages(pool *pgxpool.Pool) *Messages { return &Messages{pool: pool} } -// claimSQL binds the key and inserts the message in one statement, so the -// key's lock never spans a round trip. An expired key is taken over; a valid -// one leaves claimed, and so the message insert, empty. +// claimSQL binds the key and inserts the message in one statement. An expired +// key is taken over; a valid one leaves claimed and the message insert empty. const claimSQL = ` WITH claimed AS ( INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) @@ -132,30 +132,100 @@ const claimSQL = ` INSERT INTO messages (chat_id, id, sender_id, text, created_at) SELECT $2, message_id, $1, $6, $7 FROM claimed` -// The bound message is read by a second statement: the claim's snapshot -// cannot see a key that a concurrent request has just committed. +// The transaction keeps attachment readiness, message joins and the key atomic. +// The replay uses a new statement snapshot to see a concurrent winner. func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { - tag, err := r.pool.Exec(ctx, claimSQL, - m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) - if err != nil { - return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) - } - if tag.RowsAffected() == 1 { - return m, nil - } var stored chat.Message - err = r.pool.QueryRow(ctx, ` - SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at - FROM message_idempotency_keys k - JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id - WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, - m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) + err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil { + return err + } + for { + if err := ctx.Err(); err != nil { + return err + } + tag, err := tx.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return fmt.Errorf("insert message: %w", err) + } + if tag.RowsAffected() == 1 { + if len(m.AttachmentIDs) > 0 { + _, err := tx.Exec(ctx, ` + INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal) + SELECT $1, $2, id, (ord - 1)::smallint + FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)`, + m.ChatID, m.ID, m.AttachmentIDs) + if err != nil { + return fmt.Errorf("insert message attachments: %w", err) + } + } + stored = m + return nil + } + stored, err = scanMessage(tx.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key)) + if errors.Is(err, pgx.ErrNoRows) { + // A vanished key is available again; retry its atomic claim. + continue + } + if err != nil { + return fmt.Errorf("select message by key: %w", err) + } + return nil + } + }) if err != nil { - return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) + return chat.Message{}, postgres.Classify(err) } return stored, nil } +func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids []uuid.UUID) error { + if len(ids) == 0 { + return nil + } + // Shared row locks prevent deletion or a readiness change until joins commit. + rows, err := tx.Query(ctx, ` + SELECT f.status + FROM attachments a + JOIN files f ON f.hash = a.file_hash + WHERE a.chat_id = $1 AND a.id = ANY($2::uuid[]) + ORDER BY a.id + FOR SHARE OF a, f`, chatID, ids) + if err != nil { + return fmt.Errorf("check attachments: %w", err) + } + statuses, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return fmt.Errorf("read attachment status: %w", err) + } + if len(statuses) != len(ids) { + return apperr.NotFound("attachment not found") + } + for _, status := range statuses { + if status != "ready" { + return apperr.Conflict("attachment is not ready") + } + } + return nil +} + +func scanMessage(row pgx.Row) (chat.Message, error) { + var m chat.Message + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs) + if len(m.AttachmentIDs) == 0 { + m.AttachmentIDs = nil + } + return m, err +} + // DeleteExpiredKeys skips rows locked by another cleaner or a request rather // than waiting for them. func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) { @@ -179,14 +249,16 @@ func interval(d time.Duration) string { return fmt.Sprintf("%d milliseconds", d.Milliseconds()) } -// ListByChat is the history query; scripts/explain-history.sql runs the same -// statement for the EXPLAIN evidence. +// ListByChat keeps keyset pagination on messages and collects each ordered +// attachment array in the same statement. func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) { rows, err := r.pool.Query(ctx, ` - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 - AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM messages m + WHERE m.chat_id = $1 + AND m.id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) ORDER BY id LIMIT $3`, chatID, after, limit) @@ -194,9 +266,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) } messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { - var m chat.Message - err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) - return m, err + return scanMessage(row) }) if err != nil { return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go index 8e843d23..d2b01270 100644 --- a/internal/chat/store/postgres_test.go +++ b/internal/chat/store/postgres_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "reflect" "sync" "testing" "time" @@ -214,7 +215,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 3 || got[0] != first || got[1] != second || got[2] != third { + if len(got) != 3 || !reflect.DeepEqual(got[0], first) || !reflect.DeepEqual(got[1], second) || !reflect.DeepEqual(got[2], third) { t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got) } } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 936b8ddc..10b4b523 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -15,6 +15,9 @@ import ( "time" ) +// Allow a 30s binary request, its independent 3s cleanup, and 5s margin. +const defaultShutdownTimeout = 38 * time.Second + type Logging struct { Env string Level slog.Level @@ -43,6 +46,7 @@ type Config struct { IdempotencyTTL time.Duration DBMaxConns int32 DebugAddr string + S3 S3 } type Migrate struct { @@ -58,7 +62,7 @@ func Load(lookup Lookup) (Config, error) { cfg := Config{ Host: "0.0.0.0", Port: 8080, - ShutdownTimeout: 8 * time.Second, + ShutdownTimeout: defaultShutdownTimeout, MaxBodyBytes: 64 * 1024, SessionTTL: time.Hour, DrainDelay: 2 * time.Second, @@ -136,6 +140,11 @@ func Load(lookup Lookup) (Config, error) { if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { return Config{}, err } + if _, ok := lookup("S3_ENDPOINT"); ok { + if cfg.S3, err = LoadS3(lookup); err != nil { + return Config{}, err + } + } return cfg, nil } diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go index e58ffe19..914795b5 100644 --- a/internal/platform/config/config_test.go +++ b/internal/platform/config/config_test.go @@ -56,7 +56,7 @@ func TestDefaults(t *testing.T) { if cfg.Env != "lab" || cfg.Level != slog.LevelInfo { t.Fatalf("cfg = %+v", cfg) } - if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 { + if cfg.ShutdownTimeout != 38*time.Second || cfg.MaxBodyBytes != 65536 { t.Fatalf("cfg = %+v", cfg) } if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL { @@ -70,6 +70,19 @@ func TestDefaults(t *testing.T) { } } +func TestDefaultShutdownTimeoutCoversBinaryLifecycle(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatal(err) + } + // Binary requests can use 30s, followed by 3s independent cleanup. Keep + // another 5s for shutdown scheduling and closing the remaining resources. + budget := 30*time.Second + 3*time.Second + 5*time.Second + if cfg.ShutdownTimeout < budget { + t.Fatalf("default shutdown budget %v cannot cover binary request and cleanup budget %v", cfg.ShutdownTimeout, budget) + } +} + func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} { cases := map[string]func(map[string]string){ diff --git a/internal/platform/config/s3.go b/internal/platform/config/s3.go new file mode 100644 index 00000000..919abf8a --- /dev/null +++ b/internal/platform/config/s3.go @@ -0,0 +1,55 @@ +package config + +import ( + "fmt" + "strconv" +) + +type S3 struct { + Endpoint, Region, Bucket string + AccessKey, SecretKey string + MaxBytes, MaxPixels int64 +} + +func LoadS3(lookup Lookup) (S3, error) { + cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000} + for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} { + if v, ok := lookup(key); ok { + *dest = v + } + } + var err error + if cfg.AccessKey, err = secret(lookup, "S3_ACCESS_KEY_ID_FILE"); err != nil { + return S3{}, err + } + if cfg.SecretKey, err = secret(lookup, "S3_SECRET_ACCESS_KEY_FILE"); err != nil { + return S3{}, err + } + for _, p := range []struct { + key string + dest *int64 + max int64 + }{{"MAX_UPLOAD_BYTES", &cfg.MaxBytes, 10485760}, {"MAX_IMAGE_PIXELS", &cfg.MaxPixels, 40000000}} { + if v, ok := lookup(p.key); ok { + n, e := strconv.ParseInt(v, 10, 64) + if e != nil || n < 1 || n > p.max { + return S3{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max) + } + *p.dest = n + } + } + return cfg, nil +} + +// ReconcileS3 selects the separate listing identity used by background scans. +func ReconcileS3(lookup Lookup) Lookup { + return func(key string) (string, bool) { + if key == "S3_ACCESS_KEY_ID_FILE" { + return lookup("S3_RECONCILE_ACCESS_KEY_ID_FILE") + } + if key == "S3_SECRET_ACCESS_KEY_FILE" { + return lookup("S3_RECONCILE_SECRET_ACCESS_KEY_FILE") + } + return lookup(key) + } +} diff --git a/internal/platform/config/s3_test.go b/internal/platform/config/s3_test.go new file mode 100644 index 00000000..d05e9121 --- /dev/null +++ b/internal/platform/config/s3_test.go @@ -0,0 +1,27 @@ +package config_test + +import ( + "messenger/internal/platform/config" + "testing" +) + +func TestS3LimitsAndSecretFiles(t *testing.T) { + values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")} + cfg, err := config.LoadS3(env(values)) + if err != nil { + t.Fatal(err) + } + if cfg.AccessKey != "access" || cfg.SecretKey != "private" || cfg.MaxBytes != 10485760 || cfg.MaxPixels != 40000000 { + t.Fatal("S3 defaults or secret trimming broken") + } + for _, tc := range []struct{ k, v string }{{"MAX_UPLOAD_BYTES", "0"}, {"MAX_UPLOAD_BYTES", "10485761"}, {"MAX_IMAGE_PIXELS", "40000001"}, {"MAX_IMAGE_PIXELS", "x"}} { + v := map[string]string{} + for k, x := range values { + v[k] = x + } + v[tc.k] = tc.v + if _, err := config.LoadS3(env(v)); err == nil { + t.Fatalf("accepted %s=%s", tc.k, tc.v) + } + } +} diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index b4844e9d..052e0d2d 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -123,6 +123,11 @@ func Timeout(d time.Duration) gin.HandlerFunc { // decoder as *http.MaxBytesError. func bodyLimit(maxBytes int64) gin.HandlerFunc { return func(c *gin.Context) { + // The authenticated binary handler installs its own 10 MiB limit. + if c.Request.Method == http.MethodPost && c.FullPath() == "/api/v1/chats/:chat_id/attachments" { + c.Next() + return + } if c.Request.Body != nil { c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) } diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go new file mode 100644 index 00000000..7a6385e9 --- /dev/null +++ b/internal/platform/s3store/bootstrap_test.go @@ -0,0 +1,142 @@ +package s3store_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/s3store" +) + +func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) { + var ready atomic.Bool + var reads atomic.Int32 + var mu sync.Mutex + objects := map[string]string{".health/storage-v1": "messenger s3 readiness v1\n"} + var freshReads, deleted int + requested := make(chan struct{}, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + if r.Method == "GET" && key == ".health/storage-v1" { + reads.Add(1) + if !ready.Load() { + select { + case requested <- struct{}{}: + default: + } + w.WriteHeader(500) + io.WriteString(w, `InternalErrorvolume not registered`) + return + } + } + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "GET": + value, ok := objects[key] + if !ok { + w.WriteHeader(404) + io.WriteString(w, `NoSuchKey`) + return + } + if key != ".health/storage-v1" { + freshReads++ + } + w.Header().Set("ETag", `"probe"`) + w.Header().Set("Last-Modified", "Mon, 16 Nov 2026 08:00:00 GMT") + w.Header().Set("Content-Length", fmt.Sprint(len(value))) + io.WriteString(w, value) + case "PUT": + var body io.Reader = r.Body + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") { + body = httputil.NewChunkedReader(body) + } + data, err := io.ReadAll(io.LimitReader(body, 1024)) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + objects[key] = string(data) + w.Header().Set("ETag", `"probe"`) + case "DELETE": + if key == ".health/storage-v1" { + t.Error("deleted retained readiness marker") + } + delete(objects, key) + deleted++ + w.WriteHeader(204) + default: + t.Errorf("unexpected storage probe: %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer server.Close() + s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- s.Bootstrap(ctx) }() + select { + case err := <-done: + t.Fatalf("bootstrap returned before reading retained storage: %v", err) + case <-requested: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case err := <-done: + t.Fatalf("bootstrap accepted unavailable retained storage: %v", err) + case <-time.After(40 * time.Millisecond): + } + ready.Store(true) + if err := <-done; err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if reads.Load() < 2 || freshReads == 0 || deleted == 0 || len(objects) != 1 { + t.Fatalf("incomplete readiness proof: retainedReads=%d freshReads=%d deletes=%d remaining=%d", reads.Load(), freshReads, deleted, len(objects)) + } +} + +func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + w.WriteHeader(503) + io.WriteString(w, `ServiceUnavailablevolume not registered`) + })) + defer server.Close() + s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + if err := s.Bootstrap(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("storage readiness escaped its deadline: %v", err) + } + if time.Since(started) > time.Second { + t.Fatal("readiness did not honor caller deadline") + } +} diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go new file mode 100644 index 00000000..9798a581 --- /dev/null +++ b/internal/platform/s3store/integration_test.go @@ -0,0 +1,270 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "messenger/internal/attachment" + "messenger/internal/platform/s3store" + "messenger/internal/testenv" +) + +func TestSeaweedStore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + newClient := func(access, secret string) *minio.Client { + t.Helper() + c, err := minio.New(strings.TrimPrefix(s.Endpoint, "http://"), &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Region: s.Region, BucketLookup: minio.BucketLookupPath, MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return c + } + api := newClient(s.APIAccessKey, s.APISecretKey) + admin := newClient(s.AdminAccessKey, s.AdminSecretKey) + denied := func(t *testing.T, err error) { + t.Helper() + var response minio.ErrorResponse + if !errors.As(err, &response) || response.StatusCode != http.StatusForbidden { + t.Fatalf("expected access denied, got %v", err) + } + } + t.Run("remote gRPC cannot modify identity cache", func(t *testing.T) { + conn, err := grpc.NewClient("passthrough:///"+s.GRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) { + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, "tcp", address) + })) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + opCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + err = conn.Invoke(opCtx, "/messaging_pb.SeaweedS3IamCache/PutIdentity", &emptypb.Empty{}, &emptypb.Empty{}) + if code := status.Code(err); code != codes.PermissionDenied && code != codes.Unauthenticated { + t.Fatalf("remote IAM mutation was not rejected before argument validation: %v", err) + } + }) + t.Run("bootstrap is idempotent and API cannot administer", func(t *testing.T) { + if err := s.Admin.Bootstrap(ctx); err != nil { + t.Fatal(err) + } + denied(t, s.APIStore.Bootstrap(ctx)) + denied(t, api.MakeBucket(ctx, "forbidden-bucket", minio.MakeBucketOptions{Region: s.Region})) + denied(t, api.RemoveBucket(ctx, s.Bucket)) + buckets, err := api.ListBuckets(ctx) + // SeaweedFS returns a filtered empty result for an authenticated caller + // without List rights, rather than rejecting ListBuckets outright. + if err != nil || len(buckets) != 0 { + t.Fatalf("API enumerated buckets: count=%d error=%v", len(buckets), err) + } + _, err = s.APIStore.List(ctx, "", "", 1) + denied(t, err) + }) + t.Run("known length nonseekable put copy stat and stream", func(t *testing.T) { + const payload = "nonseekable image payload" + r := struct{ io.Reader }{strings.NewReader(payload)} + started := time.Now() + if err := s.APIStore.Put(ctx, "staging/source", r, int64(len(payload)), "image/png"); err != nil { + t.Fatal(err) + } + t.Logf("cold first PUT: %s", time.Since(started)) + if err := s.APIStore.Copy(ctx, "staging/source", "blobs/durable"); err != nil { + t.Fatal(err) + } + obj, err := s.APIStore.Stat(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + if obj.Key != "blobs/durable" || obj.Size != 25 || obj.Modified.IsZero() { + t.Fatalf("wrong metadata: %+v", obj) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != payload { + t.Fatalf("wrong body: %q", b) + } + meta, err := api.StatObject(ctx, s.Bucket, "blobs/durable", minio.StatObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if meta.ContentType != "image/png" { + t.Fatalf("copy lost MIME: %s", meta.ContentType) + } + for range 2 { + if err := s.APIStore.Delete(ctx, "staging/source"); err != nil { + t.Fatal(err) + } + } + if _, err := s.APIStore.Stat(ctx, "staging/source"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("API delete did not remove staging: %v", err) + } + }) + t.Run("anonymous get put head and listing are refused", func(t *testing.T) { + client := &http.Client{Timeout: 3 * time.Second} + defer client.CloseIdleConnections() + for _, tc := range []struct{ method, path string }{{"GET", "/"}, {"GET", "/" + s.Bucket + "?list-type=2"}, {"GET", "/" + s.Bucket + "/blobs/durable"}, {"HEAD", "/" + s.Bucket + "/blobs/durable"}, {"PUT", "/" + s.Bucket + "/anonymous"}} { + req, _ := http.NewRequestWithContext(ctx, tc.method, s.Endpoint+tc.path, strings.NewReader("x")) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("anonymous %s %s: %d", tc.method, tc.path, resp.StatusCode) + } + } + }) + t.Run("identities cannot escape their bucket", func(t *testing.T) { + if err := admin.MakeBucket(ctx, "private-other", minio.MakeBucketOptions{Region: s.Region}); err != nil { + t.Fatal(err) + } + if _, err := admin.PutObject(ctx, "private-other", "secret", strings.NewReader("private"), 7, minio.PutObjectOptions{DisableMultipart: true}); err != nil { + t.Fatal(err) + } + for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} { + outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1]) + if err != nil { + t.Fatal(err) + } + _, err = outside.Stat(ctx, "secret") + denied(t, err) + _, err = outside.Open(ctx, "secret") + denied(t, err) + denied(t, outside.Put(ctx, "secret", strings.NewReader("x"), 1, "image/png")) + denied(t, outside.Delete(ctx, "secret")) + _, err = outside.List(ctx, "", "", 1) + denied(t, err) + c := newClient(pair[0], pair[1]) + _, err = c.CopyObject(ctx, minio.CopyDestOptions{Bucket: s.Bucket, Object: "blobs/stolen"}, minio.CopySrcOptions{Bucket: "private-other", Object: "secret"}) + denied(t, err) + } + }) + t.Run("exclusive lexicographic pages including absent cursor", func(t *testing.T) { + for _, key := range []string{"page/z", "page/a", "page/a/nested", "page/b", "page/c"} { + if err := s.Store.Put(ctx, key, strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + } + var got []string + after := "" + for { + page, err := s.Store.List(ctx, "page/", after, 2) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + for _, obj := range page { + if obj.Size != 1 || obj.Modified.IsZero() { + t.Fatalf("wrong list metadata: %+v", obj) + } + got = append(got, obj.Key) + } + after = page[len(page)-1].Key + if len(got) > 5 { + t.Fatal("pagination did not advance") + } + } + if strings.Join(got, ",") != "page/a,page/a/nested,page/b,page/c,page/z" { + t.Fatalf("wrong order: %v", got) + } + page, err := s.Store.List(ctx, "page/", "page/az", 2) + if err != nil { + t.Fatal(err) + } + if len(page) != 2 || page[0].Key != "page/b" || page[1].Key != "page/c" { + t.Fatalf("absent cursor: %+v", page) + } + }) + t.Run("missing objects differ from bucket and permission failures", func(t *testing.T) { + if _, err := s.Store.Stat(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("stat missing: %v", err) + } + if _, err := s.Store.Open(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("open missing: %v", err) + } + if err := s.Store.Copy(ctx, "absent", "blobs/absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("copy missing: %v", err) + } + for range 2 { + if err := s.Store.Delete(ctx, "absent"); err != nil { + t.Fatal(err) + } + } + missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey) + if err != nil { + t.Fatal(err) + } + if _, err := missingBucket.Stat(ctx, "absent"); err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("missing bucket treated as missing object: %v", err) + } + }) + t.Run("short readers are not successful uploads", func(t *testing.T) { + if err := s.Store.Put(ctx, "staging/short", struct{ io.Reader }{strings.NewReader("short")}, 100, "image/png"); err == nil { + t.Fatal("short body accepted") + } + }) + t.Run("container replacement preserves volume data and authentication", func(t *testing.T) { + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != "nonseekable image payload" { + t.Fatalf("object lost on replacement: %q", b) + } + if _, err := s.Store.List(ctx, "page/", "", 2); err != nil { + t.Fatal(err) + } + }) + t.Run("outage is bounded and never missing", func(t *testing.T) { + if err := s.Stop(ctx); err != nil { + t.Fatal(err) + } + opCtx, stop := context.WithTimeout(ctx, 150*time.Millisecond) + defer stop() + start := time.Now() + _, err := s.Store.Stat(opCtx, "blobs/durable") + if err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("outage treated as missing: %v", err) + } + if time.Since(start) > 2*time.Second { + t.Fatal("outage exceeded caller deadline") + } + }) +} diff --git a/internal/platform/s3store/readiness.go b/internal/platform/s3store/readiness.go new file mode 100644 index 00000000..1f6830e3 --- /dev/null +++ b/internal/platform/s3store/readiness.go @@ -0,0 +1,114 @@ +package s3store + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/minio/minio-go/v7" + + "messenger/internal/attachment" +) + +const ( + storageMarkerKey = ".health/storage-v1" + storageMarkerBody = "messenger s3 readiness v1\n" +) + +var errStorageMarker = errors.New("s3: storage readiness marker has unexpected contents") + +// WaitReady is a startup barrier, deliberately separate from API readiness and +// ordinary object operations. Gateway authentication and bucket metadata can be +// available before the master elects its leader and registers retained volumes. +// Read the persistent marker before writing anything, then prove new writes, +// reads and deletes work. The marker lives outside the reconciler's namespaces. +func (s *Store) WaitReady(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + readyCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + probeKey := ".health/check-" + rand.Text() + dirty := false + defer func() { + if dirty { + cleanupCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + defer stop() + _ = s.Delete(cleanupCtx, probeKey) + } + }() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + err := s.checkStorage(readyCtx, probeKey, &dirty) + if err == nil { + return nil + } + if permanentStorageError(err) { + return err + } + select { + case <-readyCtx.Done(): + return fmt.Errorf("s3: wait for readable and writable storage: %w (last probe: %v)", readyCtx.Err(), err) + case <-ticker.C: + } + } +} + +func (s *Store) checkStorage(ctx context.Context, probeKey string, dirty *bool) error { + err := s.readStorageMarker(ctx, storageMarkerKey) + if errors.Is(err, attachment.ErrObjectMissing) { + // Only a genuinely absent marker permits its first creation. Never + // overwrite retained data to hide an unavailable volume or a bad marker. + if err := s.Put(ctx, storageMarkerKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + err = s.readStorageMarker(ctx, storageMarkerKey) + } + if err != nil { + return err + } + *dirty = true + if err := s.Put(ctx, probeKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + if err := s.readStorageMarker(ctx, probeKey); err != nil { + return err + } + if err := s.Delete(ctx, probeKey); err != nil { + return err + } + *dirty = false + return nil +} + +func (s *Store) readStorageMarker(ctx context.Context, key string) error { + r, err := s.Open(ctx, key) + if err != nil { + return err + } + data, readErr := io.ReadAll(io.LimitReader(r, int64(len(storageMarkerBody))+1)) + closeErr := r.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + if string(data) != storageMarkerBody { + return errStorageMarker + } + return nil +} + +func permanentStorageError(err error) bool { + if errors.Is(err, errStorageMarker) { + return true + } + var response minio.ErrorResponse + return errors.As(err, &response) && response.StatusCode >= 400 && response.StatusCode < 500 && response.StatusCode != 408 && response.StatusCode != 429 +} diff --git a/internal/platform/s3store/readiness_test.go b/internal/platform/s3store/readiness_test.go new file mode 100644 index 00000000..c92c5fb2 --- /dev/null +++ b/internal/platform/s3store/readiness_test.go @@ -0,0 +1,48 @@ +package s3store_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "messenger/internal/testenv" +) + +func TestRecreateReturnsWithRetainedObjectReadable(t *testing.T) { + // Three independent server replacements each have the production 60-second + // startup budget; no per-request transport timeout is changed by this test. + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + defer func() { + if t.Failed() { + t.Log(s.Diagnostics(context.Background())) + } + }() + const payload = "retained storage readiness proof" + if err := s.Store.Put(ctx, "blobs/retained", strings.NewReader(payload), int64(len(payload)), "application/octet-stream"); err != nil { + t.Fatal(err) + } + for attempt := range 3 { + started := time.Now() + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + r, err := s.Store.Open(ctx, "blobs/retained") + if err != nil { + t.Fatalf("replacement %d returned before retained bytes were readable (%s): %v", attempt+1, time.Since(started), err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil || string(data) != payload { + t.Fatalf("replacement %d lost retained bytes: %q %v", attempt+1, data, err) + } + t.Logf("replacement %d ready and retained bytes verified in %s", attempt+1, time.Since(started)) + } +} diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go new file mode 100644 index 00000000..2c4b9b9e --- /dev/null +++ b/internal/platform/s3store/store.go @@ -0,0 +1,198 @@ +package s3store + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + + "messenger/internal/attachment" +) + +const MaxPageSize = 500 + +type Store struct { + client *minio.Core + region string + bucket string +} + +var _ attachment.Blobs = (*Store)(nil) + +// New validates configuration without contacting S3, so text-only service startup +// and readiness do not depend on object storage being reachable. +func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) { + u, err := url.Parse(endpoint) + if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment") + } + if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" { + return nil, errors.New("s3: region and credentials are required") + } + if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil { + return nil, fmt.Errorf("s3: invalid bucket: %w", err) + } + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: time.Second, KeepAlive: 30 * time.Second}).DialContext, + TLSHandshakeTimeout: time.Second, + // The first write can wait for the single master's election and volume + // creation. The caller's shorter context still takes precedence. + ResponseHeaderTimeout: 30 * time.Second, + ExpectContinueTimeout: time.Second, + IdleConnTimeout: 30 * time.Second, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 32, + MaxConnsPerHost: 64, + DisableCompression: true, + } + client, err := minio.NewCore(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: u.Scheme == "https", + Region: region, + BucketLookup: minio.BucketLookupPath, + Transport: transport, + // A consumed upload stream cannot be replayed. Let the caller decide + // whether to retry a whole operation within its own deadline. + MaxRetries: 1, + }) + if err != nil { + return nil, fmt.Errorf("s3: client: %w", err) + } + return &Store{client: client, region: region, bucket: bucket}, nil +} + +// Bootstrap is used only by the admin init command, never by API startup. +func (s *Store) Bootstrap(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region}) + code := minio.ToErrorResponse(err).Code + if err != nil && code != "BucketAlreadyOwnedByYou" { + return wrap("create bucket", err) + } + return s.WaitReady(ctx) +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + if err := ctx.Err(); err != nil { + return err + } + if size < 0 || r == nil { + return errors.New("s3: PUT requires a reader and known nonnegative length") + } + _, err := s.client.PutObject(ctx, s.bucket, key, r, size, "", "", minio.PutObjectOptions{ + ContentType: mime, + DisableMultipart: true, + }) + return wrap("put object", err) +} + +func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Core.GetObject performs GET eagerly and returns the streaming body; + // Client.GetObject is lazy and would defer missing/access errors to Read. + r, _, _, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, wrap("open object", err) + } + return r, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (attachment.Object, error) { + if err := ctx.Err(); err != nil { + return attachment.Object{}, err + } + obj, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + // HEAD has no XML error body. The SDK synthesizes NoSuchKey for any + // object-path 404, including a missing bucket; confirm the bucket first. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + exists, bucketErr := s.client.BucketExists(ctx, s.bucket) + if bucketErr != nil { + return attachment.Object{}, wrap("stat bucket", bucketErr) + } + if !exists { + return attachment.Object{}, errors.New("s3: stat object: bucket does not exist") + } + } + return attachment.Object{}, wrap("stat object", err) + } + return attachment.Object{Key: key, Size: obj.Size, Modified: obj.LastModified}, nil +} + +func (s *Store) Copy(ctx context.Context, source, dest string) error { + if err := ctx.Err(); err != nil { + return err + } + _, err := s.client.Client.CopyObject(ctx, + minio.CopyDestOptions{Bucket: s.bucket, Object: dest}, + minio.CopySrcOptions{Bucket: s.bucket, Object: source}, + ) + return wrap("copy object", err) +} + +func (s *Store) Delete(ctx context.Context, key string) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) + // S3 DELETE is idempotent even when the object no longer exists. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return wrap("delete object", err) +} + +// List returns at most limit keys in byte-lexicographic order, strictly after +// after. The synchronous iterator stops before fetching the next page. +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit < 1 || limit > MaxPageSize { + return nil, fmt.Errorf("s3: list limit must be between 1 and %d", MaxPageSize) + } + listCtx, cancel := context.WithCancel(ctx) + defer cancel() + objects := make([]attachment.Object, 0, limit) + for obj := range s.client.ListObjectsIter(listCtx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, StartAfter: after, Recursive: true, MaxKeys: limit, + }) { + if obj.Err != nil { + return nil, wrap("list objects", obj.Err) + } + objects = append(objects, attachment.Object{Key: obj.Key, Size: obj.Size, Modified: obj.LastModified}) + if len(objects) == limit { + break + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + return objects, nil +} + +func wrap(op string, err error) error { + if err == nil { + return nil + } + // Never treat authorization errors, a missing bucket, or an outage as a + // missing blob: the reconciler would otherwise mark valid files failed. + if code := minio.ToErrorResponse(err).Code; code == "NoSuchKey" || code == "NoSuchObject" { + return fmt.Errorf("s3: %s: %w", op, attachment.ErrObjectMissing) + } + return fmt.Errorf("s3: %s: %w", op, err) +} diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go new file mode 100644 index 00000000..cac6af56 --- /dev/null +++ b/internal/platform/s3store/store_test.go @@ -0,0 +1,171 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/s3store" +) + +func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) })) + defer server.Close() + if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatal("constructor performed network I/O") + } + for _, tc := range []struct{ name, endpoint, region, bucket, access, secret string }{ + {"scheme", "ftp://localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"path", "http://localhost:8333/path", "us-east-1", "attachments", "access", "secret"}, + {"userinfo", "http://user:password@localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"query", "http://localhost:8333?x=1", "us-east-1", "attachments", "access", "secret"}, + {"fragment", "http://localhost:8333/#x", "us-east-1", "attachments", "access", "secret"}, + {"region", server.URL, "", "attachments", "access", "secret"}, + {"bucket", server.URL, "us-east-1", "INVALID", "access", "secret"}, + {"access", server.URL, "us-east-1", "attachments", "", "secret"}, + {"secret", server.URL, "us-east-1", "attachments", "access", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil { + t.Fatal("accepted invalid configuration") + } + }) + } +} + +func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) { + first := make(chan struct{}) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodPut || r.URL.Path != "/attachments/staging/stream" || r.URL.RawQuery != "" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(400) + return + } + if r.ContentLength <= 0 || len(r.TransferEncoding) != 0 { + t.Error("PUT is not a known-length single request") + } + b := make([]byte, 1) + if _, err := io.ReadFull(r.Body, b); err != nil { + t.Error(err) + return + } + close(first) + if _, err := io.Copy(io.Discard, r.Body); err != nil { + t.Error(err) + return + } + w.Header().Set("ETag", `"probe"`) + })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // One signing chunk can be buffered; the remaining bytes wait for the server. + r, w := io.Pipe() + defer r.Close() + go func() { + defer w.Close() + if _, err := w.Write(make([]byte, 256*1024)); err != nil { + return + } + select { + case <-first: + case <-ctx.Done(): + _ = w.CloseWithError(ctx.Err()) + return + } + _, _ = w.Write(make([]byte, 256*1024)) + }() + if err := store.Put(ctx, "staging/stream", r, 512*1024, "image/png"); err != nil { + t.Fatal(err) + } + if requests.Load() != 1 { + t.Fatalf("got %d requests, want one PUT", requests.Load()) + } +} + +func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for name, call := range map[string]func() error{ + "put": func() error { return store.Put(ctx, "key", strings.NewReader("x"), 1, "image/png") }, + "open": func() error { + r, e := store.Open(ctx, "key") + if r != nil { + r.Close() + } + return e + }, + "stat": func() error { _, e := store.Stat(ctx, "key"); return e }, + "copy": func() error { return store.Copy(ctx, "a", "b") }, + "delete": func() error { return store.Delete(ctx, "key") }, + "list": func() error { _, e := store.List(ctx, "", "", 1); return e }, + "bootstrap": func() error { return store.Bootstrap(ctx) }, + } { + t.Run(name, func(t *testing.T) { + if err := call(); !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want canceled", err) + } + }) + } + if err := store.Put(context.Background(), "key", strings.NewReader("x"), -1, "image/png"); err == nil { + t.Fatal("accepted unknown length") + } + for _, limit := range []int{-1, 0, 501, 1000000000} { + if _, err := store.List(context.Background(), "", "", limit); err == nil { + t.Fatalf("accepted unbounded page %d", limit) + } + } + if requests.Load() != 0 { + t.Fatalf("invalid calls reached server %d times", requests.Load()) + } +} + +func TestListStopsAfterOneBoundedPage(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Query().Get("start-after") != "blobs/a" || r.URL.Query().Get("prefix") != "blobs/" || r.URL.Query().Get("max-keys") != "2" { + t.Errorf("wrong list query: %s", r.URL.RawQuery) + } + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`) + })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + objects, err := store.List(context.Background(), "blobs/", "blobs/a", 2) + if err != nil { + t.Fatal(err) + } + if len(objects) != 2 || objects[0].Key != "blobs/b" || objects[1].Key != "blobs/c" { + t.Fatalf("wrong page: %+v", objects) + } + if requests.Load() != 1 { + t.Fatalf("prefetched %d pages", requests.Load()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 64f7abf8..661c5e10 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -24,8 +24,8 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, srv := &http.Server{ Handler: h, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + ReadTimeout: 35 * time.Second, + WriteTimeout: 35 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 64 * 1024, ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 8a2caa39..c8685c72 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,21 +1,198 @@ package server_test import ( + "bytes" "context" + "fmt" "io" "log/slog" "net" "net/http" + "os" + "path/filepath" + "strconv" + "sync" "testing" "time" "github.com/gin-gonic/gin" "messenger/internal/health" + "messenger/internal/platform/config" "messenger/internal/platform/httpx" "messenger/internal/server" ) +func serverDefaultConfig(t *testing.T) config.Config { + t.Helper() + values := make(map[string]string) + for key, content := range map[string]string{ + "DATABASE_URL_FILE": "postgres://unused:unused@localhost:5432/unused", + "SESSION_STORE_URL_FILE": "redis://localhost:6379/0", + } { + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + values[key] = path + } + cfg, err := config.Load(func(key string) (string, bool) { value, ok := values[key]; return value, ok }) + if err != nil { + t.Fatal(err) + } + return cfg +} + +func TestDefaultShutdownAllowsActiveBinaryTransfers(t *testing.T) { + for _, direction := range []string{"upload", "download"} { + t.Run(direction, func(t *testing.T) { + t.Parallel() + cfg := serverDefaultConfig(t) + payload := []byte("binary\x00\xff\x01request-and-response\x00\xfe") + prefix := payload[:3] + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + bodyReader, bodyWriter := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + requestCtx, cancelRequest := context.WithTimeout(context.Background(), 25*time.Second) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + probes := health.New() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + if direction == "upload" { + first := make([]byte, len(prefix)) + if _, err := io.ReadFull(r.Body, first); err != nil { + return + } + close(started) + rest, err := io.ReadAll(r.Body) + if err != nil || !bytes.Equal(append(first, rest...), payload) { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = w.Write(payload) + return + } + _, _ = w.Write(prefix) + w.(http.Flusher).Flush() + close(started) + select { + case <-release: + _, _ = w.Write(payload[len(prefix):]) + case <-r.Context().Done(): + } + }) + stopped, exited := make(chan error, 1), make(chan struct{}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + go func() { + defer close(exited) + stopped <- server.Run(ctx, log, ln, handler, probes, cfg.DrainDelay, cfg.ShutdownTimeout) + }() + t.Cleanup(func() { + cancel() + cancelRequest() + unblock() + _ = bodyReader.Close() + _ = bodyWriter.Close() + _ = ln.Close() + select { + case <-exited: + case <-time.After(3 * time.Second): + t.Error("server did not stop after transfer cleanup") + } + }) + client := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + defer client.CloseIdleConnections() + method := http.MethodGet + var body io.Reader + if direction == "upload" { + method = http.MethodPut + body = bodyReader + } + req, err := http.NewRequestWithContext(requestCtx, method, "http://"+ln.Addr().String()+"/binary", body) + if err != nil { + t.Fatal(err) + } + if direction == "upload" { + req.ContentLength = int64(len(payload)) + req.Header.Set("Content-Type", "application/octet-stream") + } + completed := make(chan error, 1) + go func() { + resp, err := client.Do(req) + if err != nil { + completed <- err + return + } + defer resp.Body.Close() + got, err := io.ReadAll(resp.Body) + if err == nil && (resp.StatusCode != http.StatusOK || !bytes.Equal(got, payload)) { + err = fmt.Errorf("binary response status=%d body=%q, want exact payload", resp.StatusCode, got) + } + completed <- err + }() + if direction == "upload" { + if _, err := bodyWriter.Write(prefix); err != nil { + t.Fatal(err) + } + } + select { + case <-started: + case err := <-completed: + t.Fatalf("transfer failed before shutdown: %v", err) + case <-requestCtx.Done(): + t.Fatal("transfer never became active") + } + cancel() + // Nine seconds inside Shutdown exceeds the former 8s budget. Include + // the configured drain phase so it cannot hide a premature force-close. + timer := time.NewTimer(cfg.DrainDelay + 9*time.Second) + defer timer.Stop() + select { + case err := <-stopped: + t.Fatalf("shutdown interrupted active %s before binary transfer completed: %v", direction, err) + case err := <-completed: + t.Fatalf("active %s ended before remaining bytes were released: %v", direction, err) + case <-timer.C: + } + if direction == "upload" { + if _, err := bodyWriter.Write(payload[len(prefix):]); err != nil { + t.Fatal(err) + } + if err := bodyWriter.Close(); err != nil { + t.Fatal(err) + } + } else { + unblock() + } + select { + case err := <-completed: + if err != nil { + t.Fatalf("active %s was truncated during shutdown: %v", direction, err) + } + case <-requestCtx.Done(): + t.Fatal("binary transfer did not finish after release") + } + select { + case err := <-stopped: + if err != nil { + t.Fatalf("graceful shutdown: %v", err) + } + case <-requestCtx.Done(): + t.Fatal("server did not finish graceful shutdown") + } + if probes.IsReady() { + t.Fatal("readiness remained enabled after shutdown") + } + }) + } +} + // newHandler builds the smallest handler with probes; the lifecycle does not // depend on the business modules or their stores. func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go new file mode 100644 index 00000000..062c9f99 --- /dev/null +++ b/internal/testenv/s3.go @@ -0,0 +1,206 @@ +package testenv + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/google/uuid" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "messenger/internal/platform/s3store" +) + +const S3Image = "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0" + +// S3 keeps independent API, reconciler and bootstrap identities. Store uses the +// reconciler identity so tests can inspect and clean up their objects. +type S3 struct { + Store, APIStore, Admin *s3store.Store + Endpoint, Region, Bucket string + GRPCAddress string + AccessKey, SecretKey string + APIAccessKey, APISecretKey string + AdminAccessKey, AdminSecretKey string + container testcontainers.Container + volume string + identities, options []byte +} + +func StartS3(ctx context.Context) (*S3, error) { + useDockerDesktopPipe() + s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()} + keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey} + for _, key := range keys { + b := make([]byte, 24) + if _, err := rand.Read(b); err != nil { + return nil, err + } + *key = hex.EncodeToString(b) + } + type credential struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + } + type identity struct { + Name string `json:"name"` + Credentials []credential `json:"credentials"` + Actions []string `json:"actions"` + } + config := struct { + Identities []identity `json:"identities"` + }{Identities: []identity{ + {"bootstrap", []credential{{s.AdminAccessKey, s.AdminSecretKey}}, []string{"Admin", "Read", "Write", "List"}}, + {"api", []credential{{s.APIAccessKey, s.APISecretKey}}, []string{"Read:" + s.Bucket, "Write:" + s.Bucket}}, + {"reconciler", []credential{{s.AccessKey, s.SecretKey}}, []string{"List:" + s.Bucket, "Read:" + s.Bucket, "Write:" + s.Bucket}}, + }} + var err error + s.identities, err = json.Marshal(config) + if err != nil { + return nil, err + } + _, source, _, _ := runtime.Caller(0) + s.options, err = os.ReadFile(filepath.Join(filepath.Dir(source), "../../infra/s3/seaweedfs.options")) + if err != nil { + return nil, fmt.Errorf("read S3 options: %w", err) + } + startCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + if err := s.launch(startCtx); err != nil { + s.Close() + return nil, err + } + if err := s.Admin.Bootstrap(startCtx); err != nil { + s.Close() + return nil, err + } + return s, nil +} + +func (s *S3) launch(ctx context.Context) error { + pids := int64(256) + firstStatus := 0 + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: S3Image, + Cmd: []string{"server", "-options=/etc/seaweedfs/lab.options"}, + Env: map[string]string{"GOMEMLIMIT": "460MiB"}, + ExposedPorts: []string{"8333/tcp", "18333/tcp"}, + Mounts: testcontainers.Mounts(testcontainers.VolumeMount(s.volume, "/data")), + Files: []testcontainers.ContainerFile{ + {Reader: bytes.NewReader(s.identities), ContainerFilePath: "/run/secrets/s3_identities", FileMode: 0444}, + {Reader: bytes.NewReader(s.options), ContainerFilePath: "/etc/seaweedfs/lab.options", FileMode: 0444}, + }, + HostConfigModifier: func(h *container.HostConfig) { + h.Resources.Memory = 512 * 1024 * 1024 + h.Resources.NanoCPUs = 1_000_000_000 + h.Resources.PidsLimit = &pids + h.PortBindings = nat.PortMap{"8333/tcp": {{HostIP: "127.0.0.1"}}, "18333/tcp": {{HostIP: "127.0.0.1"}}} + }, + // The first reachable S3 response must already reject anonymous access. + WaitingFor: wait.ForHTTP("/").WithPort("8333/tcp").WithStatusCodeMatcher(func(code int) bool { firstStatus = code; return true }).WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + s.container = c + if err != nil { + return fmt.Errorf("start S3: %w", err) + } + if firstStatus != http.StatusForbidden { + return fmt.Errorf("first anonymous S3 response was %d, expected 403", firstStatus) + } + s.Endpoint, err = c.PortEndpoint(ctx, "8333/tcp", "http") + if err != nil { + return err + } + s.GRPCAddress, err = c.PortEndpoint(ctx, "18333/tcp", "") + if err != nil { + return err + } + s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey) + if err != nil { + return err + } + s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey) + if err != nil { + return err + } + s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey) + return err +} + +func (s *S3) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return s.container.Stop(ctx, &timeout) +} + +func (s *S3) Start(ctx context.Context) error { + if err := s.container.Start(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Recreate replaces the container while keeping only its named /data volume. +// Bootstrap is deliberately not called: bucket metadata must survive as well. +func (s *S3) Recreate(ctx context.Context) error { + if err := testcontainers.TerminateContainer(s.container); err != nil { + return err + } + s.container = nil + if err := s.launch(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Diagnostics retains bounded startup evidence without exposing fixture keys. +func (s *S3) Diagnostics(ctx context.Context) string { + if s.container == nil { + return "no container" + } + r, err := s.container.Logs(ctx) + if err != nil { + return err.Error() + } + defer r.Close() + redact := strings.NewReplacer(s.AccessKey, "", s.SecretKey, "", s.APIAccessKey, "", s.APISecretKey, "", s.AdminAccessKey, "", s.AdminSecretKey, "") + var lines []string + scan := bufio.NewScanner(io.LimitReader(r, 256*1024)) + for scan.Scan() { + line := scan.Text() + lower := strings.ToLower(line) + if strings.Contains(lower, "volume") || strings.Contains(lower, "master") || strings.Contains(lower, "leader") || strings.Contains(lower, "error") || strings.Contains(lower, "lookup") { + lines = append(lines, redact.Replace(line)) + if len(lines) > 60 { + lines = lines[1:] + } + } + } + return strings.Join(lines, "\n") +} + +func (s *S3) Close() { + if s.container != nil { + _ = testcontainers.TerminateContainer(s.container) + } + if strings.HasPrefix(s.volume, "lab4-s3-test-") { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) }) + } +} diff --git a/migrations/00003_attachments.sql b/migrations/00003_attachments.sql new file mode 100644 index 00000000..abb4756e --- /dev/null +++ b/migrations/00003_attachments.sql @@ -0,0 +1,35 @@ +-- +goose Up +CREATE TABLE files ( + hash bytea PRIMARY KEY CHECK (octet_length(hash) = 32), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + mime_type text NOT NULL CHECK (mime_type IN ('image/png', 'image/jpeg')), + status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + file_hash bytea NOT NULL REFERENCES files(hash), + created_at timestamptz NOT NULL +); + +-- The reconciler must distinguish a shared file from an unused file. +CREATE INDEX attachments_file_hash_idx ON attachments(file_hash); + +CREATE TABLE message_attachments ( + chat_id uuid NOT NULL, + message_id uuid NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments(id), + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY(chat_id, message_id, attachment_id), + UNIQUE(chat_id, message_id, ordinal), + FOREIGN KEY(chat_id, message_id) REFERENCES messages(chat_id, id) +); + +-- +goose Down +DROP TABLE message_attachments; +DROP TABLE attachments; +DROP TABLE files; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh index ae7df48b..081d1023 100755 --- a/scripts/init-secrets.sh +++ b/scripts/init-secrets.sh @@ -14,11 +14,7 @@ for f in $files; do present=$((present + 1)) fi done -if [ "$present" -eq 4 ]; then - echo "secrets already exist in ./secrets, kept" - exit 0 -fi -if [ "$present" -ne 0 ]; then +if [ "$present" -ne 0 ] && [ "$present" -ne 4 ]; then echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2 exit 1 fi @@ -27,16 +23,74 @@ fi random() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n' } -pg=$(random) -kv=$(random) - # The directory is private; the files stay readable for the unprivileged # users inside the containers, which see them through bind mounts. umask 022 mkdir -p secrets chmod 700 secrets -printf '%s\n' "$pg" > secrets/postgres_password -printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url -printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf -printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url -echo "created ./secrets" +if [ "$present" -eq 0 ]; then + pg=$(random) + kv=$(random) + printf '%s\n' "$pg" > secrets/postgres_password + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url + printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf + printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url +fi + +# Read just the bucket setting, never execute the local .env file as a script. +bucket=${S3_BUCKET:-} +if [ -z "$bucket" ] && [ -f .env ]; then + bucket=$(sed -n 's/^S3_BUCKET=//p' .env | tail -n 1 | tr -d '\r') +fi +bucket=${bucket:-messenger-attachments} +case "$bucket" in + *[!a-z0-9.-]*|.*|*.|-*|*-|*..*|*.-*|*-.*) + echo "S3_BUCKET must be a lowercase S3 bucket name" >&2; exit 1 ;; +esac +if [ "${#bucket}" -lt 3 ] || [ "${#bucket}" -gt 63 ]; then + echo "S3_BUCKET must contain 3 to 63 characters" >&2; exit 1 +fi + +s3files="s3_admin_access_key_id s3_admin_secret_access_key s3_access_key_id s3_secret_access_key s3_reconcile_access_key_id s3_reconcile_secret_access_key s3_identities.json" +s3present=0 +for f in $s3files; do + if [ -s "secrets/$f" ]; then s3present=$((s3present + 1)); fi +done +if [ "$s3present" -eq 7 ]; then + if ! grep -Fq "\"Read:$bucket\"" secrets/s3_identities.json; then + echo "existing S3 identities use another bucket; keep S3_BUCKET consistent with the retained volume" >&2 + exit 1 + fi + echo "existing PostgreSQL, Valkey and S3 secrets kept" + exit 0 +fi +if [ "$s3present" -ne 0 ]; then + echo "S3 secrets are incomplete; restore the s3_* secret files together; PostgreSQL and Valkey secrets were kept" >&2 + exit 1 +fi + +admin_key=$(random) +admin_secret=$(random) +api_key=$(random) +api_secret=$(random) +recon_key=$(random) +recon_secret=$(random) +printf '%s\n' "$admin_key" > secrets/s3_admin_access_key_id +printf '%s\n' "$admin_secret" > secrets/s3_admin_secret_access_key +printf '%s\n' "$api_key" > secrets/s3_access_key_id +printf '%s\n' "$api_secret" > secrets/s3_secret_access_key +printf '%s\n' "$recon_key" > secrets/s3_reconcile_access_key_id +printf '%s\n' "$recon_secret" > secrets/s3_reconcile_secret_access_key + +# The server loads all identities before listening. There is no anonymous +# identity, and no post-start allow-all provisioning phase. +cat > secrets/s3_identities.json </dev/null 2>&1; then + stable=$((stable + 1)) + if [ "$stable" -ge 3 ]; then + echo "balancer readiness confirmed" + exit 0 + fi + else + stable=0 + fi + attempt=$((attempt + 1)) + sleep 1 +done +echo "balancer did not become ready within 60 probes" >&2 +exit 1