Добавьте S3-вложения с дедупликацией и безопасной сверкой хранилищ
This commit is contained in:
+3
-1
@@ -11,10 +11,12 @@ SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_RECONCILE_ACCESS_KEY_ID_FILE=/run/secrets/s3_reconcile_access_key_id
|
||||
S3_RECONCILE_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_reconcile_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ COPY cmd ./cmd
|
||||
COPY internal ./internal
|
||||
COPY migrations ./migrations
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init
|
||||
|
||||
# The load generator gets an image of its own; the API image does not carry it.
|
||||
FROM build AS loadgen-build
|
||||
@@ -25,7 +25,7 @@ ENTRYPOINT ["/usr/local/bin/loadgen"]
|
||||
# Last, so that `build: .` builds it; it also runs the migrate service.
|
||||
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
|
||||
RUN adduser -D -H -u 10001 app
|
||||
COPY --from=build /out/server /out/migrate /usr/local/bin/
|
||||
COPY --from=build /out/server /out/migrate /out/reconcile /out/s3init /usr/local/bin/
|
||||
USER app
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
// Command reconcile reports attachment inconsistencies as JSON lines. Mutations
|
||||
// require --apply; --deep also streams referenced objects to verify SHA256.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/postgres"
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
code := run(ctx, os.Args[1:], os.Stdout, os.Stderr)
|
||||
stop()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, out, errOut io.Writer) int {
|
||||
flags := flag.NewFlagSet("reconcile", flag.ContinueOnError)
|
||||
flags.SetOutput(errOut)
|
||||
apply := flags.Bool("apply", false, "apply repairs (default: read-only dry run)")
|
||||
deep := flags.Bool("deep", false, "stream referenced objects and verify SHA256 and size")
|
||||
grace := flags.Duration("grace", time.Hour, "minimum file/object age; must be at least 2m")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if flags.NArg() != 0 || *grace < 2*time.Minute {
|
||||
fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m")
|
||||
return 2
|
||||
}
|
||||
dbConfig, err := config.LoadMigrate(os.LookupEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
}
|
||||
// Compose mounts the reconciler identity at the standard credential paths.
|
||||
s3Config, err := config.LoadS3(os.LookupEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
}
|
||||
blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
}
|
||||
db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "reconcile:", err)
|
||||
return 1
|
||||
}
|
||||
defer db.Close()
|
||||
r, err := attachment.NewReconciler(db, blobs, *grace)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "reconcile:", err)
|
||||
return 2
|
||||
}
|
||||
// A broken output stream cancels the scan so repairs cannot continue after
|
||||
// the caller loses the audit trail. Print partial statistics on scan errors.
|
||||
scanCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
encoder := json.NewEncoder(out)
|
||||
var outputErr error
|
||||
stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) {
|
||||
if outputErr == nil {
|
||||
outputErr = encoder.Encode(f)
|
||||
if outputErr != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
})
|
||||
if outputErr == nil {
|
||||
outputErr = encoder.Encode(stats)
|
||||
}
|
||||
if outputErr != nil {
|
||||
fmt.Fprintln(errOut, "reconcile output:", outputErr)
|
||||
return 1
|
||||
}
|
||||
if runErr != nil {
|
||||
fmt.Fprintln(errOut, "reconcile:", runErr)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "bucket bootstrap failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
cfg, err := config.LoadS3(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
return s.Bootstrap(ctx)
|
||||
}
|
||||
@@ -17,10 +17,12 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/platform/postgres"
|
||||
"messenger/internal/platform/s3store"
|
||||
"messenger/internal/platform/valkey"
|
||||
"messenger/internal/server"
|
||||
)
|
||||
@@ -59,6 +61,26 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
return err
|
||||
}
|
||||
defer sessions.Close()
|
||||
var blobs attachment.Blobs
|
||||
var reconciler *attachment.Reconciler
|
||||
if cfg.S3.Endpoint != "" {
|
||||
blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ln, err := net.Listen("tcp", cfg.Addr())
|
||||
if err != nil {
|
||||
@@ -77,6 +99,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
MaxBodyBytes: cfg.MaxBodyBytes,
|
||||
InstanceID: alias,
|
||||
IdempotencyTTL: cfg.IdempotencyTTL,
|
||||
Blobs: blobs,
|
||||
MaxUploadBytes: cfg.S3.MaxBytes,
|
||||
MaxImagePixels: cfg.S3.MaxPixels,
|
||||
})
|
||||
var debugLn net.Listener
|
||||
if cfg.DebugAddr != "" {
|
||||
@@ -89,6 +114,27 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
var background sync.WaitGroup
|
||||
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
|
||||
if reconciler != nil {
|
||||
background.Go(func() {
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
scan, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
stats, err := reconciler.Run(scan, true, false, nil)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("attachment cleanup incomplete", "error", err)
|
||||
} else {
|
||||
log.Info("attachment cleanup", "scanned_files", stats.ScannedFiles, "repaired", stats.Repaired)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if debugLn != nil {
|
||||
background.Go(func() {
|
||||
if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil {
|
||||
|
||||
+131
-6
@@ -1,5 +1,5 @@
|
||||
# Local stand: HAProxy in front of interchangeable API replicas over shared
|
||||
# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data
|
||||
# PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data
|
||||
# network has no way out. The load-test knobs are in compose.bench.yaml.
|
||||
name: lab4
|
||||
|
||||
@@ -38,11 +38,24 @@ services:
|
||||
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
# Go does not read the cgroup memory limit.
|
||||
GOMEMLIMIT: 230MiB
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
# The background reconciler uses a separate identity with List rights.
|
||||
S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id
|
||||
S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key
|
||||
# One decoder per replica. Progressive JPEG retains coefficient buffers;
|
||||
# a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer.
|
||||
GOMEMLIMIT: 1536MiB
|
||||
secrets:
|
||||
- database_url
|
||||
- session_store_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_reconcile_access_key_id
|
||||
- s3_reconcile_secret_access_key
|
||||
networks:
|
||||
- edge
|
||||
- data
|
||||
@@ -53,14 +66,16 @@ services:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
# Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL.
|
||||
stop_grace_period: 15s
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
# Binary deadline 30s plus compensation, shutdown and drain margin.
|
||||
stop_grace_period: 45s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 256M
|
||||
memory: 2G
|
||||
|
||||
migrate:
|
||||
image: mtusi-messenger:lab4
|
||||
@@ -138,9 +153,105 @@ services:
|
||||
# maxmemory 256mb (valkey.conf) plus the server's own overhead.
|
||||
memory: 320M
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0
|
||||
command: ["server", "-options=/etc/seaweedfs/lab.options"]
|
||||
environment:
|
||||
GOMEMLIMIT: 460MiB
|
||||
secrets:
|
||||
- s3_identities
|
||||
volumes:
|
||||
- ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro
|
||||
# Both volume data and the image's /data/filerldb2 metadata live here.
|
||||
- s3data:/data
|
||||
networks:
|
||||
- data
|
||||
healthcheck:
|
||||
# A reachable anonymous request must be refused, including before init.
|
||||
test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 15s
|
||||
start_interval: 1s
|
||||
stop_grace_period: 30s
|
||||
pids_limit: 256
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
pids: 256
|
||||
|
||||
s3-init:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
entrypoint: ["/usr/local/bin/s3init"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
GOMEMLIMIT: 100MiB
|
||||
secrets:
|
||||
- source: s3_admin_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_admin_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
networks:
|
||||
- data
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
reconcile:
|
||||
profiles: ["recon"]
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
entrypoint: ["/usr/local/bin/reconcile"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
GOMEMLIMIT: 100MiB
|
||||
secrets:
|
||||
- database_url
|
||||
- source: s3_reconcile_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_reconcile_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
networks:
|
||||
- data
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
s3data:
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
@@ -156,3 +267,17 @@ secrets:
|
||||
file: ./secrets/postgres_password
|
||||
valkey_auth:
|
||||
file: ./secrets/valkey_auth.conf
|
||||
s3_identities:
|
||||
file: ./secrets/s3_identities.json
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_reconcile_access_key_id:
|
||||
file: ./secrets/s3_reconcile_access_key_id
|
||||
s3_reconcile_secret_access_key:
|
||||
file: ./secrets/s3_reconcile_secret_access_key
|
||||
|
||||
@@ -3,15 +3,20 @@ module messenger
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
github.com/docker/docker v28.3.3+incompatible
|
||||
github.com/docker/go-connections v0.6.0
|
||||
github.com/gin-gonic/gin v1.12.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/pressly/goose/v3 v3.27.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
github.com/testcontainers/testcontainers-go v0.39.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0
|
||||
github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
google.golang.org/grpc v1.84.0
|
||||
google.golang.org/protobuf v1.36.12
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -31,9 +36,8 @@ require (
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/creack/pty v1.1.24 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker v28.3.3+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.12 // indirect
|
||||
@@ -54,12 +58,15 @@ require (
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mdelapenya/tlscert v0.2.0 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
@@ -72,20 +79,24 @@ require (
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.59.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sethvargo/go-retry v0.4.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
github.com/stretchr/testify v1.12.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.3.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
|
||||
@@ -97,11 +108,11 @@ require (
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/arch v0.22.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
|
||||
golang.org/x/net v0.59.0 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/time v0.11.0 // indirect
|
||||
google.golang.org/grpc v1.84.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
|
||||
@@ -79,6 +79,8 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
|
||||
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -100,8 +102,11 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
@@ -116,6 +121,12 @@ github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5L
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
|
||||
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
@@ -145,8 +156,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
@@ -162,12 +175,14 @@ github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBP
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw=
|
||||
github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -180,6 +195,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts=
|
||||
@@ -188,6 +204,8 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwN
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE=
|
||||
github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g=
|
||||
github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
@@ -202,6 +220,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE=
|
||||
@@ -249,8 +269,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -281,6 +301,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8=
|
||||
@@ -290,6 +312,8 @@ google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2Q
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
|
||||
@@ -55,6 +55,8 @@ frontend entry
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
# Binary handlers have a 30s budget, ordinary API handlers retain 5s.
|
||||
http-request set-timeout server 35s if { path_reg ^/api/v1/chats/[^/]+/attachments$|^/api/v1/attachments/[^/]+/content$ }
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# Private object storage
|
||||
|
||||
`s3` runs SeaweedFS 4.53 in one process. The image is pinned by digest in
|
||||
`compose.yaml` and `internal/testenv/s3.go`. `seaweedfs.options` is shared by
|
||||
the stand and the disposable integration-test server.
|
||||
|
||||
S3 HTTP (8333) and SeaweedFS's companion gRPC (18333) listen on the private
|
||||
container network. This image has no flag to disable the companion listener;
|
||||
its identity-cache mutations reject remote peers, which the integration test
|
||||
checks. Master, volume and filer HTTP/gRPC endpoints bind to loopback.
|
||||
No host port is published. `/data`
|
||||
contains both object volumes and the image's LevelDB filer metadata; the
|
||||
named `s3data` volume must be retained together with PostgreSQL data.
|
||||
|
||||
Run `sh scripts/init-secrets.sh` before starting the stand. It preserves the
|
||||
existing PostgreSQL and Valkey credentials and creates three independent S3
|
||||
identities plus the static server configuration. Credential values are never
|
||||
printed. Repeated initialization keeps all existing keys. Set `S3_BUCKET` in
|
||||
the environment or the local `.env` before the first initialization; changing
|
||||
it later requires a planned migration of identities and retained data.
|
||||
|
||||
The server loads its static identity secret before serving requests. There is
|
||||
no anonymous identity. The bootstrap identity has administrative rights and
|
||||
its individual key files are mounted only in `s3-init`. The API identity has
|
||||
`Read:<bucket>` and `Write:<bucket>`; it cannot list objects or administer
|
||||
buckets. The reconciler identity adds `List:<bucket>`. All three identities
|
||||
are necessarily present in the S3 server's own identity configuration secret.
|
||||
|
||||
`s3-init` creates the bucket idempotently and waits for storage before API
|
||||
replicas start. An anonymous 403 and bucket metadata alone do not prove that
|
||||
retained volume locations have registered with the elected master. Bootstrap
|
||||
first reads a persistent `.health/storage-v1` marker (creating it only when
|
||||
absent), then verifies a fresh write/read/delete. This startup-only barrier
|
||||
uses the caller's deadline, capped at 60 seconds; ordinary S3 operations and
|
||||
API readiness do not retry through it. The marker is outside the reconciler's
|
||||
`staging/` and `blobs/` namespaces. `scripts/restart.sh` already runs this
|
||||
bootstrap before restoring API replicas.
|
||||
|
||||
The bootstrap uses `/usr/local/bin/s3init`. The optional `recon` profile runs
|
||||
`/usr/local/bin/reconcile`, with reconciler credentials mapped to the standard
|
||||
`S3_ACCESS_KEY_ID_FILE` and `S3_SECRET_ACCESS_KEY_FILE` paths:
|
||||
|
||||
```sh
|
||||
docker compose --profile recon run --rm reconcile
|
||||
docker compose --profile recon run --rm reconcile --apply
|
||||
```
|
||||
|
||||
The API receives its own standard S3 key files. Its background reconciliation
|
||||
worker must construct a separate store from
|
||||
`S3_RECONCILE_ACCESS_KEY_ID_FILE` and `S3_RECONCILE_SECRET_ACCESS_KEY_FILE`;
|
||||
listing with the API identity is denied. S3 is intentionally absent from API
|
||||
readiness so text traffic can remain available during an object-store outage.
|
||||
|
||||
The S3 cgroup is limited to one CPU, 512 MiB, and 256 processes. Upload limits
|
||||
are four concurrent requests and 64 MiB in flight; reader buffers have a
|
||||
32 MiB budget and the chunk cache is disabled. Disk allocation is capped at
|
||||
64 volumes of approximately 128 MiB each (with normal volume rollover
|
||||
overhead); this is a lab capacity limit, not a filesystem quota.
|
||||
|
||||
Each API replica has 2 GiB and `GOMEMLIMIT=1536MiB`, with one image decoder.
|
||||
Pixels alone are not the full budget: progressive JPEG retains coefficients,
|
||||
and a wide RGBA64 PNG keeps two large scanlines beside its pixel buffer.
|
||||
The 40 MP RGB/CMYK progressive JPEG and wide-PNG measurements are recorded in
|
||||
`evidence/decoder-memory-*.json`; this does not change the S3 service's 512 MiB limit.
|
||||
|
||||
Verify the adapter and real server with Docker running:
|
||||
|
||||
```sh
|
||||
go test -race -count=1 ./internal/platform/s3store
|
||||
```
|
||||
|
||||
`testenv.StartS3(ctx)` returns `Store` with reconciler rights, `APIStore`,
|
||||
`Admin`, endpoint and credentials. `Recreate(ctx)` replaces only its own
|
||||
container while keeping its named volume and does not recreate the bucket.
|
||||
It waits for the same retained-marker and write/read/delete barrier before
|
||||
returning. `Start(ctx)` does so as well. Always call `Close()` to remove the
|
||||
fixture's container and volume.
|
||||
@@ -0,0 +1,31 @@
|
||||
dir=/data
|
||||
ip=127.0.0.1
|
||||
ip.bind=127.0.0.1
|
||||
master.telemetry=false
|
||||
master.volumeSizeLimitMB=128
|
||||
volume.port=9340
|
||||
volume.max=64
|
||||
volume.index=leveldb
|
||||
volume.fileSizeLimitMB=10
|
||||
volume.concurrentUploadLimitMB=64
|
||||
volume.concurrentDownloadLimitMB=64
|
||||
volume.readBufferSizeMB=1
|
||||
volume.preStopSeconds=1
|
||||
filer=true
|
||||
filer.exposeDirectoryData=false
|
||||
filer.disableDirListing=true
|
||||
filer.concurrentFileUploadLimit=4
|
||||
filer.concurrentUploadLimitMB=64
|
||||
s3=true
|
||||
s3.ip.bind=0.0.0.0
|
||||
s3.port=8333
|
||||
s3.config=/run/secrets/s3_identities
|
||||
s3.autoCreateBucket=false
|
||||
s3.iam=false
|
||||
s3.port.iceberg=0
|
||||
s3.port.lance=0
|
||||
s3.concurrentFileUploadLimit=4
|
||||
s3.concurrentUploadLimitMB=64
|
||||
s3.readerCacheSizeMB=32
|
||||
s3.cacheCapacityMB=0
|
||||
s3.idleTimeout=35
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
attachmentstore "messenger/internal/attachment/store"
|
||||
"messenger/internal/auth"
|
||||
authstore "messenger/internal/auth/store"
|
||||
"messenger/internal/chat"
|
||||
@@ -42,6 +44,9 @@ type Deps struct {
|
||||
InstanceID string
|
||||
// IdempotencyTTL zero means the contract's 24 hours.
|
||||
IdempotencyTTL time.Duration
|
||||
Blobs attachment.Blobs
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
}
|
||||
|
||||
// defaultRequestTimeout stays below the server's 10s write timeout, so a
|
||||
@@ -76,6 +81,9 @@ func New(d Deps) *App {
|
||||
auth.NewHandler(sessions).Register(public, protected)
|
||||
user.NewHandler(users).Register(public, protected)
|
||||
chat.NewHandler(chats).Register(protected)
|
||||
attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels)
|
||||
binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions))
|
||||
attachment.NewHandler(attachments).Register(protected, binary)
|
||||
|
||||
return &App{Engine: engine, Health: probes, Chats: chats}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -415,11 +416,11 @@ func TestMessagesEndToEnd(t *testing.T) {
|
||||
}
|
||||
|
||||
page1 := ts.history(bob.token, room.ID, "?limit=1")
|
||||
if len(page1.Items) != 1 || page1.NextCursor == nil || page1.Items[0] != first {
|
||||
if len(page1.Items) != 1 || page1.NextCursor == nil || !reflect.DeepEqual(page1.Items[0], first) {
|
||||
t.Fatalf("page1 = %+v", page1)
|
||||
}
|
||||
page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor))
|
||||
if len(page2.Items) != 1 || page2.NextCursor != nil || page2.Items[0] != second {
|
||||
if len(page2.Items) != 1 || page2.NextCursor != nil || !reflect.DeepEqual(page2.Items[0], second) {
|
||||
t.Fatalf("page2 = %+v", page2)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package app_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
|
||||
func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
actor := ts.user("deadline_user", "User")
|
||||
ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond})
|
||||
var held []*pgxpool.Conn
|
||||
for range db.Pool.Config().MaxConns {
|
||||
conn, err := db.Pool.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held = append(held, conn)
|
||||
}
|
||||
defer func() {
|
||||
for _, conn := range held {
|
||||
conn.Release()
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
|
||||
defer cancel()
|
||||
req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx)
|
||||
req.Header.Set("Authorization", "Bearer "+actor.token)
|
||||
rec := httptest.NewRecorder()
|
||||
started := time.Now()
|
||||
ts.a.Engine.ServeHTTP(rec, req)
|
||||
if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
|
||||
t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *tester) binary(method, path, token, mime string, data []byte, length int64) response {
|
||||
ts.t.Helper()
|
||||
req := httptest.NewRequest(method, path, bytes.NewReader(data))
|
||||
req.ContentLength = length
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if mime != "" {
|
||||
req.Header.Set("Content-Type", mime)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
ts.a.Engine.ServeHTTP(rec, req)
|
||||
res := response{status: rec.Code, requestID: rec.Header().Get("X-Request-Id"), challenge: rec.Header().Get("WWW-Authenticate"), raw: rec.Body.Bytes()}
|
||||
if res.requestID == "" || rec.Header().Get("X-Instance-Id") == "" {
|
||||
ts.t.Fatal("binary response missing correlation headers")
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s3, err := testenv.StartS3(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s3.Close()
|
||||
ts := newTester(t)
|
||||
ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, Blobs: s3.APIStore})
|
||||
alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E")
|
||||
room := ts.chat(alice.token, "files", alice.ID, bob.ID)
|
||||
path := "/api/v1/chats/" + room.ID + "/attachments?filename=" + url.QueryEscape("фото.png")
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data)))
|
||||
ts.expect(res, 201, "")
|
||||
a := decode[attachment.Response](t, res)
|
||||
digest := sha256.Sum256(data)
|
||||
if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "ready" || a.ErrorCode != nil || len(a.Variants) != 0 {
|
||||
t.Fatalf("attachment: %+v", a)
|
||||
}
|
||||
base := "/api/v1/attachments/" + a.ID
|
||||
for _, suffix := range []string{"", "/content"} {
|
||||
ts.expect(ts.do("GET", base+suffix, eve.token, nil, nil), 404, "not_found")
|
||||
}
|
||||
ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "")
|
||||
got := ts.binary("GET", base+"/content", bob.token, "", nil, 0)
|
||||
ts.expect(got, 200, "")
|
||||
if !bytes.Equal(got.raw, data) {
|
||||
t.Fatal("download bytes changed")
|
||||
}
|
||||
ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 400, "invalid_request")
|
||||
msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey())
|
||||
ts.expect(msg, 201, "")
|
||||
if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID {
|
||||
t.Fatal("lost attachment on message")
|
||||
}
|
||||
other := ts.chat(alice.token, "other", alice.ID)
|
||||
ts.expect(ts.do("POST", "/api/v1/chats/"+other.ID+"/messages", alice.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()), 404, "not_found")
|
||||
for _, tc := range []struct {
|
||||
name, mime string
|
||||
data []byte
|
||||
length int64
|
||||
want int
|
||||
}{
|
||||
{"MIME spoof", "image/jpeg", data, int64(len(data)), 415},
|
||||
{"broken body", "image/png", data[:len(data)/2], int64(len(data) / 2), 415},
|
||||
{"unsupported", "image/gif", data, int64(len(data)), 415},
|
||||
{"oversize", "image/png", nil, 10485761, 413},
|
||||
{"unknown length", "image/png", data, -1, 400},
|
||||
{"empty", "image/png", nil, 0, 400},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := ts.binary("POST", path, alice.token, tc.mime, tc.data, tc.length)
|
||||
if r.status != tc.want {
|
||||
t.Fatalf("got %d want %d: %s", r.status, tc.want, r.raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, name := range []string{"../x.png", `a\b.png`, "bad\nname", strings.Repeat("я", 256)} {
|
||||
ts.expect(ts.binary("POST", "/api/v1/chats/"+room.ID+"/attachments?filename="+url.QueryEscape(name), alice.token, "image/png", data, int64(len(data))), 400, "invalid_request")
|
||||
}
|
||||
var j bytes.Buffer
|
||||
if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 201, "")
|
||||
ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found")
|
||||
// An interrupted read is never a 500 or a successfully published identity.
|
||||
before := 0
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
short := ts.binary("POST", path, alice.token, "image/png", data[:10], int64(len(data)))
|
||||
if short.status != 400 && short.status != 503 {
|
||||
t.Fatalf("interrupted body: %d %s", short.status, short.raw)
|
||||
}
|
||||
after := 0
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after {
|
||||
t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err)
|
||||
}
|
||||
if err := s3.Stop(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts.expect(ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))), 503, "unavailable")
|
||||
ts.expect(ts.do("GET", base+"/content", bob.token, nil, nil), 503, "unavailable")
|
||||
ts.expect(ts.do("GET", "/health/ready", "", nil, nil), 200, "")
|
||||
ts.message(alice.token, room.ID, fmt.Sprintf("text works with S3 down %d", time.Now().UnixNano()))
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/uuid"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID, ChatID, OwnerID uuid.UUID
|
||||
Filename string
|
||||
FileHash []byte
|
||||
SizeBytes int64
|
||||
MIMEType, Status string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Repository interface {
|
||||
IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error)
|
||||
Publish(context.Context, Attachment, string) error
|
||||
GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrObjectMissing = errors.New("object missing")
|
||||
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
Modified time.Time
|
||||
}
|
||||
|
||||
type Blobs interface {
|
||||
Put(context.Context, string, io.Reader, int64, string) error
|
||||
Open(context.Context, string) (io.ReadCloser, error)
|
||||
Stat(context.Context, string) (Object, error)
|
||||
Copy(context.Context, string, string) error
|
||||
Delete(context.Context, string) error
|
||||
List(context.Context, string, string, int) ([]Object, error)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/httpx"
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
type Handler struct{ s *Service }
|
||||
|
||||
func NewHandler(s *Service) *Handler { return &Handler{s} }
|
||||
func (h *Handler) Register(protected, binary gin.IRouter) {
|
||||
binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
|
||||
protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata))
|
||||
binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func toResponse(a Attachment) Response {
|
||||
return Response{
|
||||
ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
|
||||
id, err := ident.Parse(c.Param("chat_id"))
|
||||
if err != nil {
|
||||
httpx.Error(c, apperr.Invalid("chat_id must be a UUID"))
|
||||
return
|
||||
}
|
||||
names := c.Request.URL.Query()["filename"]
|
||||
if len(names) != 1 {
|
||||
httpx.Error(c, apperr.Invalid("one filename is required"))
|
||||
return
|
||||
}
|
||||
if err := ValidateFilename(names[0]); err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
media, params, err := mime.ParseMediaType(c.GetHeader("Content-Type"))
|
||||
if err != nil || len(params) > 0 {
|
||||
httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
|
||||
return
|
||||
}
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes)
|
||||
a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
httpx.JSON(c, http.StatusCreated, toResponse(a))
|
||||
}
|
||||
func attachmentID(c *gin.Context) (uuid.UUID, error) {
|
||||
id, err := ident.Parse(c.Param("attachment_id"))
|
||||
if err != nil {
|
||||
return uuid.Nil, apperr.Invalid("attachment_id must be a UUID")
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
func (h *Handler) metadata(c *gin.Context, actor uuid.UUID) {
|
||||
id, err := attachmentID(c)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
a, err := h.s.Get(c.Request.Context(), actor, id)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
httpx.JSON(c, http.StatusOK, toResponse(a))
|
||||
}
|
||||
func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
|
||||
id, err := attachmentID(c)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
variant, present := c.GetQuery("variant")
|
||||
if present && variant != "original" {
|
||||
httpx.Error(c, apperr.Invalid("variant must be original"))
|
||||
return
|
||||
}
|
||||
a, r, err := h.s.Content(c.Request.Context(), actor, id)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename}))
|
||||
c.Header("X-Content-Type-Options", "nosniff")
|
||||
c.Header("Content-Type", a.MIMEType)
|
||||
c.Header("Content-Length", strconv.FormatInt(a.SizeBytes, 10))
|
||||
c.Status(http.StatusOK)
|
||||
if _, err := io.CopyN(c.Writer, r, a.SizeBytes); err != nil {
|
||||
slog.Error("download interrupted", "request_id", httpx.RequestID(c), "error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// LockHash serializes CAS publication and collection until the transaction ends.
|
||||
// A collision in the signed first eight digest bytes only adds serialization.
|
||||
func LockHash(ctx context.Context, tx pgx.Tx, hash []byte) error {
|
||||
if len(hash) != sha256.Size {
|
||||
return errors.New("attachment: lock requires a SHA256 hash")
|
||||
}
|
||||
_, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(hash[:8])))
|
||||
return err
|
||||
}
|
||||
|
||||
func BlobKey(hash []byte) string { return "blobs/" + hex.EncodeToString(hash) }
|
||||
@@ -0,0 +1,479 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
reconcilePageSize = 500
|
||||
reconcileOperationTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type Finding struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
Action string `json:"action"`
|
||||
Applied bool `json:"applied"`
|
||||
}
|
||||
|
||||
// SQLQueries counts SQL statements, including transaction control and locks.
|
||||
// S3Calls counts calls to the Blobs port, including each list page and deep GET.
|
||||
type Stats struct {
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
SQLQueries int64 `json:"sql_queries"`
|
||||
S3Calls int64 `json:"s3_calls"`
|
||||
ScannedFiles int64 `json:"scanned_files"`
|
||||
ScannedObjects int64 `json:"scanned_objects"`
|
||||
Findings int64 `json:"findings"`
|
||||
Repaired int64 `json:"repaired"`
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
db *pgxpool.Pool
|
||||
blobs Blobs
|
||||
grace time.Duration
|
||||
}
|
||||
|
||||
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
|
||||
if db == nil || blobs == nil {
|
||||
return nil, errors.New("reconcile: database and blobs are required")
|
||||
}
|
||||
if grace == 0 {
|
||||
grace = time.Hour
|
||||
}
|
||||
if grace < 2*time.Minute {
|
||||
return nil, errors.New("reconcile: grace must be at least two minutes")
|
||||
}
|
||||
return &Reconciler{db: db, blobs: blobs, grace: grace}, nil
|
||||
}
|
||||
|
||||
// Run streams findings and keeps at most one page of metadata in memory.
|
||||
// Dry runs only read storage. Each apply candidate is checked again under the
|
||||
// publisher's transaction lock; no scan snapshot authorizes a later mutation.
|
||||
func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) {
|
||||
started := time.Now()
|
||||
defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }()
|
||||
run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)}
|
||||
for _, prefix := range []string{"staging/", "blobs/"} {
|
||||
if err = run.objects(prefix); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
}
|
||||
err = run.files()
|
||||
return stats, err
|
||||
}
|
||||
|
||||
type reconcileRun struct {
|
||||
r *Reconciler
|
||||
ctx context.Context
|
||||
apply, deep bool
|
||||
emit func(Finding)
|
||||
stats *Stats
|
||||
cutoff time.Time
|
||||
}
|
||||
|
||||
func (s *reconcileRun) note(f *Finding) {
|
||||
if f == nil {
|
||||
return
|
||||
}
|
||||
s.stats.Findings++
|
||||
if f.Applied {
|
||||
s.stats.Repaired++
|
||||
}
|
||||
if s.emit != nil {
|
||||
s.emit(*f)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reconcileRun) old(modified time.Time) bool {
|
||||
// Unknown timestamps must never authorize collection.
|
||||
return !modified.IsZero() && modified.Before(s.cutoff)
|
||||
}
|
||||
|
||||
func (s *reconcileRun) objects(prefix string) error {
|
||||
after := ""
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.stats.S3Calls++
|
||||
opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout)
|
||||
page, err := s.r.blobs.List(opCtx, prefix, after, reconcilePageSize)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile list %s: %w", prefix, err)
|
||||
}
|
||||
if len(page) == 0 {
|
||||
return nil
|
||||
}
|
||||
var known map[string]bool
|
||||
if prefix == "blobs/" {
|
||||
known, err = s.knownFiles(page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, obj := range page {
|
||||
if obj.Key <= after {
|
||||
return errors.New("reconcile: object page did not advance")
|
||||
}
|
||||
after = obj.Key
|
||||
s.stats.ScannedObjects++
|
||||
if !s.old(obj.Modified) {
|
||||
continue
|
||||
}
|
||||
var finding *Finding
|
||||
if prefix == "staging/" {
|
||||
finding, err = s.staging(obj)
|
||||
} else {
|
||||
finding, err = s.orphan(obj, known[obj.Key])
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile %s: %w", obj.Key, err)
|
||||
}
|
||||
s.note(finding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reconcileRun) stat(key string) (Object, error) {
|
||||
s.stats.S3Calls++
|
||||
opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout)
|
||||
defer cancel()
|
||||
return s.r.blobs.Stat(opCtx, key)
|
||||
}
|
||||
|
||||
func (s *reconcileRun) delete(key string) error {
|
||||
s.stats.S3Calls++
|
||||
opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout)
|
||||
defer cancel()
|
||||
err := s.r.blobs.Delete(opCtx, key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *reconcileRun) staging(obj Object) (*Finding, error) {
|
||||
f := &Finding{Kind: "stale_staging", Key: obj.Key, Action: "delete_object"}
|
||||
if !s.apply {
|
||||
return f, nil
|
||||
}
|
||||
current, err := s.stat(obj.Key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.old(current.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
if err = s.delete(obj.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.Applied = true
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
|
||||
hashes := make([][]byte, 0, len(page))
|
||||
for _, obj := range page {
|
||||
if !s.old(obj.Modified) {
|
||||
continue
|
||||
}
|
||||
hash, err := hex.DecodeString(obj.Key[len("blobs/"):])
|
||||
if err == nil && len(hash) == sha256.Size && BlobKey(hash) == obj.Key {
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
}
|
||||
known := make(map[string]bool, len(hashes))
|
||||
if len(hashes) == 0 {
|
||||
return known, nil
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var hash []byte
|
||||
if err := rows.Scan(&hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
known[BlobKey(hash)] = true
|
||||
}
|
||||
return known, rows.Err()
|
||||
}
|
||||
|
||||
func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
|
||||
hash, err := hex.DecodeString(obj.Key[len("blobs/"):])
|
||||
if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key {
|
||||
return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil
|
||||
}
|
||||
if known {
|
||||
return nil, nil
|
||||
}
|
||||
f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"}
|
||||
if !s.apply {
|
||||
return f, nil
|
||||
}
|
||||
return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
op := *s
|
||||
op.ctx = opCtx
|
||||
var exists bool
|
||||
s.stats.SQLQueries++
|
||||
if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
return nil, nil
|
||||
}
|
||||
current, err := op.stat(obj.Key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.old(current.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
if err = op.delete(obj.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.Applied = true
|
||||
return f, nil
|
||||
})
|
||||
}
|
||||
|
||||
// locked also covers hashes without a files row: a publisher and an orphan
|
||||
// collector therefore cannot pass each other's existence check concurrently.
|
||||
func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Finding, error)) (*Finding, error) {
|
||||
opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout)
|
||||
defer cancel()
|
||||
s.stats.SQLQueries++
|
||||
tx, err := s.r.db.Begin(opCtx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s.stats.SQLQueries++
|
||||
_ = tx.Rollback(cleanup)
|
||||
}
|
||||
}()
|
||||
// Pool defaults cover ordinary requests; object I/O can outlive those
|
||||
// idle limits. The operation deadline bounds the entire locked candidate.
|
||||
s.stats.SQLQueries++
|
||||
if _, err = tx.Exec(opCtx, `SET LOCAL idle_in_transaction_session_timeout='35s'`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
if err = LockHash(opCtx, tx, hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := fn(opCtx, tx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
err = tx.Commit(opCtx)
|
||||
committed = true // pgx closes the transaction even when COMMIT fails.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
type reconcileFile struct {
|
||||
hash []byte
|
||||
size int64
|
||||
status string
|
||||
created time.Time
|
||||
referenced bool
|
||||
}
|
||||
|
||||
func (s *reconcileRun) files() error {
|
||||
after := []byte{}
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
|
||||
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
|
||||
FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
page := make([]reconcileFile, 0, reconcilePageSize)
|
||||
for rows.Next() {
|
||||
var f reconcileFile
|
||||
if err = rows.Scan(&f.hash, &f.size, &f.status, &f.created, &f.referenced); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
page = append(page, f)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(page) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Close the page's SQL rows before acquiring a transaction connection.
|
||||
for _, f := range page {
|
||||
after = f.hash
|
||||
s.stats.ScannedFiles++
|
||||
if !s.candidate(f) {
|
||||
continue
|
||||
}
|
||||
var finding *Finding
|
||||
if s.apply {
|
||||
finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
op := *s
|
||||
op.ctx = opCtx
|
||||
var current reconcileFile
|
||||
s.stats.SQLQueries++
|
||||
err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
|
||||
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
|
||||
FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash).
|
||||
Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.candidate(current) {
|
||||
return nil, nil
|
||||
}
|
||||
return op.inspect(current, tx)
|
||||
})
|
||||
} else {
|
||||
finding, err = s.inspect(f, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err)
|
||||
}
|
||||
s.note(finding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reconcileRun) candidate(f reconcileFile) bool {
|
||||
if !s.old(f.created) {
|
||||
return false
|
||||
}
|
||||
if !f.referenced {
|
||||
return f.status == "pending" || f.status == "ready"
|
||||
}
|
||||
return f.status == "ready" || f.status == "failed"
|
||||
}
|
||||
|
||||
func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
|
||||
key := BlobKey(f.hash)
|
||||
if !f.referenced {
|
||||
// A publication that rolled back can leave fresh bytes behind an old
|
||||
// pending row. Both the metadata and the current object must be old.
|
||||
obj, err := s.stat(key)
|
||||
if err != nil && !errors.Is(err, ErrObjectMissing) {
|
||||
return nil, err
|
||||
}
|
||||
if err == nil && !s.old(obj.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"}
|
||||
if !s.apply {
|
||||
return finding, nil
|
||||
}
|
||||
// S3 must succeed first. A failed SQL commit then leaves a missing,
|
||||
// unreferenced row that the next run can safely finish collecting.
|
||||
if err := s.delete(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
return finding, nil
|
||||
}
|
||||
obj, err := s.stat(key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
if f.status == "failed" {
|
||||
return nil, nil
|
||||
}
|
||||
finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}
|
||||
if s.apply {
|
||||
s.stats.SQLQueries++
|
||||
if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
}
|
||||
return finding, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if obj.Size != f.size {
|
||||
return &Finding{Kind: "size_mismatch", Key: key, Action: "report"}, nil
|
||||
}
|
||||
if s.deep {
|
||||
match, err := s.verify(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !match {
|
||||
return &Finding{Kind: "hash_mismatch", Key: key, Action: "report"}, nil
|
||||
}
|
||||
}
|
||||
// A failed file stays failed even when the key reappears. Recovery requires
|
||||
// an explicit publisher; a shallow HEAD cannot prove content integrity.
|
||||
if f.status == "failed" {
|
||||
return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *reconcileRun) verify(f reconcileFile) (bool, error) {
|
||||
if f.size < 0 || f.size == math.MaxInt64 {
|
||||
return false, errors.New("reconcile: invalid file size")
|
||||
}
|
||||
s.stats.S3Calls++
|
||||
opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout)
|
||||
defer cancel()
|
||||
body, err := s.r.blobs.Open(opCtx, BlobKey(f.hash))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
hash := sha256.New()
|
||||
n, readErr := io.Copy(hash, io.LimitReader(body, f.size+1))
|
||||
closeErr := body.Close()
|
||||
if readErr != nil {
|
||||
return false, readErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return false, closeErr
|
||||
}
|
||||
return n == f.size && bytes.Equal(hash.Sum(nil), f.hash), nil
|
||||
}
|
||||
@@ -0,0 +1,708 @@
|
||||
package attachment_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
func TestReconcileBlobKey(t *testing.T) {
|
||||
hash, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||
if got := attachment.BlobKey(hash); got != "blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
|
||||
t.Fatalf("blob key: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only object storage is doubled: SQL, transactions, FKs and advisory locks are real.
|
||||
type reconcileBlobs struct {
|
||||
mu sync.Mutex
|
||||
objects map[string]attachment.Object
|
||||
bodies map[string][]byte
|
||||
statErr, deleteErr, openErr error
|
||||
listed func(string)
|
||||
stating func(string)
|
||||
opening func(string) io.ReadCloser
|
||||
writes, readBytes int
|
||||
limits []int
|
||||
}
|
||||
|
||||
func newReconcileBlobs() *reconcileBlobs {
|
||||
return &reconcileBlobs{objects: make(map[string]attachment.Object), bodies: make(map[string][]byte)}
|
||||
}
|
||||
func (b *reconcileBlobs) add(key string, body []byte, modified time.Time) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.objects[key] = attachment.Object{Key: key, Size: int64(len(body)), Modified: modified}
|
||||
b.bodies[key] = bytes.Clone(body)
|
||||
}
|
||||
func (b *reconcileBlobs) has(key string) bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
_, ok := b.objects[key]
|
||||
return ok
|
||||
}
|
||||
func (b *reconcileBlobs) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error {
|
||||
body, err := io.ReadAll(io.LimitReader(r, size+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(len(body)) != size {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b.add(key, body, time.Now())
|
||||
b.mu.Lock()
|
||||
b.writes++
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
func (b *reconcileBlobs) Copy(_ context.Context, source, dest string) error {
|
||||
b.mu.Lock()
|
||||
body, ok := b.bodies[source]
|
||||
b.mu.Unlock()
|
||||
if !ok {
|
||||
return attachment.ErrObjectMissing
|
||||
}
|
||||
b.add(dest, body, time.Now())
|
||||
b.mu.Lock()
|
||||
b.writes++
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
func (b *reconcileBlobs) Open(_ context.Context, key string) (io.ReadCloser, error) {
|
||||
if b.openErr != nil {
|
||||
return nil, b.openErr
|
||||
}
|
||||
if b.opening != nil {
|
||||
return b.opening(key), nil
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
body, ok := b.bodies[key]
|
||||
if !ok {
|
||||
return nil, attachment.ErrObjectMissing
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(body)), nil
|
||||
}
|
||||
func (b *reconcileBlobs) Stat(_ context.Context, key string) (attachment.Object, error) {
|
||||
if b.stating != nil {
|
||||
b.stating(key)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.statErr != nil {
|
||||
return attachment.Object{}, b.statErr
|
||||
}
|
||||
obj, ok := b.objects[key]
|
||||
if !ok {
|
||||
return attachment.Object{}, fmt.Errorf("head: %w", attachment.ErrObjectMissing)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
func (b *reconcileBlobs) Delete(_ context.Context, key string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.deleteErr != nil {
|
||||
return b.deleteErr
|
||||
}
|
||||
delete(b.objects, key)
|
||||
delete(b.bodies, key)
|
||||
b.writes++
|
||||
return nil
|
||||
}
|
||||
func (b *reconcileBlobs) List(_ context.Context, prefix, after string, limit int) ([]attachment.Object, error) {
|
||||
b.mu.Lock()
|
||||
b.limits = append(b.limits, limit)
|
||||
if limit < 1 || limit > 500 {
|
||||
b.mu.Unlock()
|
||||
return nil, errors.New("unbounded page")
|
||||
}
|
||||
var page []attachment.Object
|
||||
for key, obj := range b.objects {
|
||||
if strings.HasPrefix(key, prefix) && key > after {
|
||||
page = append(page, obj)
|
||||
}
|
||||
}
|
||||
sort.Slice(page, func(i, j int) bool { return page[i].Key < page[j].Key })
|
||||
if len(page) > limit {
|
||||
page = page[:limit]
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if b.listed != nil {
|
||||
b.listed(prefix)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
type reconcileFixture struct {
|
||||
t *testing.T
|
||||
db *pgxpool.Pool
|
||||
b *reconcileBlobs
|
||||
user, chat, message uuid.UUID
|
||||
}
|
||||
|
||||
func reconcileSetup(t *testing.T, db *testenv.DB) *reconcileFixture {
|
||||
t.Helper()
|
||||
db.Reset(t)
|
||||
f := &reconcileFixture{t: t, db: db.Pool, b: newReconcileBlobs(), user: uuid.New(), chat: uuid.New(), message: uuid.New()}
|
||||
f.exec(`INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,'reconcile','Reconcile','unused',now())`, f.user)
|
||||
f.exec(`INSERT INTO chats(id,title,created_at) VALUES($1,'reconcile',now())`, f.chat)
|
||||
f.exec(`INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, f.message, f.user)
|
||||
return f
|
||||
}
|
||||
func (f *reconcileFixture) exec(sql string, args ...any) {
|
||||
f.t.Helper()
|
||||
if _, err := f.db.Exec(context.Background(), sql, args...); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func (f *reconcileFixture) file(body, status string, age time.Duration, referenced, present bool) []byte {
|
||||
f.t.Helper()
|
||||
hash := sha256.Sum256([]byte(body))
|
||||
f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, time.Now().Add(-age))
|
||||
if referenced {
|
||||
id := uuid.New()
|
||||
f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'image.png',$4,now())`, id, f.chat, f.user, hash[:])
|
||||
f.exec(`INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal)
|
||||
VALUES($1,$2,$3,(SELECT count(*) FROM message_attachments WHERE chat_id=$1 AND message_id=$2))`, f.chat, f.message, id)
|
||||
}
|
||||
if present {
|
||||
f.b.add(attachment.BlobKey(hash[:]), []byte(body), time.Now().Add(-age))
|
||||
}
|
||||
return hash[:]
|
||||
}
|
||||
func (f *reconcileFixture) status(hash []byte) string {
|
||||
f.t.Helper()
|
||||
var s string
|
||||
err := f.db.QueryRow(context.Background(), `SELECT status FROM files WHERE hash=$1`, hash).Scan(&s)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment.Finding, error) {
|
||||
f.t.Helper()
|
||||
r, err := attachment.NewReconciler(f.db, f.b, 0)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
var found []attachment.Finding
|
||||
stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) })
|
||||
return stats, found, err
|
||||
}
|
||||
|
||||
func TestReconciler(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
t.Run("constructor validates and defaults grace", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
for _, grace := range []time.Duration{-time.Second, time.Minute} {
|
||||
if _, err := attachment.NewReconciler(f.db, f.b, grace); err == nil {
|
||||
t.Fatalf("accepted grace %v", grace)
|
||||
}
|
||||
}
|
||||
if _, err := attachment.NewReconciler(nil, f.b, 0); err == nil {
|
||||
t.Fatal("accepted nil DB")
|
||||
}
|
||||
if _, err := attachment.NewReconciler(f.db, nil, 0); err == nil {
|
||||
t.Fatal("accepted nil blobs")
|
||||
}
|
||||
if _, err := attachment.NewReconciler(f.db, f.b, 2*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh := f.file("half hour", "ready", 30*time.Minute, false, true)
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || f.status(fresh) != "ready" {
|
||||
t.Fatalf("default grace: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("dry run then apply then repeat preserves fresh referenced and message links", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
old := 2 * time.Hour
|
||||
pending := f.file("pending", "pending", old, false, true)
|
||||
unused := f.file("unused", "ready", old, false, true)
|
||||
used := f.file("used", "ready", old, true, true)
|
||||
fresh := f.file("fresh", "pending", time.Minute, false, true)
|
||||
missing := f.file("missing", "ready", old, true, false)
|
||||
f.b.add("staging/old", []byte("old"), time.Now().Add(-old))
|
||||
f.b.add("staging/fresh", []byte("new"), time.Now())
|
||||
orphan := sha256.Sum256([]byte("orphan"))
|
||||
orphanKey := attachment.BlobKey(orphan[:])
|
||||
f.b.add(orphanKey, []byte("orphan"), time.Now().Add(-old))
|
||||
stats, findings, err := f.run(false, false)
|
||||
if err != nil || stats.Repaired != 0 || stats.Findings != 5 || len(findings) != 5 {
|
||||
t.Fatalf("dry: %+v %v %+v", stats, err, findings)
|
||||
}
|
||||
if f.b.writes != 0 || f.status(pending) != "pending" || f.status(missing) != "ready" {
|
||||
t.Fatal("dry run mutated storage")
|
||||
}
|
||||
for _, finding := range findings {
|
||||
if finding.Applied {
|
||||
t.Fatal("dry finding marked applied")
|
||||
}
|
||||
}
|
||||
stats, findings, err = f.run(true, false)
|
||||
if err != nil || stats.Repaired != 5 {
|
||||
t.Fatalf("apply: %+v %v %+v", stats, err, findings)
|
||||
}
|
||||
if f.status(pending) != "" || f.status(unused) != "" || f.status(missing) != "failed" {
|
||||
t.Fatal("metadata repair incorrect")
|
||||
}
|
||||
if f.b.has(orphanKey) || f.b.has("staging/old") || !f.b.has("staging/fresh") || !f.b.has(attachment.BlobKey(used)) || !f.b.has(attachment.BlobKey(fresh)) {
|
||||
t.Fatal("incorrect object collection")
|
||||
}
|
||||
var links int
|
||||
if err := f.db.QueryRow(ctx, `SELECT count(*) FROM message_attachments`).Scan(&links); err != nil || links != 2 {
|
||||
t.Fatalf("message links lost: %d %v", links, err)
|
||||
}
|
||||
stats, _, err = f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 {
|
||||
t.Fatalf("repeat: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("outage is not missing and failed delete keeps row", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
used := f.file("used", "ready", 2*time.Hour, true, true)
|
||||
outage := errors.New("injected S3 outage")
|
||||
f.b.statErr = outage
|
||||
stats, _, err := f.run(true, false)
|
||||
if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(used) != "ready" {
|
||||
t.Fatalf("outage: %+v %v", stats, err)
|
||||
}
|
||||
f.b.statErr = nil
|
||||
unused := f.file("unused", "pending", 2*time.Hour, false, true)
|
||||
f.b.deleteErr = outage
|
||||
_, _, err = f.run(true, false)
|
||||
if !errors.Is(err, outage) || f.status(unused) != "pending" {
|
||||
t.Fatalf("delete failure: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("deep reports corruption and never revives failed without verification", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("good", "ready", 2*time.Hour, true, true)
|
||||
f.b.add(attachment.BlobKey(hash), []byte("evil"), time.Now().Add(-2*time.Hour))
|
||||
stats, found, err := f.run(true, true)
|
||||
if err != nil || stats.Findings != 1 || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) {
|
||||
t.Fatalf("deep: %+v %v %+v", stats, err, found)
|
||||
}
|
||||
f.exec(`UPDATE files SET status='failed' WHERE hash=$1`, hash)
|
||||
stats, _, err = f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" {
|
||||
t.Fatal("shallow revived failed file")
|
||||
}
|
||||
stats, _, err = f.run(true, true)
|
||||
if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" {
|
||||
t.Fatal("corrupt file revived")
|
||||
}
|
||||
})
|
||||
t.Run("wrong size reports only and fresh missing stays ready", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("sized", "ready", 2*time.Hour, true, true)
|
||||
f.b.add(attachment.BlobKey(hash), []byte("longer body"), time.Now().Add(-2*time.Hour))
|
||||
fresh := f.file("new missing", "ready", time.Minute, true, false)
|
||||
stats, found, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || stats.Findings != 1 || found[0].Kind != "size_mismatch" || f.status(fresh) != "ready" {
|
||||
t.Fatalf("size/fresh: %+v %v %+v", stats, err, found)
|
||||
}
|
||||
})
|
||||
t.Run("orphan rechecks current object age", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := sha256.Sum256([]byte("orphan"))
|
||||
key := attachment.BlobKey(hash[:])
|
||||
f.b.add(key, []byte("orphan"), time.Now().Add(-2*time.Hour))
|
||||
f.b.listed = func(prefix string) {
|
||||
if prefix == "blobs/" {
|
||||
f.b.add(key, []byte("republished"), time.Now())
|
||||
}
|
||||
}
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || !f.b.has(key) {
|
||||
t.Fatalf("fresh replacement deleted: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("row rechecks references after taking lock", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("race refs", "ready", 2*time.Hour, false, true)
|
||||
tx, err := f.db.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if err = attachment.LockHash(ctx, tx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
r, _ := attachment.NewReconciler(f.db, f.b, 0)
|
||||
_, err := r.Run(ctx, true, false, nil)
|
||||
done <- err
|
||||
}()
|
||||
waitReconcileLock(t, ctx, f.db)
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'race.png',$4,now())`, uuid.New(), f.chat, f.user, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) {
|
||||
t.Fatal("new reference collected")
|
||||
}
|
||||
})
|
||||
t.Run("concurrent upload lock prevents orphan collection", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := sha256.Sum256([]byte("publishing"))
|
||||
key := attachment.BlobKey(hash[:])
|
||||
f.b.add(key, []byte("publishing"), time.Now().Add(-2*time.Hour))
|
||||
tx, err := f.db.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if err = attachment.LockHash(ctx, tx, hash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
r, _ := attachment.NewReconciler(f.db, f.b, 0)
|
||||
_, err := r.Run(ctx, true, false, nil)
|
||||
done <- err
|
||||
}()
|
||||
waitReconcileLock(t, ctx, f.db)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,10,'image/png','ready')`, hash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !f.b.has(key) || f.status(hash[:]) != "ready" {
|
||||
t.Fatal("published blob collected")
|
||||
}
|
||||
})
|
||||
t.Run("signed big endian lock key matches publisher", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := make([]byte, 32)
|
||||
for i := 0; i < 8; i++ {
|
||||
hash[i] = 255
|
||||
}
|
||||
hash[7] = 254
|
||||
tx, err := f.db.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if err = attachment.LockHash(ctx, tx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := f.db.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer other.Rollback(ctx)
|
||||
var acquired bool
|
||||
if err = other.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(-2::bigint)`).Scan(&acquired); err != nil || acquired {
|
||||
t.Fatalf("lock mismatch: %v %v", acquired, err)
|
||||
}
|
||||
})
|
||||
t.Run("keyset pages tolerate deletes and report costs", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
for i := 0; i < 1003; i++ {
|
||||
f.b.add(fmt.Sprintf("staging/%04d", i), []byte("x"), time.Now().Add(-2*time.Hour))
|
||||
}
|
||||
f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','pending',now()-interval '2 hour' FROM generate_series(1,1003) AS i`)
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.ScannedFiles != 1003 || stats.ScannedObjects != 1003 || stats.Repaired != 2006 {
|
||||
t.Fatalf("pages: %+v %v", stats, err)
|
||||
}
|
||||
var n int
|
||||
if err := f.db.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("skipped rows: %d %v", n, err)
|
||||
}
|
||||
if len(f.b.limits) < 4 || stats.SQLQueries == 0 || stats.S3Calls == 0 {
|
||||
t.Fatalf("missing paging/costs: %+v", stats)
|
||||
}
|
||||
raw, err := json.Marshal(stats)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"elapsed_ms", "sql_queries", "s3_calls", "scanned_files", "scanned_objects", "findings", "repaired"} {
|
||||
if !bytes.Contains(raw, []byte(`"`+key+`":`)) {
|
||||
t.Fatalf("missing JSON key %s: %s", key, raw)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("old blob existence checks are batched by page", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','ready',now()-interval '2 hour' FROM generate_series(1,1003) AS i`)
|
||||
f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),$1,$2,'batch.png',hash,now() FROM files`, f.chat, f.user)
|
||||
for i := 1; i <= 1003; i++ {
|
||||
f.b.add(fmt.Sprintf("blobs/%064x", i), []byte("x"), time.Now().Add(-2*time.Hour))
|
||||
}
|
||||
stats, _, err := f.run(false, false)
|
||||
if err != nil || stats.SQLQueries > 8 || stats.SQLQueries == 0 || stats.ScannedFiles != 1003 || stats.Findings != 0 {
|
||||
t.Fatalf("per-object SQL lookup: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
deadline := time.NewTimer(5 * time.Second)
|
||||
defer deadline.Stop()
|
||||
tick := time.NewTicker(10 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
var waiting bool
|
||||
if err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted)`).Scan(&waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if waiting {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline.C:
|
||||
t.Fatal("reconciler did not wait on publisher advisory lock")
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type reconcileEndless struct {
|
||||
read int64
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *reconcileEndless) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
r.read += int64(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
func (r *reconcileEndless) Close() error { r.closed = true; return nil }
|
||||
|
||||
func TestReconcilerFailureBoundaries(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
t.Run("slow object call survives pool idle transaction timeout", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("slow object", "pending", 2*time.Hour, false, true)
|
||||
cfg := f.db.Config()
|
||||
cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "100ms"
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
f.db = pool
|
||||
f.b.stating = func(string) { time.Sleep(250 * time.Millisecond) }
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 1 || f.status(hash) != "" {
|
||||
t.Fatalf("slow S3 transaction rolled back: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("fresh object with old unused metadata is preserved", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("republished", "pending", 2*time.Hour, false, true)
|
||||
f.b.add(attachment.BlobKey(hash), []byte("republished"), time.Now())
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || f.status(hash) != "pending" || !f.b.has(attachment.BlobKey(hash)) {
|
||||
t.Fatalf("fresh blob lost: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("deep read stops at size plus one and closes body", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("good", "ready", 2*time.Hour, true, true)
|
||||
body := &reconcileEndless{}
|
||||
f.b.opening = func(string) io.ReadCloser { return body }
|
||||
stats, found, err := f.run(true, true)
|
||||
if err != nil || body.read != 5 || !body.closed || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" {
|
||||
t.Fatalf("unbounded deep read: read=%d closed=%v %+v %v", body.read, body.closed, stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("deep GET outage preserves referenced state", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("good", "ready", 2*time.Hour, true, true)
|
||||
outage := errors.New("injected GET outage")
|
||||
f.b.openErr = outage
|
||||
stats, _, err := f.run(true, true)
|
||||
if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(hash) != "ready" {
|
||||
t.Fatalf("GET outage: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("valid failed object remains stable even deep", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("good", "failed", 2*time.Hour, true, true)
|
||||
for _, deep := range []bool{false, true} {
|
||||
stats, found, err := f.run(true, deep)
|
||||
if err != nil || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "failed_present" || f.status(hash) != "failed" {
|
||||
t.Fatalf("failed state unstable: %+v %v %+v", stats, err, found)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("dry run works with a read only PostgreSQL session", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("missing", "ready", 2*time.Hour, true, false)
|
||||
cfg := f.db.Config()
|
||||
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
readOnly, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer readOnly.Close()
|
||||
r, err := attachment.NewReconciler(readOnly, f.b, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats, err := r.Run(ctx, false, true, nil)
|
||||
if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" {
|
||||
t.Fatalf("read only dry run: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("failed SQL commit after blob deletion is retryable", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("unused", "pending", 2*time.Hour, false, true)
|
||||
f.exec(`CREATE FUNCTION reconcile_reject_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected commit failure'; END $$`)
|
||||
f.exec(`CREATE CONSTRAINT TRIGGER reconcile_reject_delete AFTER DELETE ON files DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reconcile_reject_delete()`)
|
||||
t.Cleanup(func() {
|
||||
f.exec(`DROP TRIGGER IF EXISTS reconcile_reject_delete ON files`)
|
||||
f.exec(`DROP FUNCTION IF EXISTS reconcile_reject_delete()`)
|
||||
})
|
||||
stats, _, err := f.run(true, false)
|
||||
if err == nil || stats.Repaired != 0 || f.status(hash) != "pending" || f.b.has(attachment.BlobKey(hash)) {
|
||||
t.Fatalf("failed commit: %+v %v", stats, err)
|
||||
}
|
||||
f.exec(`DROP TRIGGER reconcile_reject_delete ON files`)
|
||||
stats, _, err = f.run(true, false)
|
||||
if err != nil || stats.Repaired != 1 || f.status(hash) != "" {
|
||||
t.Fatalf("retry: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcilerS3(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
s3, err := testenv.StartS3(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s3.Close()
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("good", "ready", 2*time.Hour, true, false)
|
||||
missing := f.file("missing", "ready", 2*time.Hour, true, false)
|
||||
key := attachment.BlobKey(hash)
|
||||
if err = s3.Store.Put(ctx, key, strings.NewReader("evil"), 4, "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = s3.Store.Put(ctx, "staging/fresh", strings.NewReader("x"), 1, "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := attachment.NewReconciler(f.db, s3.Store, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Exercise the command with file-based configuration and the real listing
|
||||
// identity. Its default must remain read-only, including missing metadata.
|
||||
secretDir := t.TempDir()
|
||||
cliEnv := os.Environ()
|
||||
for name, value := range map[string]string{
|
||||
"DATABASE_URL_FILE": db.DSN,
|
||||
"S3_ACCESS_KEY_ID_FILE": s3.AccessKey,
|
||||
"S3_SECRET_ACCESS_KEY_FILE": s3.SecretKey,
|
||||
} {
|
||||
path := filepath.Join(secretDir, name)
|
||||
if err := os.WriteFile(path, []byte(value), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cliEnv = append(cliEnv, name+"="+path)
|
||||
}
|
||||
cliEnv = append(cliEnv, "S3_ENDPOINT="+s3.Endpoint, "S3_REGION="+s3.Region, "S3_BUCKET="+s3.Bucket)
|
||||
cmd := exec.CommandContext(ctx, "go", "run", "../../cmd/reconcile", "--deep")
|
||||
cmd.Env = cliEnv
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatalf("CLI dry run: %v: %s", err, stderr.String())
|
||||
}
|
||||
decoder := json.NewDecoder(&stdout)
|
||||
var cliStats attachment.Stats
|
||||
for i := 0; i < 2; i++ {
|
||||
var finding attachment.Finding
|
||||
if err := decoder.Decode(&finding); err != nil || finding.Applied || finding.Kind == "" || finding.Key == "" || finding.Action == "" {
|
||||
t.Fatalf("CLI finding: %+v %v", finding, err)
|
||||
}
|
||||
}
|
||||
if err := decoder.Decode(&cliStats); err != nil || cliStats.Findings != 2 || cliStats.Repaired != 0 || f.status(missing) != "ready" {
|
||||
t.Fatalf("CLI mutated by default: %+v %v", cliStats, err)
|
||||
}
|
||||
var found []attachment.Finding
|
||||
stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) })
|
||||
if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" {
|
||||
t.Fatalf("S3 dry: %+v %v %+v", stats, err, found)
|
||||
}
|
||||
stats, err = r.Run(ctx, true, true, nil)
|
||||
if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" {
|
||||
t.Fatalf("S3 apply: %+v %v", stats, err)
|
||||
}
|
||||
if _, err = s3.Store.Stat(ctx, key); err != nil {
|
||||
t.Fatal("referenced corrupt object deleted", err)
|
||||
}
|
||||
if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil {
|
||||
t.Fatal("fresh staging deleted", err)
|
||||
}
|
||||
stats, err = r.Run(ctx, true, true, nil)
|
||||
if err != nil || stats.Repaired != 0 {
|
||||
t.Fatalf("S3 repeat: %+v %v", stats, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileLockRejectsMalformedHash(t *testing.T) {
|
||||
if err := attachment.LockHash(context.Background(), nil, []byte("bad")); err == nil {
|
||||
t.Fatal("malformed hash accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package attachment_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"messenger/internal/testenv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFilesSchemaRejectsMalformedHashes(t *testing.T) {
|
||||
db, err := testenv.StartPostgres(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
_, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`)
|
||||
if err != nil {
|
||||
t.Fatalf("valid content-addressed file: %v", err)
|
||||
}
|
||||
_, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ('bad',12,'image/png','ready')`)
|
||||
if err == nil {
|
||||
t.Fatal("accepted a malformed SHA256")
|
||||
}
|
||||
_, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`)
|
||||
if err == nil {
|
||||
t.Fatal("accepted duplicate content hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
repo Repository
|
||||
blobs Blobs
|
||||
maxBytes, maxPixels int64
|
||||
decode chan struct{}
|
||||
}
|
||||
|
||||
func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service {
|
||||
if maxBytes == 0 {
|
||||
maxBytes = MaxUploadBytes
|
||||
}
|
||||
if maxPixels == 0 {
|
||||
maxPixels = MaxImagePixels
|
||||
}
|
||||
return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) {
|
||||
if err := ValidateFilename(filename); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if mime != "image/png" && mime != "image/jpeg" {
|
||||
return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted")
|
||||
}
|
||||
ok, err := s.repo.IsMember(ctx, chatID, actor)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if !ok {
|
||||
return Attachment{}, apperr.NotFound("chat not found")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if size <= 0 {
|
||||
return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required")
|
||||
}
|
||||
if s.blobs == nil {
|
||||
return Attachment{}, apperr.Unavailable("object storage is unavailable")
|
||||
}
|
||||
id := ident.New()
|
||||
stage := "staging/" + id.String()
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := s.blobs.Delete(cleanup, stage); err != nil {
|
||||
slog.Warn("staging cleanup deferred", "error", err)
|
||||
}
|
||||
}()
|
||||
hash := sha256.New()
|
||||
counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)}
|
||||
err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime)
|
||||
if err != nil {
|
||||
_, tooLarge := errors.AsType[*http.MaxBytesError](err)
|
||||
if counter.n > s.maxBytes || tooLarge {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
return Attachment{}, apperr.Invalid("incomplete upload")
|
||||
}
|
||||
return Attachment{}, unavailable(err)
|
||||
}
|
||||
var extra [1]byte
|
||||
n, readErr := counter.Read(extra[:])
|
||||
if counter.n > s.maxBytes {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) {
|
||||
return Attachment{}, apperr.Invalid("body length does not match Content-Length")
|
||||
}
|
||||
// Full decoding is bounded by dimensions and one decoder per replica. Body
|
||||
// transfer itself stays streaming and does not occupy the decoder slot.
|
||||
select {
|
||||
case s.decode <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return Attachment{}, unavailable(ctx.Err())
|
||||
}
|
||||
err = s.validate(ctx, stage, mime)
|
||||
<-s.decode
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)}
|
||||
if err := s.repo.Publish(ctx, a, stage); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *Service) validate(ctx context.Context, stage, mime string) error {
|
||||
r, err := s.blobs.Open(ctx, stage)
|
||||
if err != nil {
|
||||
return unavailable(err)
|
||||
}
|
||||
defer r.Close()
|
||||
tracked := &storageReader{reader: r}
|
||||
err = ValidateImage(tracked, mime, s.maxPixels)
|
||||
if tracked.err != nil {
|
||||
return unavailable(tracked.err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return unavailable(ctx.Err())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Image sniffing can replace a reader failure with ErrFormat. Preserve the
|
||||
// storage failure independently, while ordinary EOF still means malformed bytes.
|
||||
type storageReader struct {
|
||||
reader io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *storageReader) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
r.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
|
||||
a, err := s.repo.GetForMember(ctx, id, actor)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if a.Status != "ready" {
|
||||
return Attachment{}, apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *Service) Content(ctx context.Context, actor, id uuid.UUID) (Attachment, io.ReadCloser, error) {
|
||||
a, err := s.Get(ctx, actor, id)
|
||||
if err != nil {
|
||||
return Attachment{}, nil, err
|
||||
}
|
||||
if s.blobs == nil {
|
||||
return Attachment{}, nil, apperr.Unavailable("object storage is unavailable")
|
||||
}
|
||||
r, err := s.blobs.Open(ctx, BlobKey(a.FileHash))
|
||||
if err != nil {
|
||||
return Attachment{}, nil, unavailable(err)
|
||||
}
|
||||
return a, r, nil
|
||||
}
|
||||
|
||||
type countReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e }
|
||||
func unavailable(err error) error {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
package attachment_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/attachment/store"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
type testBlobs struct {
|
||||
mu sync.Mutex
|
||||
data map[string][]byte
|
||||
copies int
|
||||
failCopy bool
|
||||
}
|
||||
|
||||
type blockingPublication struct {
|
||||
*testBlobs
|
||||
entered, resume chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockingPublication) Copy(ctx context.Context, from, to string) error {
|
||||
close(b.entered)
|
||||
select {
|
||||
case <-b.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
return b.testBlobs.Copy(ctx, from, to)
|
||||
}
|
||||
func (b *blockingPublication) Stat(ctx context.Context, key string) (attachment.Object, error) {
|
||||
o, e := b.testBlobs.Stat(ctx, key)
|
||||
o.Modified = time.Now().Add(-2 * time.Hour)
|
||||
return o, e
|
||||
}
|
||||
|
||||
func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
actor, room := ident.New(), ident.New()
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'race_user','User','hash',now())`, actor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'race',now())`, room); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, room, actor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := sha256.Sum256(data)
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png','pending',now()-interval '2 hours')`, hash[:], len(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0)
|
||||
uploaded := make(chan error, 1)
|
||||
go func() {
|
||||
_, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data))
|
||||
uploaded <- e
|
||||
}()
|
||||
select {
|
||||
case <-b.entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("publisher never reached copy")
|
||||
}
|
||||
r, err := attachment.NewReconciler(db.Pool, b, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collected := make(chan error, 1)
|
||||
go func() {
|
||||
stats, e := r.Run(ctx, true, false, nil)
|
||||
if e == nil && stats.Repaired != 0 {
|
||||
e = errors.New("collector removed a newly referenced file")
|
||||
}
|
||||
collected <- e
|
||||
}()
|
||||
// Observe a real waiting advisory lock, rather than guessing which goroutine won.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
waiting := false
|
||||
for time.Now().Before(deadline) {
|
||||
var n int
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n > 0 {
|
||||
waiting = true
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
close(b.resume)
|
||||
if !waiting {
|
||||
t.Fatal("collector did not serialize with the publisher")
|
||||
}
|
||||
if err := <-uploaded; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-collected; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.Stat(ctx, attachment.BlobKey(hash[:])); err != nil {
|
||||
t.Fatal("collector deleted published data", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *testBlobs) Put(_ context.Context, k string, r io.Reader, n int64, m string) error {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.data[k] = data
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) Open(_ context.Context, k string) (io.ReadCloser, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
data, ok := b.data[k]
|
||||
if !ok {
|
||||
return nil, attachment.ErrObjectMissing
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
func (b *testBlobs) Stat(_ context.Context, k string) (attachment.Object, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
data, ok := b.data[k]
|
||||
if !ok {
|
||||
return attachment.Object{}, attachment.ErrObjectMissing
|
||||
}
|
||||
return attachment.Object{Key: k, Size: int64(len(data)), Modified: time.Now()}, nil
|
||||
}
|
||||
func (b *testBlobs) Copy(_ context.Context, from, to string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.failCopy {
|
||||
return errors.New("copy outage")
|
||||
}
|
||||
b.copies++
|
||||
b.data[to] = bytes.Clone(b.data[from])
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) Delete(_ context.Context, k string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.data, k)
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) List(context.Context, string, string, int) ([]attachment.Object, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
actor, c1, c2 := ident.New(), ident.New(), ident.New()
|
||||
_, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'alice','Alice','hash',now());`, actor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []uuid.UUID{c1, c2} {
|
||||
_, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'room',now())`, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, c, actor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b := &testBlobs{data: map[string][]byte{}}
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels)
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var results [10]attachment.Attachment
|
||||
var wg sync.WaitGroup
|
||||
for i := range results {
|
||||
wg.Go(func() {
|
||||
c := c1
|
||||
if i%2 == 1 {
|
||||
c = c2
|
||||
}
|
||||
var e error
|
||||
results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data))
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
var files, atts int
|
||||
err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments)`).Scan(&files, &atts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if files != 1 || atts != 10 || b.copies != 1 || len(b.data) != 1 {
|
||||
t.Fatalf("files=%d attachments=%d copies=%d objects=%d", files, atts, b.copies, len(b.data))
|
||||
}
|
||||
if results[0].ID == results[1].ID {
|
||||
t.Fatal("deduplication merged chat ACL identity")
|
||||
}
|
||||
if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound {
|
||||
t.Fatalf("outsider got %v", err)
|
||||
}
|
||||
// A removed blob is republished from the new upload, preserving old references.
|
||||
b.mu.Lock()
|
||||
clear(b.data)
|
||||
b.mu.Unlock()
|
||||
if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.copies != 2 {
|
||||
t.Fatalf("missing ready blob was not restored: copies=%d", b.copies)
|
||||
}
|
||||
b.failCopy = true
|
||||
other := bytes.Clone(data)
|
||||
other = append(other, 0)
|
||||
if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("copy outage: %v", err)
|
||||
}
|
||||
if len(b.data) != 1 {
|
||||
t.Fatalf("failed publication left staging: %d objects", len(b.data))
|
||||
}
|
||||
b.failCopy = false
|
||||
// S3 publication succeeds, then PostgreSQL refuses the logical attachment.
|
||||
// Compensation may remove staging but must leave shared CAS data intact.
|
||||
_, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$;
|
||||
CREATE TRIGGER reject_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_attachment()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil {
|
||||
t.Fatal("publication survived rejected metadata")
|
||||
}
|
||||
if len(b.data) != 2 {
|
||||
t.Fatalf("expected original and recoverable orphan, got %d objects", len(b.data))
|
||||
}
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&files); err != nil || files != 1 {
|
||||
t.Fatalf("failed DB insert committed file metadata: %d %v", files, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
)
|
||||
|
||||
type Attachments struct {
|
||||
pool *pgxpool.Pool
|
||||
blobs attachment.Blobs
|
||||
}
|
||||
|
||||
func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments {
|
||||
return &Attachments{pool, blobs}
|
||||
}
|
||||
|
||||
func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) {
|
||||
var ok bool
|
||||
err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok)
|
||||
return ok, postgres.Classify(err)
|
||||
}
|
||||
|
||||
// Publication and garbage collection use the same hash lock. The transaction
|
||||
// spans only publication, never client transfer or image decoding.
|
||||
func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error {
|
||||
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
|
||||
return err
|
||||
}
|
||||
var status string
|
||||
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
copyNeeded := status != "ready"
|
||||
if !copyNeeded {
|
||||
obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash))
|
||||
if errors.Is(e, attachment.ErrObjectMissing) {
|
||||
copyNeeded = true
|
||||
} else if e != nil {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e)
|
||||
} else if obj.Size != a.SizeBytes {
|
||||
copyNeeded = true
|
||||
}
|
||||
}
|
||||
if copyNeeded {
|
||||
if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
|
||||
}
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish file: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
|
||||
return err
|
||||
}))
|
||||
}
|
||||
|
||||
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
|
||||
var a attachment.Attachment
|
||||
err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at
|
||||
FROM attachments a JOIN files f ON f.hash=a.file_hash
|
||||
WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return attachment.Attachment{}, apperr.NotFound("attachment not found")
|
||||
}
|
||||
return a, postgres.Classify(err)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"messenger/internal/platform/apperr"
|
||||
)
|
||||
|
||||
const MaxUploadBytes int64 = 10 * 1024 * 1024
|
||||
const MaxImagePixels int64 = 40_000_000
|
||||
|
||||
func ValidateFilename(name string) error {
|
||||
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 {
|
||||
return apperr.Invalid("filename must contain 1-255 characters")
|
||||
}
|
||||
for _, r := range name {
|
||||
if r == '/' || r == '\\' || unicode.IsControl(r) {
|
||||
return apperr.Invalid("filename contains a forbidden character")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The small header is replayed into the decoder; pixels are allocated only
|
||||
// after checking dimensions. The caller serializes decoding per replica.
|
||||
func ValidateImage(r io.Reader, mime string, maxPixels int64) error {
|
||||
limited := io.LimitReader(r, MaxUploadBytes+1)
|
||||
var header bytes.Buffer
|
||||
cfg, format, err := image.DecodeConfig(io.TeeReader(limited, &header))
|
||||
if err != nil {
|
||||
return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err)
|
||||
}
|
||||
if (format != "png" && format != "jpeg") || "image/"+format != mime {
|
||||
return apperr.UnsupportedMedia("image does not match Content-Type")
|
||||
}
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) {
|
||||
return apperr.PayloadTooLarge("image exceeds pixel limit")
|
||||
}
|
||||
_, _, err = image.Decode(io.MultiReader(bytes.NewReader(header.Bytes()), limited))
|
||||
if err != nil {
|
||||
return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"messenger/internal/platform/apperr"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateFilename(t *testing.T) {
|
||||
for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} {
|
||||
if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid {
|
||||
t.Errorf("%q: %v", name, err)
|
||||
}
|
||||
}
|
||||
if err := ValidateFilename("фото 👋.png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type failingRead struct{ prefix []byte }
|
||||
|
||||
func (r *failingRead) Read(p []byte) (int, error) {
|
||||
if len(r.prefix) > 0 {
|
||||
n := copy(p, r.prefix)
|
||||
r.prefix = r.prefix[n:]
|
||||
return n, nil
|
||||
}
|
||||
return 0, errors.New("injected storage read outage")
|
||||
}
|
||||
|
||||
type openedBlob struct {
|
||||
Blobs
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(b.reader), nil
|
||||
}
|
||||
|
||||
func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) {
|
||||
var pngData bytes.Buffer
|
||||
if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} {
|
||||
s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0)
|
||||
err := s.validate(context.Background(), "staging/test", "image/png")
|
||||
if apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImageRejectsCorruptionAndLimitsPixelsBeforeDecode(t *testing.T) {
|
||||
im := image.NewNRGBA(image.Rect(0, 0, 3, 2))
|
||||
im.Set(1, 1, color.NRGBA{R: 200, A: 255})
|
||||
var b bytes.Buffer
|
||||
if err := png.Encode(&b, im); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, mime string
|
||||
data []byte
|
||||
pixels int64
|
||||
kind apperr.Kind
|
||||
}{
|
||||
{"valid", "image/png", b.Bytes(), 6, apperr.KindInternal},
|
||||
{"spoof", "image/jpeg", b.Bytes(), 6, apperr.KindUnsupportedMedia},
|
||||
{"truncated", "image/png", b.Bytes()[:len(b.Bytes())-20], 6, apperr.KindUnsupportedMedia},
|
||||
{"pixels", "image/png", b.Bytes(), 5, apperr.KindPayloadTooLarge},
|
||||
{"garbage", "image/png", []byte("not a picture"), 6, apperr.KindUnsupportedMedia},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := ValidateImage(bytes.NewReader(tc.data), tc.mime, tc.pixels)
|
||||
if tc.name == "valid" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || apperr.KindOf(err) != tc.kind {
|
||||
t.Fatalf("got %v, want kind %v", err, tc.kind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package chat_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/httpx"
|
||||
)
|
||||
|
||||
func messageRouter(f fixture, actor uuid.UUID) *gin.Engine {
|
||||
r := gin.New()
|
||||
r.Use(func(c *gin.Context) { httpx.SetUserID(c, actor) })
|
||||
chat.NewHandler(f.svc).Register(r)
|
||||
return r
|
||||
}
|
||||
|
||||
func sendJSON(r http.Handler, chatID uuid.UUID, key, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/chats/"+chatID.String()+"/messages", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Idempotency-Key", key)
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestMessageAttachmentJSONEmptyArrayAndOmitted(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice)
|
||||
r := messageRouter(f, f.alice)
|
||||
for i, body := range []string{`{"text":"hello","attachment_ids":[]}`, `{"text":"hello"}`} {
|
||||
rec := sendJSON(r, c, fmt.Sprint(i), body)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &fields); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(fields["attachment_ids"]) != "[]" {
|
||||
t.Fatalf("attachment_ids = %s, want []", fields["attachment_ids"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageAttachmentJSONRejectsInvalidArrays(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice)
|
||||
r := messageRouter(f, f.alice)
|
||||
id := uuid.NewString()
|
||||
for _, raw := range []string{
|
||||
"null", `"not an array"`, "{}", "1", "[null]", "[1]", `["bad"]`,
|
||||
`["00000000-0000-0000-0000-000000000000"]`,
|
||||
fmt.Sprintf("[%q]", strings.ReplaceAll(id, "-", "")),
|
||||
fmt.Sprintf("[%q,%q]", id, strings.ToUpper(id)),
|
||||
"[" + strings.TrimSuffix(strings.Repeat(fmt.Sprintf("%q,", id), 11), ",") + "]",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
rec := sendJSON(r, c, "invalid", `{"text":"hello","attachment_ids":`+raw+"}")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("send: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package chat_test
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/platform/pagination"
|
||||
)
|
||||
|
||||
func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID {
|
||||
t.Helper()
|
||||
id := ident.New()
|
||||
hash := sha256.Sum256([]byte(id.String()))
|
||||
_, err := db.Pool.Exec(f.ctx, `
|
||||
WITH file AS (
|
||||
INSERT INTO files (hash, size_bytes, mime_type, status, created_at)
|
||||
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], status, id, chatID, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
a, b := f.attachment(t, c, f.bob, "ready"), f.attachment(t, c, f.alice, "ready")
|
||||
r := messageRouter(f, f.alice)
|
||||
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a)
|
||||
first := sendJSON(r, c, "attachments", body)
|
||||
if first.Code != http.StatusCreated {
|
||||
t.Fatalf("attachment-only send: %d %s", first.Code, first.Body)
|
||||
}
|
||||
var message chat.MessageResponse
|
||||
if err := json.Unmarshal(first.Body.Bytes(), &message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if message.Text != "" || !reflect.DeepEqual(message.AttachmentIDs, []string{b.String(), a.String()}) {
|
||||
t.Fatalf("message = %+v", message)
|
||||
}
|
||||
if again := sendJSON(r, c, "attachments", body); again.Code != http.StatusCreated || again.Body.String() != first.Body.String() {
|
||||
t.Fatalf("replay: %d %s, want %s", again.Code, again.Body, first.Body)
|
||||
}
|
||||
for _, changed := range []string{
|
||||
fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, a, b),
|
||||
fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, b),
|
||||
fmt.Sprintf(`{"text":"different","attachment_ids":[%q,%q]}`, b, a),
|
||||
} {
|
||||
if rec := sendJSON(r, c, "attachments", changed); rec.Code != http.StatusConflict {
|
||||
t.Fatalf("changed payload: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/chats/"+c.String()+"/messages", nil))
|
||||
var page pagination.PageResponse[chat.MessageResponse]
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("history: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], message) {
|
||||
t.Fatalf("history = %+v, want %+v", page, message)
|
||||
}
|
||||
without := sendJSON(r, c, "empty", `{"text":"text-only"}`)
|
||||
empty := sendJSON(r, c, "empty", `{"attachment_ids":[],"text":"text-only"}`)
|
||||
if without.Code != http.StatusCreated || empty.Code != http.StatusCreated || without.Body.String() != empty.Body.String() {
|
||||
t.Fatalf("omitted vs empty replay: %d %s; %d %s", without.Code, without.Body, empty.Code, empty.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
other := f.newChat(t, f.alice)
|
||||
ready := f.attachment(t, c, f.bob, "ready")
|
||||
foreign := f.attachment(t, other, f.alice, "ready")
|
||||
failed := f.attachment(t, c, f.alice, "failed")
|
||||
pending := f.attachment(t, c, f.alice, "pending")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
actor, attachment uuid.UUID
|
||||
want int
|
||||
}{
|
||||
{"unknown", f.alice, ident.New(), http.StatusNotFound},
|
||||
{"foreign", f.alice, foreign, http.StatusNotFound},
|
||||
{"failed", f.alice, failed, http.StatusConflict},
|
||||
{"pending", f.alice, pending, http.StatusConflict},
|
||||
{"outsider", f.eve, ready, http.StatusNotFound},
|
||||
{"other-owner", f.alice, ready, http.StatusCreated},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment)
|
||||
rec := sendJSON(messageRouter(f, tc.actor), c, "same", body)
|
||||
if rec.Code != tc.want {
|
||||
t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageJSONRequiresTextString(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice)
|
||||
r := messageRouter(f, f.alice)
|
||||
for _, body := range []string{"null", `{}`, `{"text":null}`, `{"text":1}`, `{"text":[]}`, `{"text":"","attachment_ids":[]}`} {
|
||||
if rec := sendJSON(r, c, "text", body); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
if rec := sendJSON(r, c, "text", `{"text":"valid"}`); rec.Code != http.StatusCreated {
|
||||
t.Fatalf("valid payload after rejected requests: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package chat_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
func TestAttachmentMessageValidatesAndOwnsOrderedIDs(t *testing.T) {
|
||||
a, b := ident.New(), ident.New()
|
||||
ids := []uuid.UUID{b, a}
|
||||
m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), "", ids...)
|
||||
if err != nil || m.Text != "" || !reflect.DeepEqual(m.AttachmentIDs, ids) {
|
||||
t.Fatalf("attachment-only message = %+v, %v", m, err)
|
||||
}
|
||||
ids[0] = a
|
||||
if m.AttachmentIDs[0] != b {
|
||||
t.Fatal("message aliases caller's attachment slice")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
text string
|
||||
ids []uuid.UUID
|
||||
}{
|
||||
{"ok", []uuid.UUID{a, a}},
|
||||
{"ok", []uuid.UUID{uuid.Nil}},
|
||||
{"ok", make([]uuid.UUID, 11)},
|
||||
{strings.Repeat("x", 4001), []uuid.UUID{a}},
|
||||
{"nul\x00", []uuid.UUID{a}},
|
||||
} {
|
||||
_, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), tc.text, tc.ids...)
|
||||
if err == nil || apperr.KindOf(err) != apperr.KindInvalid {
|
||||
t.Fatalf("invalid message accepted: %v", err)
|
||||
}
|
||||
}
|
||||
ten := make([]uuid.UUID, 10)
|
||||
for i := range ten {
|
||||
ten[i] = ident.New()
|
||||
}
|
||||
if _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " \n", ten...); err != nil {
|
||||
t.Fatalf("ten attachments: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameContentIncludesOrderedAttachmentsAndTreatsNilAsEmpty(t *testing.T) {
|
||||
a, b := ident.New(), ident.New()
|
||||
m := chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{b, a}}
|
||||
for _, tc := range []struct {
|
||||
ids []uuid.UUID
|
||||
equal bool
|
||||
}{
|
||||
{[]uuid.UUID{b, a}, true}, {[]uuid.UUID{a, b}, false}, {[]uuid.UUID{a}, false}, {nil, false},
|
||||
} {
|
||||
if got := m.SameContent(chat.Message{Text: "hello", AttachmentIDs: tc.ids}); got != tc.equal {
|
||||
t.Fatalf("attachments %v: equality=%v, want %v", tc.ids, got, tc.equal)
|
||||
}
|
||||
}
|
||||
if !(chat.Message{Text: "hello"}).SameContent(chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{}}) {
|
||||
t.Fatal("nil attachments must equal empty attachments")
|
||||
}
|
||||
}
|
||||
+56
-12
@@ -1,6 +1,8 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
@@ -36,7 +38,8 @@ type createChatRequest struct {
|
||||
}
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text string `json:"text"`
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type ChatResponse struct {
|
||||
@@ -47,11 +50,12 @@ type ChatResponse struct {
|
||||
}
|
||||
|
||||
type MessageResponse struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
func toChatResponse(c ChatWithMembers) ChatResponse {
|
||||
@@ -68,12 +72,17 @@ func toChatResponse(c ChatWithMembers) ChatResponse {
|
||||
}
|
||||
|
||||
func toMessageResponse(m Message) MessageResponse {
|
||||
ids := make([]string, 0, len(m.AttachmentIDs))
|
||||
for _, id := range m.AttachmentIDs {
|
||||
ids = append(ids, id.String())
|
||||
}
|
||||
return MessageResponse{
|
||||
ID: m.ID.String(),
|
||||
ChatID: m.ChatID.String(),
|
||||
SenderID: m.SenderID.String(),
|
||||
Text: m.Text,
|
||||
CreatedAt: httpx.Timestamp(m.CreatedAt),
|
||||
AttachmentIDs: ids,
|
||||
ID: m.ID.String(),
|
||||
ChatID: m.ChatID.String(),
|
||||
SenderID: m.SenderID.String(),
|
||||
Text: m.Text,
|
||||
CreatedAt: httpx.Timestamp(m.CreatedAt),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,7 +153,16 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text})
|
||||
if req.Text == nil {
|
||||
httpx.Error(c, apperr.Invalid("text must be a string"))
|
||||
return
|
||||
}
|
||||
attachments, err := decodeAttachmentIDs(req.AttachmentIDs)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: *req.Text, AttachmentIDs: attachments})
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
@@ -207,3 +225,29 @@ func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) {
|
||||
}
|
||||
return limit, &last, nil
|
||||
}
|
||||
|
||||
// RawMessage distinguishes an omitted optional array from an explicit null.
|
||||
func decodeAttachmentIDs(raw json.RawMessage) ([]uuid.UUID, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
|
||||
return nil, apperr.Invalid("attachment_ids must be an array")
|
||||
}
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return nil, apperr.Invalid("attachment_ids must be an array of UUIDs")
|
||||
}
|
||||
if len(values) > MaxAttachments {
|
||||
return nil, apperr.Invalid("attachment_ids must have at most 10 entries")
|
||||
}
|
||||
ids := make([]uuid.UUID, 0, len(values))
|
||||
for _, value := range values {
|
||||
id, err := ident.Parse(value)
|
||||
if err != nil {
|
||||
return nil, apperr.Invalid("attachment_ids must contain valid UUIDs")
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
+34
-15
@@ -2,6 +2,7 @@ package chat
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
@@ -12,19 +13,36 @@ import (
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
const MaxTextCodePoints = 4000
|
||||
const (
|
||||
MaxTextCodePoints = 4000
|
||||
MaxAttachments = 10
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
SenderID uuid.UUID
|
||||
Text string
|
||||
CreatedAt time.Time
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
SenderID uuid.UUID
|
||||
Text string
|
||||
CreatedAt time.Time
|
||||
AttachmentIDs []uuid.UUID
|
||||
}
|
||||
|
||||
// NewMessage keeps the text verbatim; only blank or oversized text is rejected.
|
||||
func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
// NewMessage keeps text and attachment order verbatim; it owns its attachment slice.
|
||||
func NewMessage(id, chatID, senderID uuid.UUID, text string, attachmentIDs ...uuid.UUID) (Message, error) {
|
||||
if len(attachmentIDs) > MaxAttachments {
|
||||
return Message{}, apperr.Invalid("attachment_ids must have at most 10 entries")
|
||||
}
|
||||
seen := make(map[uuid.UUID]struct{}, len(attachmentIDs))
|
||||
for _, id := range attachmentIDs {
|
||||
if id == uuid.Nil {
|
||||
return Message{}, apperr.Invalid("attachment_ids must contain non-nil UUIDs")
|
||||
}
|
||||
if _, duplicate := seen[id]; duplicate {
|
||||
return Message{}, apperr.Invalid("attachment_ids must be unique")
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 {
|
||||
return Message{}, apperr.Invalid("text must not be blank")
|
||||
}
|
||||
if utf8.RuneCountInString(text) > MaxTextCodePoints {
|
||||
@@ -35,16 +53,17 @@ func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) {
|
||||
return Message{}, apperr.Invalid("text must not contain the NUL character")
|
||||
}
|
||||
return Message{
|
||||
ID: id,
|
||||
ChatID: chatID,
|
||||
SenderID: senderID,
|
||||
Text: text,
|
||||
CreatedAt: ident.TimeOf(id),
|
||||
ID: id,
|
||||
ChatID: chatID,
|
||||
SenderID: senderID,
|
||||
Text: text,
|
||||
CreatedAt: ident.TimeOf(id),
|
||||
AttachmentIDs: slices.Clone(attachmentIDs),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SameContent reports whether two messages say the same thing, whatever their
|
||||
// ids and times.
|
||||
func (m Message) SameContent(other Message) bool {
|
||||
return m.Text == other.Text
|
||||
return m.Text == other.Text && slices.Equal(m.AttachmentIDs, other.AttachmentIDs)
|
||||
}
|
||||
|
||||
@@ -78,15 +78,16 @@ type MessagePayload struct {
|
||||
ChatID uuid.UUID
|
||||
IdempotencyKey string
|
||||
Text string
|
||||
AttachmentIDs []uuid.UUID
|
||||
}
|
||||
|
||||
// Send checks access and the text on every attempt, a replay included, as the
|
||||
// Send checks access and content on every attempt, a replay included, as the
|
||||
// contract requires of a repeat.
|
||||
func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) {
|
||||
if err := s.requireMember(ctx, actor, p.ChatID); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text)
|
||||
message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text, p.AttachmentIDs...)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -217,14 +218,14 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0] != first || page.Next == nil || *page.Next != first.ID {
|
||||
if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], first) || page.Next == nil || *page.Next != first.ID {
|
||||
t.Fatalf("first page = %+v", page)
|
||||
}
|
||||
rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rest.Items) != 1 || rest.Items[0] != second || rest.Next != nil {
|
||||
if len(rest.Items) != 1 || !reflect.DeepEqual(rest.Items[0], second) || rest.Next != nil {
|
||||
t.Fatalf("second page = %+v", rest)
|
||||
}
|
||||
if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) {
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/chat/store"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID {
|
||||
t.Helper()
|
||||
id := ident.New()
|
||||
hash := sha256.Sum256([]byte(id.String()))
|
||||
_, err := db.Pool.Exec(f.ctx, `
|
||||
WITH file AS (
|
||||
INSERT INTO files (hash, size_bytes, mime_type, status, created_at)
|
||||
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`,
|
||||
hash[:], status, id, chatID, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func attachedMessage(t *testing.T, c, sender uuid.UUID, ids ...uuid.UUID) chat.Message {
|
||||
t.Helper()
|
||||
m, err := chat.NewMessage(ident.New(), c, sender, "", ids...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func assertCounts(t *testing.T, messages, keys, joins int) {
|
||||
t.Helper()
|
||||
var gotMessages, gotKeys, gotJoins int
|
||||
err := db.Pool.QueryRow(context.Background(), `
|
||||
SELECT (SELECT count(*) FROM messages),
|
||||
(SELECT count(*) FROM message_idempotency_keys),
|
||||
(SELECT count(*) FROM message_attachments)`).Scan(&gotMessages, &gotKeys, &gotJoins)
|
||||
if err != nil || gotMessages != messages || gotKeys != keys || gotJoins != joins {
|
||||
t.Fatalf("counts = %d/%d/%d, %v; want %d/%d/%d", gotMessages, gotKeys, gotJoins, err, messages, keys, joins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
m := attachedMessage(t, c.ID, alice, ident.New())
|
||||
_, err := f.messages.AppendOnce(f.ctx, m, "unknown", keyTTL)
|
||||
if err == nil || apperr.KindOf(err) != apperr.KindNotFound {
|
||||
t.Fatalf("unknown attachment: %v; want not found", err)
|
||||
}
|
||||
var messages, keys int
|
||||
err = db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM messages), (SELECT count(*) FROM message_idempotency_keys)`).Scan(&messages, &keys)
|
||||
if err != nil || messages != 0 || keys != 0 {
|
||||
t.Fatalf("failed validation persisted %d messages/%d keys: %v", messages, keys, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) {
|
||||
for _, status := range []string{"foreign", "failed", "pending", "mixed"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice, bob := f.mustUser(t), f.mustUser(t)
|
||||
c := f.mustChat(t, alice, bob)
|
||||
other := f.mustChat(t, alice)
|
||||
ready := f.attachment(t, c.ID, bob, "ready")
|
||||
var invalid uuid.UUID
|
||||
want := apperr.KindConflict
|
||||
switch status {
|
||||
case "foreign":
|
||||
invalid = f.attachment(t, other.ID, alice, "ready")
|
||||
want = apperr.KindNotFound
|
||||
case "mixed":
|
||||
ready = f.attachment(t, c.ID, bob, "failed")
|
||||
invalid = ident.New()
|
||||
want = apperr.KindNotFound
|
||||
default:
|
||||
invalid = f.attachment(t, c.ID, alice, status)
|
||||
}
|
||||
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ready, invalid), "bad", keyTTL)
|
||||
if err == nil || apperr.KindOf(err) != want {
|
||||
t.Fatalf("validation: %v; want %v", err, want)
|
||||
}
|
||||
assertCounts(t, 0, 0, 0)
|
||||
good := f.attachment(t, c.ID, bob, "ready")
|
||||
f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad")
|
||||
assertCounts(t, 1, 1, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice, bob := f.mustUser(t), f.mustUser(t)
|
||||
c := f.mustChat(t, alice, bob)
|
||||
a, b := f.attachment(t, c.ID, bob, "ready"), f.attachment(t, c.ID, bob, "ready")
|
||||
first := attachedMessage(t, c.ID, alice, b, a)
|
||||
got := f.appendOnce(t, first, "k")
|
||||
if !reflect.DeepEqual(got, first) {
|
||||
t.Fatalf("append = %+v, want %+v", got, first)
|
||||
}
|
||||
replayed := f.appendOnce(t, attachedMessage(t, c.ID, alice, a, b), "k")
|
||||
if !reflect.DeepEqual(replayed, first) {
|
||||
t.Fatalf("replay must return original ordered attachments: %+v", replayed)
|
||||
}
|
||||
assertCounts(t, 1, 1, 2)
|
||||
expire(t)
|
||||
second := attachedMessage(t, c.ID, bob, a)
|
||||
f.appendOnce(t, second, "k2")
|
||||
replacement := attachedMessage(t, c.ID, alice, a, b)
|
||||
f.appendOnce(t, replacement, "k")
|
||||
assertCounts(t, 3, 2, 5)
|
||||
page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 1)
|
||||
if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) {
|
||||
t.Fatalf("first page = %+v, %v", page, err)
|
||||
}
|
||||
page, err = f.messages.ListByChat(f.ctx, c.ID, &first.ID, 10)
|
||||
if err != nil || !reflect.DeepEqual(page, []chat.Message{second, replacement}) {
|
||||
t.Fatalf("remaining page = %+v, %v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceRevalidatesAttachmentOnReplay(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a := f.attachment(t, c.ID, alice, "ready")
|
||||
first := attachedMessage(t, c.ID, alice, a)
|
||||
f.appendOnce(t, first, "k")
|
||||
if _, err := db.Pool.Exec(f.ctx, `UPDATE files SET status = 'failed'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL)
|
||||
if err == nil || apperr.KindOf(err) != apperr.KindConflict {
|
||||
t.Fatalf("failed replay attachment = %v, want conflict", err)
|
||||
}
|
||||
page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10)
|
||||
if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) {
|
||||
t.Fatalf("failed blob must preserve history attachment ids: %+v, %v", page, err)
|
||||
}
|
||||
assertCounts(t, 1, 1, 1)
|
||||
}
|
||||
|
||||
func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a := f.attachment(t, c.ID, alice, "ready")
|
||||
// A database fault at the join boundary must not leave a key or message.
|
||||
_, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments DROP CONSTRAINT test_reject_join`); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL); err == nil {
|
||||
t.Fatal("join fault must fail append")
|
||||
}
|
||||
assertCounts(t, 0, 0, 0)
|
||||
}
|
||||
|
||||
func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a, b := f.attachment(t, c.ID, alice, "ready"), f.attachment(t, c.ID, alice, "ready")
|
||||
const requests = 20
|
||||
results := make(chan chat.Message, requests)
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range requests {
|
||||
ids := []uuid.UUID{a, b}
|
||||
if i%2 != 0 {
|
||||
ids = []uuid.UUID{b, a}
|
||||
}
|
||||
m := attachedMessage(t, c.ID, alice, ids...)
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
results <- got
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
var winner *chat.Message
|
||||
n := 0
|
||||
for got := range results {
|
||||
n++
|
||||
if winner == nil {
|
||||
winner = &got
|
||||
} else if !reflect.DeepEqual(got, *winner) {
|
||||
t.Fatalf("concurrent replay differs: %+v, want %+v", got, *winner)
|
||||
}
|
||||
}
|
||||
if n != requests {
|
||||
t.Fatalf("got %d successful replies, want %d", n, requests)
|
||||
}
|
||||
assertCounts(t, 1, 1, 2)
|
||||
}
|
||||
|
||||
type queryCounter struct{ count atomic.Int32 }
|
||||
|
||||
func (c *queryCounter) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
|
||||
c.count.Add(1)
|
||||
return ctx
|
||||
}
|
||||
func (*queryCounter) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {}
|
||||
|
||||
func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
var ids []uuid.UUID
|
||||
for range 10 {
|
||||
ids = append(ids, f.attachment(t, c.ID, alice, "ready"))
|
||||
}
|
||||
cfg := db.Pool.Config()
|
||||
counter := &queryCounter{}
|
||||
cfg.ConnConfig.Tracer = counter
|
||||
pool, err := pgxpool.NewWithConfig(f.ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := store.NewMessages(pool)
|
||||
counter.count.Store(0)
|
||||
if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids[0]), "one", keyTTL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one := counter.count.Swap(0)
|
||||
if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids...), "ten", keyTTL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ten := counter.count.Swap(0)
|
||||
if one != ten {
|
||||
t.Fatalf("queries grew from %d to %d for 1 vs 10 attachments", one, ten)
|
||||
}
|
||||
page, err := repo.ListByChat(f.ctx, c.ID, nil, 10)
|
||||
if err != nil || len(page) != 2 || !reflect.DeepEqual(page[1].AttachmentIDs, ids) {
|
||||
t.Fatalf("history = %+v, %v", page, err)
|
||||
}
|
||||
if n := counter.count.Load(); n != 1 {
|
||||
t.Fatalf("history used %d queries, want one", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceRetriesWhenBoundKeyDisappears(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
first := f.newMessage(t, c.ID, alice)
|
||||
f.appendOnce(t, first, "vanish")
|
||||
next := f.newMessage(t, c.ID, alice)
|
||||
// Simulate the missing-key boundary after an unsuccessful claim using a
|
||||
// statement trigger. Its second invocation sees no key and is a no-op.
|
||||
_, err := db.Pool.Exec(f.ctx, fmt.Sprintf(`
|
||||
CREATE FUNCTION test_vanish_key() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM messages WHERE id = '%s'::uuid) THEN
|
||||
DELETE FROM message_idempotency_keys WHERE key = 'vanish';
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE TRIGGER test_vanish_key AFTER INSERT ON messages
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION test_vanish_key()`, next.ID.String()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if _, err := db.Pool.Exec(f.ctx, `DROP TRIGGER test_vanish_key ON messages; DROP FUNCTION test_vanish_key()`); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(f.ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
got, err := f.messages.AppendOnce(ctx, next, "vanish", keyTTL)
|
||||
if err != nil || got.ID != next.ID {
|
||||
t.Fatalf("missing bound key = %+v, %v; want new message", got, err)
|
||||
}
|
||||
if n := f.count(t, c.ID); n != 2 {
|
||||
t.Fatalf("history has %d messages, want 2", n)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
)
|
||||
|
||||
@@ -118,9 +119,8 @@ func NewMessages(pool *pgxpool.Pool) *Messages {
|
||||
return &Messages{pool: pool}
|
||||
}
|
||||
|
||||
// claimSQL binds the key and inserts the message in one statement, so the
|
||||
// key's lock never spans a round trip. An expired key is taken over; a valid
|
||||
// one leaves claimed, and so the message insert, empty.
|
||||
// claimSQL binds the key and inserts the message in one statement. An expired
|
||||
// key is taken over; a valid one leaves claimed and the message insert empty.
|
||||
const claimSQL = `
|
||||
WITH claimed AS (
|
||||
INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id)
|
||||
@@ -132,30 +132,100 @@ const claimSQL = `
|
||||
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
|
||||
SELECT $2, message_id, $1, $6, $7 FROM claimed`
|
||||
|
||||
// The bound message is read by a second statement: the claim's snapshot
|
||||
// cannot see a key that a concurrent request has just committed.
|
||||
// The transaction keeps attachment readiness, message joins and the key atomic.
|
||||
// The replay uses a new statement snapshot to see a concurrent winner.
|
||||
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
|
||||
tag, err := r.pool.Exec(ctx, claimSQL,
|
||||
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
|
||||
if err != nil {
|
||||
return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err))
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
return m, nil
|
||||
}
|
||||
var stored chat.Message
|
||||
err = r.pool.QueryRow(ctx, `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
FROM message_idempotency_keys k
|
||||
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
|
||||
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
|
||||
m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
|
||||
err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
|
||||
if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, claimSQL,
|
||||
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
if len(m.AttachmentIDs) > 0 {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal)
|
||||
SELECT $1, $2, id, (ord - 1)::smallint
|
||||
FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)`,
|
||||
m.ChatID, m.ID, m.AttachmentIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert message attachments: %w", err)
|
||||
}
|
||||
}
|
||||
stored = m
|
||||
return nil
|
||||
}
|
||||
stored, err = scanMessage(tx.QueryRow(ctx, `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal)
|
||||
FROM message_idempotency_keys k
|
||||
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
|
||||
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
|
||||
m.SenderID, m.ChatID, key))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// A vanished key is available again; retry its atomic claim.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("select message by key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err))
|
||||
return chat.Message{}, postgres.Classify(err)
|
||||
}
|
||||
return stored, nil
|
||||
}
|
||||
|
||||
func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids []uuid.UUID) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Shared row locks prevent deletion or a readiness change until joins commit.
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT f.status
|
||||
FROM attachments a
|
||||
JOIN files f ON f.hash = a.file_hash
|
||||
WHERE a.chat_id = $1 AND a.id = ANY($2::uuid[])
|
||||
ORDER BY a.id
|
||||
FOR SHARE OF a, f`, chatID, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check attachments: %w", err)
|
||||
}
|
||||
statuses, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return fmt.Errorf("read attachment status: %w", err)
|
||||
}
|
||||
if len(statuses) != len(ids) {
|
||||
return apperr.NotFound("attachment not found")
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status != "ready" {
|
||||
return apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanMessage(row pgx.Row) (chat.Message, error) {
|
||||
var m chat.Message
|
||||
err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs)
|
||||
if len(m.AttachmentIDs) == 0 {
|
||||
m.AttachmentIDs = nil
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// DeleteExpiredKeys skips rows locked by another cleaner or a request rather
|
||||
// than waiting for them.
|
||||
func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) {
|
||||
@@ -179,14 +249,16 @@ func interval(d time.Duration) string {
|
||||
return fmt.Sprintf("%d milliseconds", d.Milliseconds())
|
||||
}
|
||||
|
||||
// ListByChat is the history query; scripts/explain-history.sql runs the same
|
||||
// statement for the EXPLAIN evidence.
|
||||
// ListByChat keeps keyset pagination on messages and collects each ordered
|
||||
// attachment array in the same statement.
|
||||
func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal)
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
AND m.id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
ORDER BY id
|
||||
LIMIT $3`,
|
||||
chatID, after, limit)
|
||||
@@ -194,9 +266,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid
|
||||
return nil, fmt.Errorf("select messages: %w", postgres.Classify(err))
|
||||
}
|
||||
messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) {
|
||||
var m chat.Message
|
||||
err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
|
||||
return m, err
|
||||
return scanMessage(row)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read messages: %w", postgres.Classify(err))
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -214,7 +215,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 || got[0] != first || got[1] != second || got[2] != third {
|
||||
if len(got) != 3 || !reflect.DeepEqual(got[0], first) || !reflect.DeepEqual(got[1], second) || !reflect.DeepEqual(got[2], third) {
|
||||
t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,9 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Allow a 30s binary request, its independent 3s cleanup, and 5s margin.
|
||||
const defaultShutdownTimeout = 38 * time.Second
|
||||
|
||||
type Logging struct {
|
||||
Env string
|
||||
Level slog.Level
|
||||
@@ -43,6 +46,7 @@ type Config struct {
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DebugAddr string
|
||||
S3 S3
|
||||
}
|
||||
|
||||
type Migrate struct {
|
||||
@@ -58,7 +62,7 @@ func Load(lookup Lookup) (Config, error) {
|
||||
cfg := Config{
|
||||
Host: "0.0.0.0",
|
||||
Port: 8080,
|
||||
ShutdownTimeout: 8 * time.Second,
|
||||
ShutdownTimeout: defaultShutdownTimeout,
|
||||
MaxBodyBytes: 64 * 1024,
|
||||
SessionTTL: time.Hour,
|
||||
DrainDelay: 2 * time.Second,
|
||||
@@ -136,6 +140,11 @@ func Load(lookup Lookup) (Config, error) {
|
||||
if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if _, ok := lookup("S3_ENDPOINT"); ok {
|
||||
if cfg.S3, err = LoadS3(lookup); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ func TestDefaults(t *testing.T) {
|
||||
if cfg.Env != "lab" || cfg.Level != slog.LevelInfo {
|
||||
t.Fatalf("cfg = %+v", cfg)
|
||||
}
|
||||
if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 {
|
||||
if cfg.ShutdownTimeout != 38*time.Second || cfg.MaxBodyBytes != 65536 {
|
||||
t.Fatalf("cfg = %+v", cfg)
|
||||
}
|
||||
if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL {
|
||||
@@ -70,6 +70,19 @@ func TestDefaults(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultShutdownTimeoutCoversBinaryLifecycle(t *testing.T) {
|
||||
cfg, err := config.Load(env(required(t, nil)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Binary requests can use 30s, followed by 3s independent cleanup. Keep
|
||||
// another 5s for shutdown scheduling and closing the remaining resources.
|
||||
budget := 30*time.Second + 3*time.Second + 5*time.Second
|
||||
if cfg.ShutdownTimeout < budget {
|
||||
t.Fatalf("default shutdown budget %v cannot cover binary request and cleanup budget %v", cfg.ShutdownTimeout, budget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) {
|
||||
for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} {
|
||||
cases := map[string]func(map[string]string){
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
Endpoint, Region, Bucket string
|
||||
AccessKey, SecretKey string
|
||||
MaxBytes, MaxPixels int64
|
||||
}
|
||||
|
||||
func LoadS3(lookup Lookup) (S3, error) {
|
||||
cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000}
|
||||
for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} {
|
||||
if v, ok := lookup(key); ok {
|
||||
*dest = v
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if cfg.AccessKey, err = secret(lookup, "S3_ACCESS_KEY_ID_FILE"); err != nil {
|
||||
return S3{}, err
|
||||
}
|
||||
if cfg.SecretKey, err = secret(lookup, "S3_SECRET_ACCESS_KEY_FILE"); err != nil {
|
||||
return S3{}, err
|
||||
}
|
||||
for _, p := range []struct {
|
||||
key string
|
||||
dest *int64
|
||||
max int64
|
||||
}{{"MAX_UPLOAD_BYTES", &cfg.MaxBytes, 10485760}, {"MAX_IMAGE_PIXELS", &cfg.MaxPixels, 40000000}} {
|
||||
if v, ok := lookup(p.key); ok {
|
||||
n, e := strconv.ParseInt(v, 10, 64)
|
||||
if e != nil || n < 1 || n > p.max {
|
||||
return S3{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max)
|
||||
}
|
||||
*p.dest = n
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// ReconcileS3 selects the separate listing identity used by background scans.
|
||||
func ReconcileS3(lookup Lookup) Lookup {
|
||||
return func(key string) (string, bool) {
|
||||
if key == "S3_ACCESS_KEY_ID_FILE" {
|
||||
return lookup("S3_RECONCILE_ACCESS_KEY_ID_FILE")
|
||||
}
|
||||
if key == "S3_SECRET_ACCESS_KEY_FILE" {
|
||||
return lookup("S3_RECONCILE_SECRET_ACCESS_KEY_FILE")
|
||||
}
|
||||
return lookup(key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"messenger/internal/platform/config"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestS3LimitsAndSecretFiles(t *testing.T) {
|
||||
values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")}
|
||||
cfg, err := config.LoadS3(env(values))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.AccessKey != "access" || cfg.SecretKey != "private" || cfg.MaxBytes != 10485760 || cfg.MaxPixels != 40000000 {
|
||||
t.Fatal("S3 defaults or secret trimming broken")
|
||||
}
|
||||
for _, tc := range []struct{ k, v string }{{"MAX_UPLOAD_BYTES", "0"}, {"MAX_UPLOAD_BYTES", "10485761"}, {"MAX_IMAGE_PIXELS", "40000001"}, {"MAX_IMAGE_PIXELS", "x"}} {
|
||||
v := map[string]string{}
|
||||
for k, x := range values {
|
||||
v[k] = x
|
||||
}
|
||||
v[tc.k] = tc.v
|
||||
if _, err := config.LoadS3(env(v)); err == nil {
|
||||
t.Fatalf("accepted %s=%s", tc.k, tc.v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -123,6 +123,11 @@ func Timeout(d time.Duration) gin.HandlerFunc {
|
||||
// decoder as *http.MaxBytesError.
|
||||
func bodyLimit(maxBytes int64) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// The authenticated binary handler installs its own 10 MiB limit.
|
||||
if c.Request.Method == http.MethodPost && c.FullPath() == "/api/v1/chats/:chat_id/attachments" {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
if c.Request.Body != nil {
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package s3store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) {
|
||||
var ready atomic.Bool
|
||||
var reads atomic.Int32
|
||||
var mu sync.Mutex
|
||||
objects := map[string]string{".health/storage-v1": "messenger s3 readiness v1\n"}
|
||||
var freshReads, deleted int
|
||||
requested := make(chan struct{}, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
key := strings.TrimPrefix(r.URL.Path, "/attachments/")
|
||||
if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" {
|
||||
w.WriteHeader(200)
|
||||
return
|
||||
}
|
||||
if r.Method == "GET" && key == ".health/storage-v1" {
|
||||
reads.Add(1)
|
||||
if !ready.Load() {
|
||||
select {
|
||||
case requested <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
w.WriteHeader(500)
|
||||
io.WriteString(w, `<Error><Code>InternalError</Code><Message>volume not registered</Message></Error>`)
|
||||
return
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
value, ok := objects[key]
|
||||
if !ok {
|
||||
w.WriteHeader(404)
|
||||
io.WriteString(w, `<Error><Code>NoSuchKey</Code></Error>`)
|
||||
return
|
||||
}
|
||||
if key != ".health/storage-v1" {
|
||||
freshReads++
|
||||
}
|
||||
w.Header().Set("ETag", `"probe"`)
|
||||
w.Header().Set("Last-Modified", "Mon, 16 Nov 2026 08:00:00 GMT")
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(value)))
|
||||
io.WriteString(w, value)
|
||||
case "PUT":
|
||||
var body io.Reader = r.Body
|
||||
if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||
body = httputil.NewChunkedReader(body)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(body, 1024))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
objects[key] = string(data)
|
||||
w.Header().Set("ETag", `"probe"`)
|
||||
case "DELETE":
|
||||
if key == ".health/storage-v1" {
|
||||
t.Error("deleted retained readiness marker")
|
||||
}
|
||||
delete(objects, key)
|
||||
deleted++
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
t.Errorf("unexpected storage probe: %s %s", r.Method, r.URL)
|
||||
w.WriteHeader(400)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Bootstrap(ctx) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("bootstrap returned before reading retained storage: %v", err)
|
||||
case <-requested:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("bootstrap accepted unavailable retained storage: %v", err)
|
||||
case <-time.After(40 * time.Millisecond):
|
||||
}
|
||||
ready.Store(true)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if reads.Load() < 2 || freshReads == 0 || deleted == 0 || len(objects) != 1 {
|
||||
t.Fatalf("incomplete readiness proof: retainedReads=%d freshReads=%d deletes=%d remaining=%d", reads.Load(), freshReads, deleted, len(objects))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" {
|
||||
w.WriteHeader(200)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(503)
|
||||
io.WriteString(w, `<Error><Code>ServiceUnavailable</Code><Message>volume not registered</Message></Error>`)
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
if err := s.Bootstrap(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("storage readiness escaped its deadline: %v", err)
|
||||
}
|
||||
if time.Since(started) > time.Second {
|
||||
t.Fatal("readiness did not honor caller deadline")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
package s3store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/emptypb"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/s3store"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
func TestSeaweedStore(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
s, err := testenv.StartS3(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
newClient := func(access, secret string) *minio.Client {
|
||||
t.Helper()
|
||||
c, err := minio.New(strings.TrimPrefix(s.Endpoint, "http://"), &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Region: s.Region, BucketLookup: minio.BucketLookupPath, MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
api := newClient(s.APIAccessKey, s.APISecretKey)
|
||||
admin := newClient(s.AdminAccessKey, s.AdminSecretKey)
|
||||
denied := func(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
var response minio.ErrorResponse
|
||||
if !errors.As(err, &response) || response.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("expected access denied, got %v", err)
|
||||
}
|
||||
}
|
||||
t.Run("remote gRPC cannot modify identity cache", func(t *testing.T) {
|
||||
conn, err := grpc.NewClient("passthrough:///"+s.GRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, "tcp", address)
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
opCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
err = conn.Invoke(opCtx, "/messaging_pb.SeaweedS3IamCache/PutIdentity", &emptypb.Empty{}, &emptypb.Empty{})
|
||||
if code := status.Code(err); code != codes.PermissionDenied && code != codes.Unauthenticated {
|
||||
t.Fatalf("remote IAM mutation was not rejected before argument validation: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("bootstrap is idempotent and API cannot administer", func(t *testing.T) {
|
||||
if err := s.Admin.Bootstrap(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
denied(t, s.APIStore.Bootstrap(ctx))
|
||||
denied(t, api.MakeBucket(ctx, "forbidden-bucket", minio.MakeBucketOptions{Region: s.Region}))
|
||||
denied(t, api.RemoveBucket(ctx, s.Bucket))
|
||||
buckets, err := api.ListBuckets(ctx)
|
||||
// SeaweedFS returns a filtered empty result for an authenticated caller
|
||||
// without List rights, rather than rejecting ListBuckets outright.
|
||||
if err != nil || len(buckets) != 0 {
|
||||
t.Fatalf("API enumerated buckets: count=%d error=%v", len(buckets), err)
|
||||
}
|
||||
_, err = s.APIStore.List(ctx, "", "", 1)
|
||||
denied(t, err)
|
||||
})
|
||||
t.Run("known length nonseekable put copy stat and stream", func(t *testing.T) {
|
||||
const payload = "nonseekable image payload"
|
||||
r := struct{ io.Reader }{strings.NewReader(payload)}
|
||||
started := time.Now()
|
||||
if err := s.APIStore.Put(ctx, "staging/source", r, int64(len(payload)), "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("cold first PUT: %s", time.Since(started))
|
||||
if err := s.APIStore.Copy(ctx, "staging/source", "blobs/durable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, err := s.APIStore.Stat(ctx, "blobs/durable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if obj.Key != "blobs/durable" || obj.Size != 25 || obj.Modified.IsZero() {
|
||||
t.Fatalf("wrong metadata: %+v", obj)
|
||||
}
|
||||
body, err := s.APIStore.Open(ctx, "blobs/durable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(body)
|
||||
body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(b) != payload {
|
||||
t.Fatalf("wrong body: %q", b)
|
||||
}
|
||||
meta, err := api.StatObject(ctx, s.Bucket, "blobs/durable", minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ContentType != "image/png" {
|
||||
t.Fatalf("copy lost MIME: %s", meta.ContentType)
|
||||
}
|
||||
for range 2 {
|
||||
if err := s.APIStore.Delete(ctx, "staging/source"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := s.APIStore.Stat(ctx, "staging/source"); !errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("API delete did not remove staging: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("anonymous get put head and listing are refused", func(t *testing.T) {
|
||||
client := &http.Client{Timeout: 3 * time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
for _, tc := range []struct{ method, path string }{{"GET", "/"}, {"GET", "/" + s.Bucket + "?list-type=2"}, {"GET", "/" + s.Bucket + "/blobs/durable"}, {"HEAD", "/" + s.Bucket + "/blobs/durable"}, {"PUT", "/" + s.Bucket + "/anonymous"}} {
|
||||
req, _ := http.NewRequestWithContext(ctx, tc.method, s.Endpoint+tc.path, strings.NewReader("x"))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("anonymous %s %s: %d", tc.method, tc.path, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("identities cannot escape their bucket", func(t *testing.T) {
|
||||
if err := admin.MakeBucket(ctx, "private-other", minio.MakeBucketOptions{Region: s.Region}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := admin.PutObject(ctx, "private-other", "secret", strings.NewReader("private"), 7, minio.PutObjectOptions{DisableMultipart: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} {
|
||||
outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = outside.Stat(ctx, "secret")
|
||||
denied(t, err)
|
||||
_, err = outside.Open(ctx, "secret")
|
||||
denied(t, err)
|
||||
denied(t, outside.Put(ctx, "secret", strings.NewReader("x"), 1, "image/png"))
|
||||
denied(t, outside.Delete(ctx, "secret"))
|
||||
_, err = outside.List(ctx, "", "", 1)
|
||||
denied(t, err)
|
||||
c := newClient(pair[0], pair[1])
|
||||
_, err = c.CopyObject(ctx, minio.CopyDestOptions{Bucket: s.Bucket, Object: "blobs/stolen"}, minio.CopySrcOptions{Bucket: "private-other", Object: "secret"})
|
||||
denied(t, err)
|
||||
}
|
||||
})
|
||||
t.Run("exclusive lexicographic pages including absent cursor", func(t *testing.T) {
|
||||
for _, key := range []string{"page/z", "page/a", "page/a/nested", "page/b", "page/c"} {
|
||||
if err := s.Store.Put(ctx, key, strings.NewReader("x"), 1, "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var got []string
|
||||
after := ""
|
||||
for {
|
||||
page, err := s.Store.List(ctx, "page/", after, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page) == 0 {
|
||||
break
|
||||
}
|
||||
for _, obj := range page {
|
||||
if obj.Size != 1 || obj.Modified.IsZero() {
|
||||
t.Fatalf("wrong list metadata: %+v", obj)
|
||||
}
|
||||
got = append(got, obj.Key)
|
||||
}
|
||||
after = page[len(page)-1].Key
|
||||
if len(got) > 5 {
|
||||
t.Fatal("pagination did not advance")
|
||||
}
|
||||
}
|
||||
if strings.Join(got, ",") != "page/a,page/a/nested,page/b,page/c,page/z" {
|
||||
t.Fatalf("wrong order: %v", got)
|
||||
}
|
||||
page, err := s.Store.List(ctx, "page/", "page/az", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page) != 2 || page[0].Key != "page/b" || page[1].Key != "page/c" {
|
||||
t.Fatalf("absent cursor: %+v", page)
|
||||
}
|
||||
})
|
||||
t.Run("missing objects differ from bucket and permission failures", func(t *testing.T) {
|
||||
if _, err := s.Store.Stat(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("stat missing: %v", err)
|
||||
}
|
||||
if _, err := s.Store.Open(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("open missing: %v", err)
|
||||
}
|
||||
if err := s.Store.Copy(ctx, "absent", "blobs/absent"); !errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("copy missing: %v", err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := s.Store.Delete(ctx, "absent"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := missingBucket.Stat(ctx, "absent"); err == nil || errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("missing bucket treated as missing object: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("short readers are not successful uploads", func(t *testing.T) {
|
||||
if err := s.Store.Put(ctx, "staging/short", struct{ io.Reader }{strings.NewReader("short")}, 100, "image/png"); err == nil {
|
||||
t.Fatal("short body accepted")
|
||||
}
|
||||
})
|
||||
t.Run("container replacement preserves volume data and authentication", func(t *testing.T) {
|
||||
if err := s.Recreate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := s.APIStore.Open(ctx, "blobs/durable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(body)
|
||||
body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(b) != "nonseekable image payload" {
|
||||
t.Fatalf("object lost on replacement: %q", b)
|
||||
}
|
||||
if _, err := s.Store.List(ctx, "page/", "", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("outage is bounded and never missing", func(t *testing.T) {
|
||||
if err := s.Stop(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opCtx, stop := context.WithTimeout(ctx, 150*time.Millisecond)
|
||||
defer stop()
|
||||
start := time.Now()
|
||||
_, err := s.Store.Stat(opCtx, "blobs/durable")
|
||||
if err == nil || errors.Is(err, attachment.ErrObjectMissing) {
|
||||
t.Fatalf("outage treated as missing: %v", err)
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
t.Fatal("outage exceeded caller deadline")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package s3store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
)
|
||||
|
||||
const (
|
||||
storageMarkerKey = ".health/storage-v1"
|
||||
storageMarkerBody = "messenger s3 readiness v1\n"
|
||||
)
|
||||
|
||||
var errStorageMarker = errors.New("s3: storage readiness marker has unexpected contents")
|
||||
|
||||
// WaitReady is a startup barrier, deliberately separate from API readiness and
|
||||
// ordinary object operations. Gateway authentication and bucket metadata can be
|
||||
// available before the master elects its leader and registers retained volumes.
|
||||
// Read the persistent marker before writing anything, then prove new writes,
|
||||
// reads and deletes work. The marker lives outside the reconciler's namespaces.
|
||||
func (s *Store) WaitReady(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
readyCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
probeKey := ".health/check-" + rand.Text()
|
||||
dirty := false
|
||||
defer func() {
|
||||
if dirty {
|
||||
cleanupCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
defer stop()
|
||||
_ = s.Delete(cleanupCtx, probeKey)
|
||||
}
|
||||
}()
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
err := s.checkStorage(readyCtx, probeKey, &dirty)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if permanentStorageError(err) {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-readyCtx.Done():
|
||||
return fmt.Errorf("s3: wait for readable and writable storage: %w (last probe: %v)", readyCtx.Err(), err)
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) checkStorage(ctx context.Context, probeKey string, dirty *bool) error {
|
||||
err := s.readStorageMarker(ctx, storageMarkerKey)
|
||||
if errors.Is(err, attachment.ErrObjectMissing) {
|
||||
// Only a genuinely absent marker permits its first creation. Never
|
||||
// overwrite retained data to hide an unavailable volume or a bad marker.
|
||||
if err := s.Put(ctx, storageMarkerKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.readStorageMarker(ctx, storageMarkerKey)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*dirty = true
|
||||
if err := s.Put(ctx, probeKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.readStorageMarker(ctx, probeKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Delete(ctx, probeKey); err != nil {
|
||||
return err
|
||||
}
|
||||
*dirty = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) readStorageMarker(ctx context.Context, key string) error {
|
||||
r, err := s.Open(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, readErr := io.ReadAll(io.LimitReader(r, int64(len(storageMarkerBody))+1))
|
||||
closeErr := r.Close()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if string(data) != storageMarkerBody {
|
||||
return errStorageMarker
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func permanentStorageError(err error) bool {
|
||||
if errors.Is(err, errStorageMarker) {
|
||||
return true
|
||||
}
|
||||
var response minio.ErrorResponse
|
||||
return errors.As(err, &response) && response.StatusCode >= 400 && response.StatusCode < 500 && response.StatusCode != 408 && response.StatusCode != 429
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package s3store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
func TestRecreateReturnsWithRetainedObjectReadable(t *testing.T) {
|
||||
// Three independent server replacements each have the production 60-second
|
||||
// startup budget; no per-request transport timeout is changed by this test.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
|
||||
defer cancel()
|
||||
s, err := testenv.StartS3(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
defer func() {
|
||||
if t.Failed() {
|
||||
t.Log(s.Diagnostics(context.Background()))
|
||||
}
|
||||
}()
|
||||
const payload = "retained storage readiness proof"
|
||||
if err := s.Store.Put(ctx, "blobs/retained", strings.NewReader(payload), int64(len(payload)), "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for attempt := range 3 {
|
||||
started := time.Now()
|
||||
if err := s.Recreate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := s.Store.Open(ctx, "blobs/retained")
|
||||
if err != nil {
|
||||
t.Fatalf("replacement %d returned before retained bytes were readable (%s): %v", attempt+1, time.Since(started), err)
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(data) != payload {
|
||||
t.Fatalf("replacement %d lost retained bytes: %q %v", attempt+1, data, err)
|
||||
}
|
||||
t.Logf("replacement %d ready and retained bytes verified in %s", attempt+1, time.Since(started))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package s3store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
)
|
||||
|
||||
const MaxPageSize = 500
|
||||
|
||||
type Store struct {
|
||||
client *minio.Core
|
||||
region string
|
||||
bucket string
|
||||
}
|
||||
|
||||
var _ attachment.Blobs = (*Store)(nil)
|
||||
|
||||
// New validates configuration without contacting S3, so text-only service startup
|
||||
// and readiness do not depend on object storage being reachable.
|
||||
func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
|
||||
return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment")
|
||||
}
|
||||
if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" {
|
||||
return nil, errors.New("s3: region and credentials are required")
|
||||
}
|
||||
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
|
||||
return nil, fmt.Errorf("s3: invalid bucket: %w", err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{Timeout: time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
TLSHandshakeTimeout: time.Second,
|
||||
// The first write can wait for the single master's election and volume
|
||||
// creation. The caller's shorter context still takes precedence.
|
||||
ResponseHeaderTimeout: 30 * time.Second,
|
||||
ExpectContinueTimeout: time.Second,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
MaxIdleConns: 64,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
MaxConnsPerHost: 64,
|
||||
DisableCompression: true,
|
||||
}
|
||||
client, err := minio.NewCore(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
Region: region,
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
Transport: transport,
|
||||
// A consumed upload stream cannot be replayed. Let the caller decide
|
||||
// whether to retry a whole operation within its own deadline.
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: client: %w", err)
|
||||
}
|
||||
return &Store{client: client, region: region, bucket: bucket}, nil
|
||||
}
|
||||
|
||||
// Bootstrap is used only by the admin init command, never by API startup.
|
||||
func (s *Store) Bootstrap(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region})
|
||||
code := minio.ToErrorResponse(err).Code
|
||||
if err != nil && code != "BucketAlreadyOwnedByYou" {
|
||||
return wrap("create bucket", err)
|
||||
}
|
||||
return s.WaitReady(ctx)
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if size < 0 || r == nil {
|
||||
return errors.New("s3: PUT requires a reader and known nonnegative length")
|
||||
}
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, r, size, "", "", minio.PutObjectOptions{
|
||||
ContentType: mime,
|
||||
DisableMultipart: true,
|
||||
})
|
||||
return wrap("put object", err)
|
||||
}
|
||||
|
||||
func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Core.GetObject performs GET eagerly and returns the streaming body;
|
||||
// Client.GetObject is lazy and would defer missing/access errors to Read.
|
||||
r, _, _, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, wrap("open object", err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (attachment.Object, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return attachment.Object{}, err
|
||||
}
|
||||
obj, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
// HEAD has no XML error body. The SDK synthesizes NoSuchKey for any
|
||||
// object-path 404, including a missing bucket; confirm the bucket first.
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
|
||||
exists, bucketErr := s.client.BucketExists(ctx, s.bucket)
|
||||
if bucketErr != nil {
|
||||
return attachment.Object{}, wrap("stat bucket", bucketErr)
|
||||
}
|
||||
if !exists {
|
||||
return attachment.Object{}, errors.New("s3: stat object: bucket does not exist")
|
||||
}
|
||||
}
|
||||
return attachment.Object{}, wrap("stat object", err)
|
||||
}
|
||||
return attachment.Object{Key: key, Size: obj.Size, Modified: obj.LastModified}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Copy(ctx context.Context, source, dest string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.client.Client.CopyObject(ctx,
|
||||
minio.CopyDestOptions{Bucket: s.bucket, Object: dest},
|
||||
minio.CopySrcOptions{Bucket: s.bucket, Object: source},
|
||||
)
|
||||
return wrap("copy object", err)
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
// S3 DELETE is idempotent even when the object no longer exists.
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
|
||||
return nil
|
||||
}
|
||||
return wrap("delete object", err)
|
||||
}
|
||||
|
||||
// List returns at most limit keys in byte-lexicographic order, strictly after
|
||||
// after. The synchronous iterator stops before fetching the next page.
|
||||
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]attachment.Object, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit < 1 || limit > MaxPageSize {
|
||||
return nil, fmt.Errorf("s3: list limit must be between 1 and %d", MaxPageSize)
|
||||
}
|
||||
listCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
objects := make([]attachment.Object, 0, limit)
|
||||
for obj := range s.client.ListObjectsIter(listCtx, s.bucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix, StartAfter: after, Recursive: true, MaxKeys: limit,
|
||||
}) {
|
||||
if obj.Err != nil {
|
||||
return nil, wrap("list objects", obj.Err)
|
||||
}
|
||||
objects = append(objects, attachment.Object{Key: obj.Key, Size: obj.Size, Modified: obj.LastModified})
|
||||
if len(objects) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func wrap(op string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// Never treat authorization errors, a missing bucket, or an outage as a
|
||||
// missing blob: the reconciler would otherwise mark valid files failed.
|
||||
if code := minio.ToErrorResponse(err).Code; code == "NoSuchKey" || code == "NoSuchObject" {
|
||||
return fmt.Errorf("s3: %s: %w", op, attachment.ErrObjectMissing)
|
||||
}
|
||||
return fmt.Errorf("s3: %s: %w", op, err)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package s3store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) }))
|
||||
defer server.Close()
|
||||
if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("constructor performed network I/O")
|
||||
}
|
||||
for _, tc := range []struct{ name, endpoint, region, bucket, access, secret string }{
|
||||
{"scheme", "ftp://localhost:8333", "us-east-1", "attachments", "access", "secret"},
|
||||
{"path", "http://localhost:8333/path", "us-east-1", "attachments", "access", "secret"},
|
||||
{"userinfo", "http://user:password@localhost:8333", "us-east-1", "attachments", "access", "secret"},
|
||||
{"query", "http://localhost:8333?x=1", "us-east-1", "attachments", "access", "secret"},
|
||||
{"fragment", "http://localhost:8333/#x", "us-east-1", "attachments", "access", "secret"},
|
||||
{"region", server.URL, "", "attachments", "access", "secret"},
|
||||
{"bucket", server.URL, "us-east-1", "INVALID", "access", "secret"},
|
||||
{"access", server.URL, "us-east-1", "attachments", "", "secret"},
|
||||
{"secret", server.URL, "us-east-1", "attachments", "access", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil {
|
||||
t.Fatal("accepted invalid configuration")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) {
|
||||
first := make(chan struct{})
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
if r.Method != http.MethodPut || r.URL.Path != "/attachments/staging/stream" || r.URL.RawQuery != "" {
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL)
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
if r.ContentLength <= 0 || len(r.TransferEncoding) != 0 {
|
||||
t.Error("PUT is not a known-length single request")
|
||||
}
|
||||
b := make([]byte, 1)
|
||||
if _, err := io.ReadFull(r.Body, b); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
close(first)
|
||||
if _, err := io.Copy(io.Discard, r.Body); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", `"probe"`)
|
||||
}))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
// One signing chunk can be buffered; the remaining bytes wait for the server.
|
||||
r, w := io.Pipe()
|
||||
defer r.Close()
|
||||
go func() {
|
||||
defer w.Close()
|
||||
if _, err := w.Write(make([]byte, 256*1024)); err != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-first:
|
||||
case <-ctx.Done():
|
||||
_ = w.CloseWithError(ctx.Err())
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(make([]byte, 256*1024))
|
||||
}()
|
||||
if err := store.Put(ctx, "staging/stream", r, 512*1024, "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requests.Load() != 1 {
|
||||
t.Fatalf("got %d requests, want one PUT", requests.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) }))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for name, call := range map[string]func() error{
|
||||
"put": func() error { return store.Put(ctx, "key", strings.NewReader("x"), 1, "image/png") },
|
||||
"open": func() error {
|
||||
r, e := store.Open(ctx, "key")
|
||||
if r != nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
},
|
||||
"stat": func() error { _, e := store.Stat(ctx, "key"); return e },
|
||||
"copy": func() error { return store.Copy(ctx, "a", "b") },
|
||||
"delete": func() error { return store.Delete(ctx, "key") },
|
||||
"list": func() error { _, e := store.List(ctx, "", "", 1); return e },
|
||||
"bootstrap": func() error { return store.Bootstrap(ctx) },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := call(); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("got %v, want canceled", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := store.Put(context.Background(), "key", strings.NewReader("x"), -1, "image/png"); err == nil {
|
||||
t.Fatal("accepted unknown length")
|
||||
}
|
||||
for _, limit := range []int{-1, 0, 501, 1000000000} {
|
||||
if _, err := store.List(context.Background(), "", "", limit); err == nil {
|
||||
t.Fatalf("accepted unbounded page %d", limit)
|
||||
}
|
||||
}
|
||||
if requests.Load() != 0 {
|
||||
t.Fatalf("invalid calls reached server %d times", requests.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestListStopsAfterOneBoundedPage(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
if r.URL.Query().Get("start-after") != "blobs/a" || r.URL.Query().Get("prefix") != "blobs/" || r.URL.Query().Get("max-keys") != "2" {
|
||||
t.Errorf("wrong list query: %s", r.URL.RawQuery)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
io.WriteString(w, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>attachments</Name><IsTruncated>true</IsTruncated><NextContinuationToken>next-page</NextContinuationToken><Contents><Key>blobs/b</Key><Size>1</Size><LastModified>2026-01-01T00:00:00Z</LastModified></Contents><Contents><Key>blobs/c</Key><Size>2</Size><LastModified>2026-01-01T00:00:01Z</LastModified></Contents></ListBucketResult>`)
|
||||
}))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objects, err := store.List(context.Background(), "blobs/", "blobs/a", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objects) != 2 || objects[0].Key != "blobs/b" || objects[1].Key != "blobs/c" {
|
||||
t.Fatalf("wrong page: %+v", objects)
|
||||
}
|
||||
if requests.Load() != 1 {
|
||||
t.Fatalf("prefetched %d pages", requests.Load())
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,8 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler,
|
||||
srv := &http.Server{
|
||||
Handler: h,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
ReadTimeout: 35 * time.Second,
|
||||
WriteTimeout: 35 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
MaxHeaderBytes: 64 * 1024,
|
||||
ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn),
|
||||
|
||||
@@ -1,21 +1,198 @@
|
||||
package server_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"messenger/internal/health"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/httpx"
|
||||
"messenger/internal/server"
|
||||
)
|
||||
|
||||
func serverDefaultConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
values := make(map[string]string)
|
||||
for key, content := range map[string]string{
|
||||
"DATABASE_URL_FILE": "postgres://unused:unused@localhost:5432/unused",
|
||||
"SESSION_STORE_URL_FILE": "redis://localhost:6379/0",
|
||||
} {
|
||||
path := filepath.Join(t.TempDir(), "secret")
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values[key] = path
|
||||
}
|
||||
cfg, err := config.Load(func(key string) (string, bool) { value, ok := values[key]; return value, ok })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestDefaultShutdownAllowsActiveBinaryTransfers(t *testing.T) {
|
||||
for _, direction := range []string{"upload", "download"} {
|
||||
t.Run(direction, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := serverDefaultConfig(t)
|
||||
payload := []byte("binary\x00\xff\x01request-and-response\x00\xfe")
|
||||
prefix := payload[:3]
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
bodyReader, bodyWriter := io.Pipe()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
requestCtx, cancelRequest := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probes := health.New()
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
|
||||
if direction == "upload" {
|
||||
first := make([]byte, len(prefix))
|
||||
if _, err := io.ReadFull(r.Body, first); err != nil {
|
||||
return
|
||||
}
|
||||
close(started)
|
||||
rest, err := io.ReadAll(r.Body)
|
||||
if err != nil || !bytes.Equal(append(first, rest...), payload) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(payload)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(prefix)
|
||||
w.(http.Flusher).Flush()
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
_, _ = w.Write(payload[len(prefix):])
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
})
|
||||
stopped, exited := make(chan error, 1), make(chan struct{})
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
go func() {
|
||||
defer close(exited)
|
||||
stopped <- server.Run(ctx, log, ln, handler, probes, cfg.DrainDelay, cfg.ShutdownTimeout)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
cancelRequest()
|
||||
unblock()
|
||||
_ = bodyReader.Close()
|
||||
_ = bodyWriter.Close()
|
||||
_ = ln.Close()
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("server did not stop after transfer cleanup")
|
||||
}
|
||||
})
|
||||
client := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}}
|
||||
defer client.CloseIdleConnections()
|
||||
method := http.MethodGet
|
||||
var body io.Reader
|
||||
if direction == "upload" {
|
||||
method = http.MethodPut
|
||||
body = bodyReader
|
||||
}
|
||||
req, err := http.NewRequestWithContext(requestCtx, method, "http://"+ln.Addr().String()+"/binary", body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if direction == "upload" {
|
||||
req.ContentLength = int64(len(payload))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
}
|
||||
completed := make(chan error, 1)
|
||||
go func() {
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
completed <- err
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
got, err := io.ReadAll(resp.Body)
|
||||
if err == nil && (resp.StatusCode != http.StatusOK || !bytes.Equal(got, payload)) {
|
||||
err = fmt.Errorf("binary response status=%d body=%q, want exact payload", resp.StatusCode, got)
|
||||
}
|
||||
completed <- err
|
||||
}()
|
||||
if direction == "upload" {
|
||||
if _, err := bodyWriter.Write(prefix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case err := <-completed:
|
||||
t.Fatalf("transfer failed before shutdown: %v", err)
|
||||
case <-requestCtx.Done():
|
||||
t.Fatal("transfer never became active")
|
||||
}
|
||||
cancel()
|
||||
// Nine seconds inside Shutdown exceeds the former 8s budget. Include
|
||||
// the configured drain phase so it cannot hide a premature force-close.
|
||||
timer := time.NewTimer(cfg.DrainDelay + 9*time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case err := <-stopped:
|
||||
t.Fatalf("shutdown interrupted active %s before binary transfer completed: %v", direction, err)
|
||||
case err := <-completed:
|
||||
t.Fatalf("active %s ended before remaining bytes were released: %v", direction, err)
|
||||
case <-timer.C:
|
||||
}
|
||||
if direction == "upload" {
|
||||
if _, err := bodyWriter.Write(payload[len(prefix):]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := bodyWriter.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
unblock()
|
||||
}
|
||||
select {
|
||||
case err := <-completed:
|
||||
if err != nil {
|
||||
t.Fatalf("active %s was truncated during shutdown: %v", direction, err)
|
||||
}
|
||||
case <-requestCtx.Done():
|
||||
t.Fatal("binary transfer did not finish after release")
|
||||
}
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("graceful shutdown: %v", err)
|
||||
}
|
||||
case <-requestCtx.Done():
|
||||
t.Fatal("server did not finish graceful shutdown")
|
||||
}
|
||||
if probes.IsReady() {
|
||||
t.Fatal("readiness remained enabled after shutdown")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newHandler builds the smallest handler with probes; the lifecycle does not
|
||||
// depend on the business modules or their stores.
|
||||
func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) {
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package testenv
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/go-connections/nat"
|
||||
"github.com/google/uuid"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
const S3Image = "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0"
|
||||
|
||||
// S3 keeps independent API, reconciler and bootstrap identities. Store uses the
|
||||
// reconciler identity so tests can inspect and clean up their objects.
|
||||
type S3 struct {
|
||||
Store, APIStore, Admin *s3store.Store
|
||||
Endpoint, Region, Bucket string
|
||||
GRPCAddress string
|
||||
AccessKey, SecretKey string
|
||||
APIAccessKey, APISecretKey string
|
||||
AdminAccessKey, AdminSecretKey string
|
||||
container testcontainers.Container
|
||||
volume string
|
||||
identities, options []byte
|
||||
}
|
||||
|
||||
func StartS3(ctx context.Context) (*S3, error) {
|
||||
useDockerDesktopPipe()
|
||||
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()}
|
||||
keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey}
|
||||
for _, key := range keys {
|
||||
b := make([]byte, 24)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
*key = hex.EncodeToString(b)
|
||||
}
|
||||
type credential struct {
|
||||
AccessKey string `json:"accessKey"`
|
||||
SecretKey string `json:"secretKey"`
|
||||
}
|
||||
type identity struct {
|
||||
Name string `json:"name"`
|
||||
Credentials []credential `json:"credentials"`
|
||||
Actions []string `json:"actions"`
|
||||
}
|
||||
config := struct {
|
||||
Identities []identity `json:"identities"`
|
||||
}{Identities: []identity{
|
||||
{"bootstrap", []credential{{s.AdminAccessKey, s.AdminSecretKey}}, []string{"Admin", "Read", "Write", "List"}},
|
||||
{"api", []credential{{s.APIAccessKey, s.APISecretKey}}, []string{"Read:" + s.Bucket, "Write:" + s.Bucket}},
|
||||
{"reconciler", []credential{{s.AccessKey, s.SecretKey}}, []string{"List:" + s.Bucket, "Read:" + s.Bucket, "Write:" + s.Bucket}},
|
||||
}}
|
||||
var err error
|
||||
s.identities, err = json.Marshal(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, source, _, _ := runtime.Caller(0)
|
||||
s.options, err = os.ReadFile(filepath.Join(filepath.Dir(source), "../../infra/s3/seaweedfs.options"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read S3 options: %w", err)
|
||||
}
|
||||
startCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
defer cancel()
|
||||
if err := s.launch(startCtx); err != nil {
|
||||
s.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.Admin.Bootstrap(startCtx); err != nil {
|
||||
s.Close()
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3) launch(ctx context.Context) error {
|
||||
pids := int64(256)
|
||||
firstStatus := 0
|
||||
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: S3Image,
|
||||
Cmd: []string{"server", "-options=/etc/seaweedfs/lab.options"},
|
||||
Env: map[string]string{"GOMEMLIMIT": "460MiB"},
|
||||
ExposedPorts: []string{"8333/tcp", "18333/tcp"},
|
||||
Mounts: testcontainers.Mounts(testcontainers.VolumeMount(s.volume, "/data")),
|
||||
Files: []testcontainers.ContainerFile{
|
||||
{Reader: bytes.NewReader(s.identities), ContainerFilePath: "/run/secrets/s3_identities", FileMode: 0444},
|
||||
{Reader: bytes.NewReader(s.options), ContainerFilePath: "/etc/seaweedfs/lab.options", FileMode: 0444},
|
||||
},
|
||||
HostConfigModifier: func(h *container.HostConfig) {
|
||||
h.Resources.Memory = 512 * 1024 * 1024
|
||||
h.Resources.NanoCPUs = 1_000_000_000
|
||||
h.Resources.PidsLimit = &pids
|
||||
h.PortBindings = nat.PortMap{"8333/tcp": {{HostIP: "127.0.0.1"}}, "18333/tcp": {{HostIP: "127.0.0.1"}}}
|
||||
},
|
||||
// The first reachable S3 response must already reject anonymous access.
|
||||
WaitingFor: wait.ForHTTP("/").WithPort("8333/tcp").WithStatusCodeMatcher(func(code int) bool { firstStatus = code; return true }).WithStartupTimeout(60 * time.Second),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
s.container = c
|
||||
if err != nil {
|
||||
return fmt.Errorf("start S3: %w", err)
|
||||
}
|
||||
if firstStatus != http.StatusForbidden {
|
||||
return fmt.Errorf("first anonymous S3 response was %d, expected 403", firstStatus)
|
||||
}
|
||||
s.Endpoint, err = c.PortEndpoint(ctx, "8333/tcp", "http")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.GRPCAddress, err = c.PortEndpoint(ctx, "18333/tcp", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3) Stop(ctx context.Context) error {
|
||||
timeout := 5 * time.Second
|
||||
return s.container.Stop(ctx, &timeout)
|
||||
}
|
||||
|
||||
func (s *S3) Start(ctx context.Context) error {
|
||||
if err := s.container.Start(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Admin.WaitReady(ctx)
|
||||
}
|
||||
|
||||
// Recreate replaces the container while keeping only its named /data volume.
|
||||
// Bootstrap is deliberately not called: bucket metadata must survive as well.
|
||||
func (s *S3) Recreate(ctx context.Context) error {
|
||||
if err := testcontainers.TerminateContainer(s.container); err != nil {
|
||||
return err
|
||||
}
|
||||
s.container = nil
|
||||
if err := s.launch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Admin.WaitReady(ctx)
|
||||
}
|
||||
|
||||
// Diagnostics retains bounded startup evidence without exposing fixture keys.
|
||||
func (s *S3) Diagnostics(ctx context.Context) string {
|
||||
if s.container == nil {
|
||||
return "no container"
|
||||
}
|
||||
r, err := s.container.Logs(ctx)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
defer r.Close()
|
||||
redact := strings.NewReplacer(s.AccessKey, "<redacted>", s.SecretKey, "<redacted>", s.APIAccessKey, "<redacted>", s.APISecretKey, "<redacted>", s.AdminAccessKey, "<redacted>", s.AdminSecretKey, "<redacted>")
|
||||
var lines []string
|
||||
scan := bufio.NewScanner(io.LimitReader(r, 256*1024))
|
||||
for scan.Scan() {
|
||||
line := scan.Text()
|
||||
lower := strings.ToLower(line)
|
||||
if strings.Contains(lower, "volume") || strings.Contains(lower, "master") || strings.Contains(lower, "leader") || strings.Contains(lower, "error") || strings.Contains(lower, "lookup") {
|
||||
lines = append(lines, redact.Replace(line))
|
||||
if len(lines) > 60 {
|
||||
lines = lines[1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func (s *S3) Close() {
|
||||
if s.container != nil {
|
||||
_ = testcontainers.TerminateContainer(s.container)
|
||||
}
|
||||
if strings.HasPrefix(s.volume, "lab4-s3-test-") {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE files (
|
||||
hash bytea PRIMARY KEY CHECK (octet_length(hash) = 32),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
mime_type text NOT NULL CHECK (mime_type IN ('image/png', 'image/jpeg')),
|
||||
status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL,
|
||||
file_hash bytea NOT NULL REFERENCES files(hash),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
-- The reconciler must distinguish a shared file from an unused file.
|
||||
CREATE INDEX attachments_file_hash_idx ON attachments(file_hash);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
chat_id uuid NOT NULL,
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(chat_id, message_id, attachment_id),
|
||||
UNIQUE(chat_id, message_id, ordinal),
|
||||
FOREIGN KEY(chat_id, message_id) REFERENCES messages(chat_id, id)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE message_attachments;
|
||||
DROP TABLE attachments;
|
||||
DROP TABLE files;
|
||||
+67
-13
@@ -14,11 +14,7 @@ for f in $files; do
|
||||
present=$((present + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$present" -eq 4 ]; then
|
||||
echo "secrets already exist in ./secrets, kept"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$present" -ne 0 ]; then
|
||||
if [ "$present" -ne 0 ] && [ "$present" -ne 4 ]; then
|
||||
echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -27,16 +23,74 @@ fi
|
||||
random() {
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
|
||||
}
|
||||
pg=$(random)
|
||||
kv=$(random)
|
||||
|
||||
# The directory is private; the files stay readable for the unprivileged
|
||||
# users inside the containers, which see them through bind mounts.
|
||||
umask 022
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
printf '%s\n' "$pg" > secrets/postgres_password
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url
|
||||
printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf
|
||||
printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url
|
||||
echo "created ./secrets"
|
||||
if [ "$present" -eq 0 ]; then
|
||||
pg=$(random)
|
||||
kv=$(random)
|
||||
printf '%s\n' "$pg" > secrets/postgres_password
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url
|
||||
printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf
|
||||
printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url
|
||||
fi
|
||||
|
||||
# Read just the bucket setting, never execute the local .env file as a script.
|
||||
bucket=${S3_BUCKET:-}
|
||||
if [ -z "$bucket" ] && [ -f .env ]; then
|
||||
bucket=$(sed -n 's/^S3_BUCKET=//p' .env | tail -n 1 | tr -d '\r')
|
||||
fi
|
||||
bucket=${bucket:-messenger-attachments}
|
||||
case "$bucket" in
|
||||
*[!a-z0-9.-]*|.*|*.|-*|*-|*..*|*.-*|*-.*)
|
||||
echo "S3_BUCKET must be a lowercase S3 bucket name" >&2; exit 1 ;;
|
||||
esac
|
||||
if [ "${#bucket}" -lt 3 ] || [ "${#bucket}" -gt 63 ]; then
|
||||
echo "S3_BUCKET must contain 3 to 63 characters" >&2; exit 1
|
||||
fi
|
||||
|
||||
s3files="s3_admin_access_key_id s3_admin_secret_access_key s3_access_key_id s3_secret_access_key s3_reconcile_access_key_id s3_reconcile_secret_access_key s3_identities.json"
|
||||
s3present=0
|
||||
for f in $s3files; do
|
||||
if [ -s "secrets/$f" ]; then s3present=$((s3present + 1)); fi
|
||||
done
|
||||
if [ "$s3present" -eq 7 ]; then
|
||||
if ! grep -Fq "\"Read:$bucket\"" secrets/s3_identities.json; then
|
||||
echo "existing S3 identities use another bucket; keep S3_BUCKET consistent with the retained volume" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "existing PostgreSQL, Valkey and S3 secrets kept"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$s3present" -ne 0 ]; then
|
||||
echo "S3 secrets are incomplete; restore the s3_* secret files together; PostgreSQL and Valkey secrets were kept" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
admin_key=$(random)
|
||||
admin_secret=$(random)
|
||||
api_key=$(random)
|
||||
api_secret=$(random)
|
||||
recon_key=$(random)
|
||||
recon_secret=$(random)
|
||||
printf '%s\n' "$admin_key" > secrets/s3_admin_access_key_id
|
||||
printf '%s\n' "$admin_secret" > secrets/s3_admin_secret_access_key
|
||||
printf '%s\n' "$api_key" > secrets/s3_access_key_id
|
||||
printf '%s\n' "$api_secret" > secrets/s3_secret_access_key
|
||||
printf '%s\n' "$recon_key" > secrets/s3_reconcile_access_key_id
|
||||
printf '%s\n' "$recon_secret" > secrets/s3_reconcile_secret_access_key
|
||||
|
||||
# The server loads all identities before listening. There is no anonymous
|
||||
# identity, and no post-start allow-all provisioning phase.
|
||||
cat > secrets/s3_identities.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{"name":"bootstrap","credentials":[{"accessKey":"$admin_key","secretKey":"$admin_secret"}],"actions":["Admin","Read","Write","List"]},
|
||||
{"name":"api","credentials":[{"accessKey":"$api_key","secretKey":"$api_secret"}],"actions":["Read:$bucket","Write:$bucket"]},
|
||||
{"name":"reconciler","credentials":[{"accessKey":"$recon_key","secretKey":"$recon_secret"}],"actions":["List:$bucket","Read:$bucket","Write:$bucket"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
echo "PostgreSQL and Valkey secrets kept or created; S3 identities created"
|
||||
|
||||
@@ -4,4 +4,9 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop
|
||||
# Re-run one-shot jobs against the retained data before restoring replicas.
|
||||
docker compose up -d --wait postgres valkey s3
|
||||
docker compose run --rm migrate
|
||||
docker compose run --rm s3-init
|
||||
docker compose up -d --wait
|
||||
sh scripts/wait-ready.sh
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Compose may retain the balancer's earlier healthy status while API backends
|
||||
# are recreated. Wait for fresh successful responses through the real LB.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
attempt=0
|
||||
stable=0
|
||||
while [ "$attempt" -lt 60 ]; do
|
||||
if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then
|
||||
stable=$((stable + 1))
|
||||
if [ "$stable" -ge 3 ]; then
|
||||
echo "balancer readiness confirmed"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
stable=0
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "balancer did not become ready within 60 probes" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user