From efaeb593e8aec6fb7abaf4d236205a6106f40cf8 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 2 Oct 2026 19:41:26 +0300 Subject: [PATCH] feat: add S3 attachments and reconciliation --- cmd/messenger/main.go | 85 ++++++- go.mod | 14 +- go.sum | 25 +- internal/config/config.go | 117 +++++++-- internal/messenger/attachments.go | 260 ++++++++++++++++++++ internal/messenger/errors.go | 1 + internal/messenger/handlers.go | 36 ++- internal/messenger/messenger_interface.go | 13 +- internal/messenger/metrics.go | 41 ++- internal/messenger/model.go | 1 + internal/messenger/routes.go | 6 + internal/messenger/server.go | 16 +- internal/messenger/server_test.go | 221 ++++++++++++++++- internal/model/model.go | 79 +++++- internal/objectstore/store.go | 180 ++++++++++++++ internal/postgres/attachments.go | 155 ++++++++++++ internal/postgres/idempotency.go | 78 +++++- internal/postgres/store.go | 17 +- internal/postgres/store_integration_test.go | 80 +++++- internal/reconcile/reconcile.go | 236 ++++++++++++++++++ migrations/0003_attachments.sql | 47 ++++ 21 files changed, 1623 insertions(+), 85 deletions(-) create mode 100644 internal/messenger/attachments.go create mode 100644 internal/objectstore/store.go create mode 100644 internal/postgres/attachments.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 migrations/0003_attachments.sql diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 76ed2d1..90f25b6 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -2,20 +2,26 @@ package main import ( "context" + "encoding/json" + "flag" "fmt" + "io" "log/slog" "net" "net/http" "os" "os/signal" "syscall" + "time" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session" ) @@ -34,7 +40,10 @@ func run(logger *slog.Logger, arguments []string) error { if len(arguments) == 1 && arguments[0] == "migrate" { return runMigrations() } - return fmt.Errorf("usage: messenger [migrate]") + if len(arguments) > 0 && arguments[0] == "reconcile-attachments" { + return runAttachmentReconciliation(arguments[1:], os.Stdout) + } + return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]") } func runServer(logger *slog.Logger) error { @@ -68,16 +77,28 @@ func runServer(logger *slog.Logger) error { if err != nil { return err } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } app := messenger.NewServer( logger, postgres.New(pool), sessions, passwords, + objects, settings.SessionTTL, settings.IdempotencyTTL, settings.RequestTimeout, instanceID, settings.SwaggerPort, + settings.AttachmentMaxBytes, + settings.AttachmentMaxPixels, + settings.AttachmentDecodeMemory, ) server := &http.Server{ Addr: settings.Address, @@ -130,3 +151,65 @@ func runMigrations() error { defer pool.Close() return database.Migrate(context.Background(), pool, settings.MigrationTimeout) } + +type reconciliationOutput struct { + Mode string `json:"mode"` + GracePeriod string `json:"grace_period"` + PageSize int `json:"page_size"` + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func runAttachmentReconciliation(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) + flags.SetOutput(io.Discard) + apply := flags.Bool("apply", false, "apply safe repairs") + gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") + pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]") + } + + settings, err := config.Load() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply, + }) + if err != nil { + return err + } + started := time.Now() + report, err := reconciler.Run(ctx) + if err != nil { + return err + } + mode := "dry-run" + if *apply { + mode = "apply" + } + return json.NewEncoder(output).Encode(reconciliationOutput{ + Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize, + DurationMillis: time.Since(started).Milliseconds(), Report: report, + }) +} diff --git a/go.mod b/go.mod index f498f1b..b3de9ff 100644 --- a/go.mod +++ b/go.mod @@ -5,37 +5,49 @@ go 1.26.0 require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.0.99 github.com/prometheus/client_golang v1.23.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/sync v0.23.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect google.golang.org/protobuf v1.36.8 // indirect diff --git a/go.sum b/go.sum index e55a574..5aa456c 100644 --- a/go.sum +++ b/go.sum @@ -5,10 +5,14 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -23,20 +27,33 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -49,6 +66,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= @@ -68,6 +87,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/internal/config/config.go b/internal/config/config.go index 4457141..2fcfab3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,29 +12,39 @@ import ( ) type Config struct { - Address string - DatabaseURL string - CacheURL string - SessionTTL time.Duration - IdempotencyTTL time.Duration - DBMaxConns int32 - DBMinConns int32 - DBMaxConnectionAge time.Duration - DBMaxConnectionIdle time.Duration - DBHealthCheckPeriod time.Duration - DependencyTimeout time.Duration - RequestTimeout time.Duration - HTTPReadHeaderTimeout time.Duration - HTTPReadTimeout time.Duration - HTTPWriteTimeout time.Duration - HTTPIdleTimeout time.Duration - HTTPShutdownTimeout time.Duration - PasswordMemoryKiB uint32 - PasswordIterations uint32 - PasswordParallelism uint8 - PasswordSaltLength uint32 - PasswordKeyLength uint32 - SwaggerPort string + Address string + DatabaseURL string + CacheURL string + SessionTTL time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentDecodeMemory int64 } type MigrationConfig struct { @@ -142,6 +152,50 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + s3Endpoint, err := requiredString(values, "S3_ENDPOINT") + if err != nil { + return Config{}, err + } + s3Region, err := requiredString(values, "S3_REGION") + if err != nil { + return Config{}, err + } + s3Bucket, err := requiredString(values, "S3_BUCKET") + if err != nil { + return Config{}, err + } + s3AccessKey, err := requiredString(values, "S3_ACCESS_KEY") + if err != nil { + return Config{}, err + } + s3SecretKey, err := requiredString(values, "S3_SECRET_KEY") + if err != nil { + return Config{}, err + } + s3UseTLS, err := boolean(values, "S3_USE_TLS") + if err != nil { + return Config{}, err + } + s3Timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return Config{}, err + } + attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + if err != nil { + return Config{}, err + } + attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + if err != nil { + return Config{}, err + } + attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) + if err != nil { + return Config{}, err + } + minimumDecodeMemory := int64(attachmentMaxBytes) + int64(attachmentMaxPixels)*8 + if int64(attachmentDecodeMemory) < minimumDecodeMemory { + return Config{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + } return Config{ Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, @@ -155,7 +209,10 @@ func Load() (Config, error) { HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), - SwaggerPort: swaggerPort, + SwaggerPort: swaggerPort, S3Endpoint: s3Endpoint, S3Region: s3Region, S3Bucket: s3Bucket, + S3AccessKey: s3AccessKey, S3SecretKey: s3SecretKey, S3UseTLS: s3UseTLS, S3Timeout: s3Timeout, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + AttachmentDecodeMemory: int64(attachmentDecodeMemory), }, nil } @@ -260,3 +317,15 @@ func integerValue(value string, minimum, maximum int) (int, error) { } return parsed, nil } + +func boolean(values *viper.Viper, name string) (bool, error) { + value, err := requiredString(values, name) + if err != nil { + return false, err + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s: must be true or false", name) + } + return parsed, nil +} diff --git a/internal/messenger/attachments.go b/internal/messenger/attachments.go new file mode 100644 index 0000000..f69a506 --- /dev/null +++ b/internal/messenger/attachments.go @@ -0,0 +1,260 @@ +package messenger + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" +) + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { + started := time.Now() + result := "client_error" + defer func() { + s.metrics.attachmentUploads.WithLabelValues(result).Inc() + s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) + }() + + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + filename, valid := attachmentFilename(r.URL.Query()) + if !valid { + return invalidRequest() + } + if _, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID); errors.Is(err, postgres.ErrNotFound) { + return notFound() + } else if err != nil { + result = "dependency_error" + return unavailable() + } + data, contentType, readErr := readImage(w, r, s.maxUploadBytes, s.maxImagePixels, s.decodeMemory) + if readErr != nil { + return readErr + } + + attachmentID, err := newID() + if err != nil { + result = "dependency_error" + return internalError() + } + objectID, err := newID() + if err != nil { + result = "dependency_error" + return internalError() + } + objectKey := "attachments/" + objectID + "/original" + if err := s.objects.Put(r.Context(), objectKey, contentType, data); err != nil { + result = "dependency_error" + return unavailable() + } + digest := sha256.Sum256(data) + attachment, err := s.messengerStore.CreateAttachment(r.Context(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: filename, + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: objectKey, + }) + if err != nil { + result = "dependency_error" + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.requestTTL) + defer cancel() + stored, lookupErr := s.messengerStore.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) + if lookupErr == nil { + result = "success" + s.metrics.attachmentBytes.Add(float64(len(data))) + writeJSON(w, http.StatusCreated, stored) + return nil + } + if !errors.Is(lookupErr, postgres.ErrNotFound) { + s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", lookupErr) + return unavailable() + } + if cleanupErr := s.objects.Delete(cleanupCtx, objectKey); cleanupErr != nil { + s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", cleanupErr) + } + if errors.Is(err, postgres.ErrNotFound) { + return notFound() + } + return unavailable() + } + result = "success" + s.metrics.attachmentBytes.Add(float64(len(data))) + writeJSON(w, http.StatusCreated, attachment) + return nil +} + +func attachmentFilename(query url.Values) (string, bool) { + if len(query) != 1 || len(query["filename"]) != 1 { + return "", false + } + filename := query["filename"][0] + length := utf8.RuneCountInString(filename) + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { + return "", false + } + for _, char := range filename { + if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { + return "", false + } + } + return filename, true +} + +func readImage( + w http.ResponseWriter, + r *http.Request, + maxBytes int64, + maxPixels int64, + decodeMemory *semaphore.Weighted, +) ([]byte, string, *apiError) { + mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { + return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} + } + if r.ContentLength > maxBytes { + return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + } + r.Body = http.MaxBytesReader(w, r.Body, maxBytes) + data, err := io.ReadAll(r.Body) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + } + return nil, "", invalidRequest() + } + if len(data) == 0 { + return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} + } + detected := http.DetectContentType(data) + if detected != mediaType { + return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "content type does not match image"} + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || imageContentType(format) != mediaType { + return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} + } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) + if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > maxPixels { + return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image dimensions are too large"} + } + decodeBytes := pixels*8 + int64(len(data)) + if !decodeMemory.TryAcquire(decodeBytes) { + w.Header().Set("Retry-After", "1") + return nil, "", &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + } + defer decodeMemory.Release(decodeBytes) + if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { + return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} + } + return data, mediaType, nil +} + +func imageContentType(format string) string { + switch format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + default: + return "" + } +} + +func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + attachment, apiErr := s.attachmentForUser(r, user.ID) + if apiErr != nil { + return apiErr + } + writeJSON(w, http.StatusOK, attachment) + return nil +} + +func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if !originalVariant(r.URL.Query()) { + return invalidRequest() + } + attachment, apiErr := s.attachmentForUser(r, user.ID) + if apiErr != nil { + return apiErr + } + content, err := s.objects.Get(r.Context(), attachment.ObjectKey) + if errors.Is(err, objectstore.ErrNotFound) { + return unavailable() + } + if err != nil { + return unavailable() + } + defer func() { _ = content.Close() }() + w.Header().Set("Content-Type", attachment.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(attachment.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename})) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, content); err != nil { + s.logger.Warn("attachment download interrupted", "attachment_id", attachment.ID, "error", err) + } + return nil +} + +func originalVariant(query url.Values) bool { + if len(query) == 0 { + return true + } + return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" +} + +func (s *Server) attachmentForUser(r *http.Request, userID string) (Attachment, *apiError) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + attachmentID, valid := canonicalID(parts[3]) + if !valid { + return Attachment{}, invalidRequest() + } + attachment, err := s.messengerStore.AttachmentForUser(r.Context(), attachmentID, userID) + if errors.Is(err, postgres.ErrNotFound) { + return Attachment{}, notFound() + } + if err != nil { + return Attachment{}, unavailable() + } + return attachment, nil +} diff --git a/internal/messenger/errors.go b/internal/messenger/errors.go index 88d2970..43923b7 100644 --- a/internal/messenger/errors.go +++ b/internal/messenger/errors.go @@ -14,6 +14,7 @@ const ( errorCodePayloadTooLarge = "payload_too_large" errorCodeUnavailable = "unavailable" errorCodeConflict = "conflict" + errorCodeRateLimited = "rate_limited" ) type apiError struct { diff --git a/internal/messenger/handlers.go b/internal/messenger/handlers.go index bedff65..29ec395 100644 --- a/internal/messenger/handlers.go +++ b/internal/messenger/handlers.go @@ -38,6 +38,9 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { if err := s.sessions.Ping(ctx); err != nil { return unavailable() } + if err := s.objects.Ping(ctx); err != nil { + return unavailable() + } writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) return nil } @@ -207,16 +210,17 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { return invalidRequest() } var input struct { - Text string `json:"text"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` } if err := readJSON(w, r, &input); err != nil { return err } - if !validText(input.Text, 4000) { + if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) { return invalidRequest() } message, replay, err := s.messengerStore.CreateMessage( - r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL, + r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL, ) if errors.Is(err, postgres.ErrNotFound) { return notFound() @@ -234,6 +238,32 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { return nil } +func validMessage(text string, attachmentIDs []string) bool { + if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 { + return false + } + return len(attachmentIDs) > 0 || strings.TrimSpace(text) != "" +} + +func canonicalAttachmentIDs(attachmentIDs []string) bool { + if len(attachmentIDs) > 10 { + return false + } + seen := make(map[string]struct{}, len(attachmentIDs)) + for index, attachmentID := range attachmentIDs { + canonical, valid := canonicalID(attachmentID) + if !valid { + return false + } + if _, duplicate := seen[canonical]; duplicate { + return false + } + seen[canonical] = struct{}{} + attachmentIDs[index] = canonical + } + return true +} + func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError { limit, cursor, err := pageParams(r, "message", chatID) if err != nil { diff --git a/internal/messenger/messenger_interface.go b/internal/messenger/messenger_interface.go index 14005ad..682a10f 100644 --- a/internal/messenger/messenger_interface.go +++ b/internal/messenger/messenger_interface.go @@ -2,6 +2,7 @@ package messenger import ( "context" + "io" "time" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" @@ -15,8 +16,18 @@ type MessengerStore interface { CreateChat(context.Context, string, []string, string) (model.Chat, error) ChatForUser(context.Context, string, string) (model.Chat, error) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) - CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) + CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) + CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) + AttachmentByStorage(context.Context, string, string) (model.Attachment, error) + AttachmentForUser(context.Context, string, string) (model.Attachment, error) +} + +type ObjectStore interface { + Ping(context.Context) error + Put(context.Context, string, string, []byte) error + Get(context.Context, string) (io.ReadCloser, error) + Delete(context.Context, string) error } type SessionStore interface { diff --git a/internal/messenger/metrics.go b/internal/messenger/metrics.go index 8073542..267d4b7 100644 --- a/internal/messenger/metrics.go +++ b/internal/messenger/metrics.go @@ -12,11 +12,14 @@ import ( ) type metrics struct { - requests *prometheus.CounterVec - duration *prometheus.HistogramVec - inFlight prometheus.Gauge - replays prometheus.Counter - handler http.Handler + requests *prometheus.CounterVec + duration *prometheus.HistogramVec + inFlight prometheus.Gauge + replays prometheus.Counter + attachmentUploads *prometheus.CounterVec + attachmentDuration prometheus.Histogram + attachmentBytes prometheus.Counter + handler http.Handler } func newMetrics() *metrics { @@ -47,12 +50,31 @@ func newMetrics() *metrics { Name: "idempotency_replays_total", Help: "Message requests served from an idempotency record.", }), + attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "messenger", + Name: "attachment_uploads_total", + Help: "Completed attachment uploads by result.", + }, []string{"result"}), + attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "messenger", + Name: "attachment_upload_duration_seconds", + Help: "Attachment upload duration in seconds.", + Buckets: prometheus.DefBuckets, + }), + attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", + Name: "attachment_upload_bytes_total", + Help: "Bytes accepted by successful attachment uploads.", + }), } registry.MustRegister( result.requests, result.duration, result.inFlight, result.replays, + result.attachmentUploads, + result.attachmentDuration, + result.attachmentBytes, collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), ) @@ -79,5 +101,14 @@ func routeTemplate(path string) string { if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { return "/api/v1/chats/{chat_id}/messages" } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + return "/api/v1/chats/{chat_id}/attachments" + } + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + return "/api/v1/attachments/{attachment_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + return "/api/v1/attachments/{attachment_id}/content" + } return "unknown" } diff --git a/internal/messenger/model.go b/internal/messenger/model.go index ca5eefa..f1509b2 100644 --- a/internal/messenger/model.go +++ b/internal/messenger/model.go @@ -8,6 +8,7 @@ import ( type User = model.User type Chat = model.Chat type Message = model.Message +type Attachment = model.Attachment func newID() (string, error) { id, err := uuid.NewRandom() diff --git a/internal/messenger/routes.go b/internal/messenger/routes.go index bb698d0..25821e7 100644 --- a/internal/messenger/routes.go +++ b/internal/messenger/routes.go @@ -33,6 +33,12 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError { handler = s.chat } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { handler = s.messages + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + handler = s.uploadAttachment + } else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + handler = s.attachmentMetadata + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + handler = s.attachmentContent } else { return notFound() } diff --git a/internal/messenger/server.go b/internal/messenger/server.go index 3cc2ba0..7d65445 100644 --- a/internal/messenger/server.go +++ b/internal/messenger/server.go @@ -6,18 +6,24 @@ import ( "net/http" "sync/atomic" "time" + + "golang.org/x/sync/semaphore" ) type Server struct { messengerStore MessengerStore sessions SessionStore passwords PasswordService + objects ObjectStore sessionTTL time.Duration idempotencyTTL time.Duration requestTTL time.Duration instanceID string logger *slog.Logger swaggerPort string + maxUploadBytes int64 + maxImagePixels int64 + decodeMemory *semaphore.Weighted metrics *metrics ready atomic.Bool } @@ -37,16 +43,22 @@ func NewServer( messages MessengerStore, sessions SessionStore, passwords PasswordService, + objects ObjectStore, sessionTTL time.Duration, idempotencyTTL time.Duration, requestTTL time.Duration, instanceID string, swaggerPort string, + maxUploadBytes int64, + maxImagePixels int64, + decodeMemoryBytes int64, ) *Server { s := &Server{ - messengerStore: messages, sessions: sessions, passwords: passwords, + messengerStore: messages, sessions: sessions, passwords: passwords, objects: objects, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL, - instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(), + instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, + maxUploadBytes: maxUploadBytes, maxImagePixels: maxImagePixels, + decodeMemory: semaphore.NewWeighted(decodeMemoryBytes), metrics: newMetrics(), } s.ready.Store(true) return s diff --git a/internal/messenger/server_test.go b/internal/messenger/server_test.go index c38aa0c..3440240 100644 --- a/internal/messenger/server_test.go +++ b/internal/messenger/server_test.go @@ -1,9 +1,13 @@ package messenger import ( + "bytes" "context" "errors" "fmt" + "image" + "image/color" + "image/png" "io" "log/slog" "net/http" @@ -13,6 +17,7 @@ import ( "time" "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" @@ -34,14 +39,19 @@ func (s *MessengerSuite) SetupTest() { } type messengerStoreStub struct { - pingErr error - userByID model.User - userByIDErr error - credentials model.UserCredentials - credentialsErr error - message model.Message - messageReplay bool - messageErr error + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error + attachment model.Attachment + attachmentErr error + storedAttachment model.Attachment + storedAttachmentExists bool + storedAttachmentErr error } func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } @@ -63,12 +73,57 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) { return nil, false, nil } -func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) { +func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) { return s.message, s.messageReplay, s.messageErr } func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { return nil, false, nil } +func (s messengerStoreStub) CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) { + return s.attachment, s.attachmentErr +} +func (s messengerStoreStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { + if s.storedAttachmentErr != nil { + return model.Attachment{}, s.storedAttachmentErr + } + if !s.storedAttachmentExists { + return model.Attachment{}, postgres.ErrNotFound + } + return s.storedAttachment, nil +} +func (s messengerStoreStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { + return s.attachment, s.attachmentErr +} + +type objectStoreStub struct { + pingErr error + putErr error + getData []byte + getErr error + deleteErr error + putKey *string + deletedKey *string +} + +func (s objectStoreStub) Ping(context.Context) error { return s.pingErr } +func (s objectStoreStub) Put(_ context.Context, key string, _ string, _ []byte) error { + if s.putKey != nil { + *s.putKey = key + } + return s.putErr +} +func (s objectStoreStub) Get(context.Context, string) (io.ReadCloser, error) { + if s.getErr != nil { + return nil, s.getErr + } + return io.NopCloser(bytes.NewReader(s.getData)), nil +} +func (s objectStoreStub) Delete(_ context.Context, key string) error { + if s.deletedKey != nil { + *s.deletedKey = key + } + return s.deleteErr +} type sessionStub struct { pingErr error @@ -100,14 +155,19 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { name string store messengerStoreStub sessions sessionStub + objects objectStoreStub }{ {name: "postgres", store: messengerStoreStub{pingErr: errDependency}}, {name: "valkey", sessions: sessionStub{pingErr: errDependency}}, + {name: "s3", objects: objectStoreStub{pingErr: errDependency}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) + server := NewServer( + s.logger, tt.store, tt.sessions, passwordsStub{}, tt.objects, time.Hour, 24*time.Hour, + time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, + ) response := serveRequest(server, http.MethodGet, "/health/ready", "") if response.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) @@ -160,6 +220,18 @@ func (s *MessengerSuite) TestAuthenticationChallenges() { } } +func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest( + server, + http.MethodGet, + "/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid", + "", + ) + s.Equal(http.StatusUnauthorized, response.Code) + s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate")) +} + func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() { t := s.T() server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) @@ -242,8 +314,135 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe } } +func (s *MessengerSuite) TestImageValidation() { + data := testPNG(s.T(), 2, 2) + tests := []struct { + name string + contentType string + data []byte + maxBytes int64 + maxPixels int64 + wantStatus int + }{ + {name: "valid", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 4}, + {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType}, + {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, wantStatus: http.StatusRequestEntityTooLarge}, + {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, wantStatus: http.StatusRequestEntityTooLarge}, + {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(tt.data)) + request.Header.Set("Content-Type", tt.contentType) + response := httptest.NewRecorder() + actual, _, apiErr := readImage( + response, request, tt.maxBytes, tt.maxPixels, semaphore.NewWeighted(384*1024*1024), + ) + if tt.wantStatus == 0 { + s.Require().Nil(apiErr) + s.Equal(tt.data, actual) + return + } + s.Require().NotNil(apiErr) + s.Equal(tt.wantStatus, apiErr.status) + }) + } +} + +func (s *MessengerSuite) TestImageValidationLimitsConcurrentDecodeMemory() { + data := testPNG(s.T(), 2, 2) + request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(data)) + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + _, _, apiErr := readImage(response, request, 1024, 4, semaphore.NewWeighted(1)) + s.Require().NotNil(apiErr) + s.Equal(http.StatusTooManyRequests, apiErr.status) + s.Equal("1", response.Header().Get("Retry-After")) +} + +func (s *MessengerSuite) TestAttachmentFilenameValidation() { + for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} { + _, valid := attachmentFilename(map[string][]string{"filename": {filename}}) + s.False(valid, filename) + } + filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}}) + s.True(valid) + s.Equal("фото.png", filename) +} + +func (s *MessengerSuite) TestUploadCompensatesObjectWhenMetadataWriteFails() { + var putKey, deletedKey string + store := messengerStoreStub{ + userByID: model.User{ID: "00000000-0000-0000-0000-000000000001"}, + attachmentErr: errDependency, + } + objects := objectStoreStub{putKey: &putKey, deletedKey: &deletedKey} + server := NewServer( + s.logger, store, sessionStub{userID: store.userByID.ID}, passwordsStub{}, objects, + time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, + ) + data := testPNG(s.T(), 2, 2) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png", + bytes.NewReader(data), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + s.Equal(http.StatusServiceUnavailable, response.Code) + s.NotEmpty(putKey) + s.Equal(putKey, deletedKey) +} + +func (s *MessengerSuite) TestUploadKeepsObjectWhenMetadataCommitSucceeded() { + var deletedKey string + stored := model.Attachment{ + ID: "00000000-0000-0000-0000-000000000003", + ChatID: "00000000-0000-0000-0000-000000000002", + OwnerID: "00000000-0000-0000-0000-000000000001", + Filename: "photo.png", ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("0", 64), + Status: "ready", Variants: []model.Variant{}, CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + store := messengerStoreStub{ + userByID: model.User{ID: stored.OwnerID}, attachmentErr: errDependency, + storedAttachment: stored, storedAttachmentExists: true, + } + server := NewServer( + s.logger, store, sessionStub{userID: stored.OwnerID}, passwordsStub{}, objectStoreStub{deletedKey: &deletedKey}, + time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, + ) + data := testPNG(s.T(), 2, 2) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png", + bytes.NewReader(data), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + s.Equal(http.StatusCreated, response.Code) + s.Empty(deletedKey) +} + +func testPNG(t *testing.T, width, height int) []byte { + t.Helper() + var data bytes.Buffer + imageData := image.NewRGBA(image.Rect(0, 0, width, height)) + imageData.Set(0, 0, color.White) + if err := png.Encode(&data, imageData); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return data.Bytes() +} + func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server { - return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081") + return NewServer( + s.logger, messages, sessions, passwords, objectStoreStub{}, time.Hour, 24*time.Hour, + time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, + ) } func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { diff --git a/internal/model/model.go b/internal/model/model.go index 57a247e..1a73c6b 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -27,11 +27,37 @@ type Chat struct { } type Message struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` } type Position struct { @@ -54,11 +80,27 @@ type chatJSON struct { } type messageJSON struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt string `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type attachmentJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } func (u User) MarshalJSON() ([]byte, error) { @@ -70,7 +112,22 @@ func (c Chat) MarshalJSON() ([]byte, error) { } func (m Message) MarshalJSON() ([]byte, error) { - return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt)}) + attachments := m.AttachmentIDs + if attachments == nil { + attachments = []string{} + } + return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments}) +} + +func (a Attachment) MarshalJSON() ([]byte, error) { + variants := a.Variants + if variants == nil { + variants = []Variant{} + } + return json.Marshal(attachmentJSON{ + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt), + }) } func FormatTime(value time.Time) string { diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go new file mode 100644 index 0000000..6cf73bf --- /dev/null +++ b/internal/objectstore/store.go @@ -0,0 +1,180 @@ +package objectstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ErrNotFound = errors.New("object not found") + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseTLS bool + Timeout time.Duration +} + +type Object struct { + Key string + Size int64 + LastModified time.Time +} + +type Store struct { + client *minio.Client + bucket string + timeout time.Duration +} + +func New(config Config) (*Store, error) { + endpoint, err := endpointHost(config.Endpoint, config.UseTLS) + if err != nil { + return nil, err + } + if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 { + return nil, errors.New("invalid object store configuration") + } + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), + Secure: config.UseTLS, + Region: config.Region, + }) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil +} + +func endpointHost(value string, useTLS bool) (string, error) { + value = strings.TrimSpace(value) + if !strings.Contains(value, "://") { + if value == "" { + return "", errors.New("S3 endpoint is required") + } + return value, nil + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" { + return "", errors.New("invalid S3 endpoint") + } + if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS") + } + return parsed.Host, nil +} + +func (s *Store) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return fmt.Errorf("check S3 bucket: %w", err) + } + if !exists { + return errors.New("S3 bucket does not exist") + } + return nil +} + +func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.client.PutObject( + ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, + ) + if err != nil { + return fmt.Errorf("put S3 object: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + cancel() + return nil, objectError("get S3 object", err) + } + if _, err := object.Stat(); err != nil { + cancel() + _ = object.Close() + return nil, objectError("stat S3 object", err) + } + return &timeoutReader{ReadCloser: object, cancel: cancel}, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (Object, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return Object{}, objectError("stat S3 object", err) + } + return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Delete(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + return objectError("delete S3 object", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]Object, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", errors.New("object list limit must be between 1 and 1000") + } + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + objects := make([]Object, 0, limit+1) + options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1} + for info := range s.client.ListObjects(ctx, s.bucket, options) { + if info.Err != nil { + return nil, "", fmt.Errorf("list S3 objects: %w", info.Err) + } + objects = append(objects, Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) + if len(objects) == limit+1 { + cancel() + break + } + } + var next string + if len(objects) > limit { + objects = objects[:limit] + next = objects[len(objects)-1].Key + } + return objects, next, nil +} + +type timeoutReader struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (r *timeoutReader) Close() error { + err := r.ReadCloser.Close() + r.cancel() + return err +} + +func objectError(action string, err error) error { + response := minio.ToErrorResponse(err) + if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" { + return ErrNotFound + } + return fmt.Errorf("%s: %w", action, err) +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go new file mode 100644 index 0000000..59a73ee --- /dev/null +++ b/internal/postgres/attachments.go @@ -0,0 +1,155 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" +) + +func (s *Store) CreateAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { + const query = ` + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8 + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, created_at, updated_at` + + err := s.pool.QueryRow( + ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename, + attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey, + ).Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.CreatedAt, &attachment.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("create attachment: %w", err) + } + return readyAttachment(attachment), nil +} + +func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + const query = ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, + a.sha256, a.object_key, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 + WHERE a.id = $1` + + var attachment model.Attachment + err := s.pool.QueryRow(ctx, query, attachmentID, userID).Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.CreatedAt, &attachment.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) + } + return readyAttachment(attachment), nil +} + +func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + const query = ` + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, created_at, updated_at + FROM attachments + WHERE id = $1 AND object_key = $2` + + var attachment model.Attachment + err := s.pool.QueryRow(ctx, query, attachmentID, objectKey).Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.CreatedAt, &attachment.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err) + } + return readyAttachment(attachment), nil +} + +func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") + } + const query = ` + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, created_at, updated_at + FROM attachments + WHERE object_key > $1 + ORDER BY object_key + LIMIT $2` + rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) + if err != nil { + return nil, "", fmt.Errorf("list attachments: %w", err) + } + defer rows.Close() + + attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { + var attachment model.Attachment + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.CreatedAt, &attachment.UpdatedAt, + ) + return readyAttachment(attachment), err + })) + if err != nil { + return nil, "", fmt.Errorf("scan attachments: %w", err) + } + var next string + if len(attachments) > limit { + attachments = attachments[:limit] + next = attachments[len(attachments)-1].ObjectKey + } + return attachments, next, nil +} + +func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` + var referenced bool + if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil { + return false, fmt.Errorf("check attachment references: %w", err) + } + return referenced, nil +} + +func (s *Store) DeleteAttachmentIfUnreferencedBefore( + ctx context.Context, + attachmentID string, + createdBefore time.Time, +) (bool, error) { + const query = ` + DELETE FROM attachments a + WHERE a.id = $1 + AND a.created_at <= $2 + AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)` + result, err := s.pool.Exec(ctx, query, attachmentID, createdBefore) + if err != nil { + return false, fmt.Errorf("delete old unreferenced attachment: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func readyAttachment(attachment model.Attachment) model.Attachment { + attachment.Status = "ready" + attachment.Variants = []model.Variant{} + return attachment +} diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index a95ab99..2eeadf2 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/json" "errors" "fmt" "time" @@ -21,6 +22,7 @@ func (s *Store) CreateMessage( chatID string, senderID string, text string, + attachmentIDs []string, idempotencyKey string, idempotencyTTL time.Duration, ) (model.Message, bool, error) { @@ -46,13 +48,23 @@ func (s *Store) CreateMessage( if !visible { return model.Message{}, false, ErrNotFound } + validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + if !validAttachments { + return model.Message{}, false, ErrNotFound + } if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { return model.Message{}, false, err } - requestHash = sha256.Sum256([]byte(text)) + requestHash, err = messageRequestHash(text, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } message, replay, err = createIdempotentMessage( - ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL, + ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL, ) if err != nil { return model.Message{}, false, err @@ -63,6 +75,35 @@ func (s *Store) CreateMessage( return message, replay, nil } +func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + payload, err := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, attachmentIDs}) + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err) + } + return sha256.Sum256(payload), nil +} + +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) { + if len(attachmentIDs) == 0 { + return true, nil + } + const query = ` + SELECT count(*) + FROM attachments + WHERE chat_id = $1 AND id::text = ANY($2)` + var count int + if err := tx.QueryRow(ctx, query, chatID, attachmentIDs).Scan(&count); err != nil { + return false, fmt.Errorf("validate message attachments: %w", err) + } + return count == len(attachmentIDs), nil +} + func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { const query = ` SELECT EXISTS ( @@ -84,6 +125,7 @@ func createIdempotentMessage( chatID string, senderID string, text string, + attachmentIDs []string, idempotencyKey string, requestHash []byte, idempotencyTTL time.Duration, @@ -126,7 +168,7 @@ func createIdempotentMessage( continue } - message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text) + message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs) return message, false, err } } @@ -235,6 +277,7 @@ func insertMessage( chatID string, senderID string, text string, + attachmentIDs []string, ) (model.Message, error) { const query = ` INSERT INTO messages (id, chat_id, sender_id, text) @@ -255,14 +298,36 @@ func insertMessage( if err != nil { return model.Message{}, fmt.Errorf("insert message: %w", err) } + if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil { + return model.Message{}, err + } + message.AttachmentIDs = append([]string(nil), attachmentIDs...) return message, nil } +func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error { + const query = ` + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES ($1, $2, $3, $4)` + for position, attachmentID := range attachmentIDs { + if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil { + return fmt.Errorf("insert message attachment: %w", err) + } + } + return nil +} + func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) { const query = ` - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE id = $1` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.id = $1` var ( message model.Message @@ -274,6 +339,7 @@ func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Messag &message.SenderID, &message.Text, &message.CreatedAt, + &message.AttachmentIDs, ) if errors.Is(err, pgx.ErrNoRows) { return model.Message{}, fmt.Errorf("idempotency result message not found") diff --git a/internal/postgres/store.go b/internal/postgres/store.go index f8b3fcc..c0c07aa 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -344,11 +344,17 @@ func (s *Store) ListMessages( } const query = ` - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 - AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid)) - ORDER BY created_at, id + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.chat_id = $1 + AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid)) + ORDER BY m.created_at, m.id LIMIT $4` var afterTime any @@ -372,6 +378,7 @@ func (s *Store) ListMessages( &message.SenderID, &message.Text, &message.CreatedAt, + &message.AttachmentIDs, ) return message, err })) diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 461c539..08a9dbc 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -100,18 +100,16 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { owner, chatID := createOwnerAndChat(t, store) key := uuid.NewString() - first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour) + first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour) if err != nil || replay { t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err) } - second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour) + second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour) if err != nil || !replay { t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err) } - if first != second { - t.Fatalf("replay message = %v, want %v", second, first) - } - if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { + s.Equal(first, second) + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err) } if count := messageCount(t, pool, chatID); count != 1 { @@ -119,6 +117,62 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { } } +func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash") + s.Require().NoError(err) + attachment := model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + } + created, err := s.store.CreateAttachment(context.Background(), attachment) + s.Require().NoError(err) + s.Equal("ready", created.Status) + s.Empty(created.Variants) + s.Nil(created.ErrorCode) + + visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID) + s.Require().NoError(err) + s.Equal(created, visible) + _, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID) + s.ErrorIs(err, postgres.ErrNotFound) + + message, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + s.Equal([]string{created.ID}, message.AttachmentIDs) + + messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10) + s.Require().NoError(err) + s.Len(messages, 1) + s.Equal([]string{created.ID}, messages[0].AttachmentIDs) + referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) + s.Require().NoError(err) + s.True(referenced) + deleted, err := s.store.DeleteAttachmentIfUnreferencedBefore(context.Background(), created.ID, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(deleted) +} + +func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { + owner, firstChatID := createOwnerAndChat(s.T(), s.store) + secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) + s.Require().NoError(err) + attachment, err := s.store.CreateAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + _, _, err = s.store.CreateMessage( + context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrNotFound) +} + func (s *PostgresSuite) TestConcurrentMessageIdempotency() { t := s.T() pool := s.pool @@ -139,7 +193,7 @@ func (s *PostgresSuite) TestConcurrentMessageIdempotency() { go func() { defer group.Done() message, replay, err := stores[worker%len(stores)].CreateMessage( - context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour, + context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour, ) results <- result{message: message, replay: replay, err: err} }() @@ -186,13 +240,13 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { } key := uuid.NewString() - if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil { + if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil { t.Fatalf("owner CreateMessage() error = %v", err) } - if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil { + if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil { t.Fatalf("second user CreateMessage() error = %v", err) } - if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil { + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil { t.Fatalf("other chat CreateMessage() error = %v", err) } if _, err := pool.Exec(context.Background(), ` @@ -202,7 +256,7 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil { t.Fatalf("expire idempotency key: %v", err) } - message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour) + message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour) if err != nil || replay || message.Text != "after expiry" { t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err) } @@ -212,7 +266,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { t := s.T() owner, chatID := createOwnerAndChat(t, s.store) expiredKey := uuid.NewString() - if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil { + if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil { t.Fatalf("create expired message: %v", err) } if _, err := s.pool.Exec(context.Background(), ` @@ -223,7 +277,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { t.Fatalf("expire idempotency key: %v", err) } if _, _, err := s.store.CreateMessage( - context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour, + context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour, ); err != nil { t.Fatalf("create new message: %v", err) } diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..34c7444 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,236 @@ +package reconcile + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" +) + +type MetadataStore interface { + ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentReferenced(context.Context, string) (bool, error) + DeleteAttachmentIfUnreferencedBefore(context.Context, string, time.Time) (bool, error) +} + +type ObjectStore interface { + List(context.Context, string, string, int) ([]objectstore.Object, string, error) + Delete(context.Context, string) error +} + +type Config struct { + Prefix string + PageSize int + GraceTime time.Duration + Apply bool + Now func() time.Time +} + +type Report struct { + MetadataScanned int `json:"metadata_scanned"` + ObjectsScanned int `json:"objects_scanned"` + Consistent int `json:"consistent"` + FreshOrphanObjects int `json:"fresh_orphan_objects"` + OldOrphanObjects int `json:"old_orphan_objects"` + DeletedOrphanObjects int `json:"deleted_orphan_objects"` + FreshMissingObjects int `json:"fresh_missing_objects"` + ReferencedMissingObjects int `json:"referenced_missing_objects"` + OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` + DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` +} + +type Reconciler struct { + metadata MetadataStore + objects ObjectStore + config Config +} + +func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) { + if config.PageSize < 1 || config.PageSize > 1000 { + return nil, fmt.Errorf("page size must be between 1 and 1000") + } + if config.GraceTime < 0 { + return nil, fmt.Errorf("grace time must not be negative") + } + if config.Now == nil { + config.Now = time.Now + } + return &Reconciler{metadata: metadata, objects: objects, config: config}, nil +} + +func (r *Reconciler) Run(ctx context.Context) (Report, error) { + metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize) + if err != nil { + return Report{}, err + } + objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize) + if err != nil { + return Report{}, err + } + + cutoff := r.config.Now().Add(-r.config.GraceTime) + var report Report + for metadata.current != nil || objects.current != nil { + switch { + case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey: + if err := r.handleOrphanObject(ctx, *objects.current, cutoff, &report); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + case objects.current == nil || metadata.current.ObjectKey < objects.current.Key: + if err := r.handleMissingObject(ctx, *metadata.current, cutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + default: + report.MetadataScanned++ + report.ObjectsScanned++ + report.Consistent++ + if err := metadata.advance(ctx); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + } + } + return report, nil +} + +func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.Object, cutoff time.Time, report *Report) error { + report.ObjectsScanned++ + if object.LastModified.After(cutoff) { + report.FreshOrphanObjects++ + return nil + } + report.OldOrphanObjects++ + if !r.config.Apply { + return nil + } + if err := r.objects.Delete(ctx, object.Key); err != nil { + return fmt.Errorf("delete orphan object %q: %w", object.Key, err) + } + report.DeletedOrphanObjects++ + return nil +} + +func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Attachment, cutoff time.Time, report *Report) error { + report.MetadataScanned++ + if metadata.CreatedAt.After(cutoff) { + report.FreshMissingObjects++ + return nil + } + referenced, err := r.metadata.AttachmentReferenced(ctx, metadata.ID) + if err != nil { + return fmt.Errorf("check attachment metadata %q: %w", metadata.ID, err) + } + if referenced { + report.ReferencedMissingObjects++ + return nil + } + report.OldUnreferencedMetadata++ + if !r.config.Apply { + return nil + } + deleted, err := r.metadata.DeleteAttachmentIfUnreferencedBefore(ctx, metadata.ID, cutoff) + if err != nil { + return fmt.Errorf("delete attachment metadata %q: %w", metadata.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +type metadataIterator struct { + store MetadataStore + pageSize int + after string + page []model.Attachment + index int + current *model.Attachment + done bool +} + +func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) { + iterator := &metadataIterator{store: store, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *metadataIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list attachment metadata: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return fmt.Errorf("metadata page is empty but has continuation") + } + } +} + +type objectIterator struct { + store ObjectStore + prefix string + pageSize int + after string + page []objectstore.Object + index int + current *objectstore.Object + done bool +} + +func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) { + iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *objectIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return fmt.Errorf("object page is empty but has continuation") + } + } +} diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..91d1cc9 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,47 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_length, + DROP CONSTRAINT messages_text_not_blank; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);