forked from mtusi2026/mtusi2026lab4
feat: add S3 attachments and reconciliation
This commit is contained in:
+84
-1
@@ -2,20 +2,26 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||
)
|
||||
|
||||
@@ -34,7 +40,10 @@ func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return runMigrations()
|
||||
}
|
||||
return fmt.Errorf("usage: messenger [migrate]")
|
||||
if len(arguments) > 0 && arguments[0] == "reconcile-attachments" {
|
||||
return runAttachmentReconciliation(arguments[1:], os.Stdout)
|
||||
}
|
||||
return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]")
|
||||
}
|
||||
|
||||
func runServer(logger *slog.Logger) error {
|
||||
@@ -68,16 +77,28 @@ func runServer(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app := messenger.NewServer(
|
||||
logger,
|
||||
postgres.New(pool),
|
||||
sessions,
|
||||
passwords,
|
||||
objects,
|
||||
settings.SessionTTL,
|
||||
settings.IdempotencyTTL,
|
||||
settings.RequestTimeout,
|
||||
instanceID,
|
||||
settings.SwaggerPort,
|
||||
settings.AttachmentMaxBytes,
|
||||
settings.AttachmentMaxPixels,
|
||||
settings.AttachmentDecodeMemory,
|
||||
)
|
||||
server := &http.Server{
|
||||
Addr: settings.Address,
|
||||
@@ -130,3 +151,65 @@ func runMigrations() error {
|
||||
defer pool.Close()
|
||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||
}
|
||||
|
||||
type reconciliationOutput struct {
|
||||
Mode string `json:"mode"`
|
||||
GracePeriod string `json:"grace_period"`
|
||||
PageSize int `json:"page_size"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func runAttachmentReconciliation(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
apply := flags.Bool("apply", false, "apply safe repairs")
|
||||
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
|
||||
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]")
|
||||
}
|
||||
|
||||
settings, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mode := "dry-run"
|
||||
if *apply {
|
||||
mode = "apply"
|
||||
}
|
||||
return json.NewEncoder(output).Encode(reconciliationOutput{
|
||||
Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize,
|
||||
DurationMillis: time.Since(started).Milliseconds(), Report: report,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,37 +5,49 @@ go 1.26.0
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.0.99
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sync v0.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
|
||||
@@ -5,10 +5,14 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -23,20 +27,33 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
|
||||
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
||||
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
@@ -49,6 +66,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
@@ -68,6 +87,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
||||
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
|
||||
+93
-24
@@ -12,29 +12,39 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Address string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HTTPReadHeaderTimeout time.Duration
|
||||
HTTPReadTimeout time.Duration
|
||||
HTTPWriteTimeout time.Duration
|
||||
HTTPIdleTimeout time.Duration
|
||||
HTTPShutdownTimeout time.Duration
|
||||
PasswordMemoryKiB uint32
|
||||
PasswordIterations uint32
|
||||
PasswordParallelism uint8
|
||||
PasswordSaltLength uint32
|
||||
PasswordKeyLength uint32
|
||||
SwaggerPort string
|
||||
Address string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HTTPReadHeaderTimeout time.Duration
|
||||
HTTPReadTimeout time.Duration
|
||||
HTTPWriteTimeout time.Duration
|
||||
HTTPIdleTimeout time.Duration
|
||||
HTTPShutdownTimeout time.Duration
|
||||
PasswordMemoryKiB uint32
|
||||
PasswordIterations uint32
|
||||
PasswordParallelism uint8
|
||||
PasswordSaltLength uint32
|
||||
PasswordKeyLength uint32
|
||||
SwaggerPort string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3UseTLS bool
|
||||
S3Timeout time.Duration
|
||||
AttachmentMaxBytes int64
|
||||
AttachmentMaxPixels int64
|
||||
AttachmentDecodeMemory int64
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
@@ -142,6 +152,50 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3Endpoint, err := requiredString(values, "S3_ENDPOINT")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3Region, err := requiredString(values, "S3_REGION")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3Bucket, err := requiredString(values, "S3_BUCKET")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3AccessKey, err := requiredString(values, "S3_ACCESS_KEY")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3SecretKey, err := requiredString(values, "S3_SECRET_KEY")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3UseTLS, err := boolean(values, "S3_USE_TLS")
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3Timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
minimumDecodeMemory := int64(attachmentMaxBytes) + int64(attachmentMaxPixels)*8
|
||||
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||
return Config{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||
}
|
||||
|
||||
return Config{
|
||||
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
||||
@@ -155,7 +209,10 @@ func Load() (Config, error) {
|
||||
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
|
||||
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
|
||||
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
|
||||
SwaggerPort: swaggerPort,
|
||||
SwaggerPort: swaggerPort, S3Endpoint: s3Endpoint, S3Region: s3Region, S3Bucket: s3Bucket,
|
||||
S3AccessKey: s3AccessKey, S3SecretKey: s3SecretKey, S3UseTLS: s3UseTLS, S3Timeout: s3Timeout,
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -260,3 +317,15 @@ func integerValue(value string, minimum, maximum int) (int, error) {
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func boolean(values *viper.Viper, name string) (bool, error) {
|
||||
value, err := requiredString(values, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: must be true or false", name)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
)
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
started := time.Now()
|
||||
result := "client_error"
|
||||
defer func() {
|
||||
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
|
||||
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
|
||||
}()
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
return methodNotAllowed(w, http.MethodPost)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
filename, valid := attachmentFilename(r.URL.Query())
|
||||
if !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
if _, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID); errors.Is(err, postgres.ErrNotFound) {
|
||||
return notFound()
|
||||
} else if err != nil {
|
||||
result = "dependency_error"
|
||||
return unavailable()
|
||||
}
|
||||
data, contentType, readErr := readImage(w, r, s.maxUploadBytes, s.maxImagePixels, s.decodeMemory)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
|
||||
attachmentID, err := newID()
|
||||
if err != nil {
|
||||
result = "dependency_error"
|
||||
return internalError()
|
||||
}
|
||||
objectID, err := newID()
|
||||
if err != nil {
|
||||
result = "dependency_error"
|
||||
return internalError()
|
||||
}
|
||||
objectKey := "attachments/" + objectID + "/original"
|
||||
if err := s.objects.Put(r.Context(), objectKey, contentType, data); err != nil {
|
||||
result = "dependency_error"
|
||||
return unavailable()
|
||||
}
|
||||
digest := sha256.Sum256(data)
|
||||
attachment, err := s.messengerStore.CreateAttachment(r.Context(), model.Attachment{
|
||||
ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: objectKey,
|
||||
})
|
||||
if err != nil {
|
||||
result = "dependency_error"
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.requestTTL)
|
||||
defer cancel()
|
||||
stored, lookupErr := s.messengerStore.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
|
||||
if lookupErr == nil {
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(len(data)))
|
||||
writeJSON(w, http.StatusCreated, stored)
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(lookupErr, postgres.ErrNotFound) {
|
||||
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", lookupErr)
|
||||
return unavailable()
|
||||
}
|
||||
if cleanupErr := s.objects.Delete(cleanupCtx, objectKey); cleanupErr != nil {
|
||||
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", cleanupErr)
|
||||
}
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
return notFound()
|
||||
}
|
||||
return unavailable()
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(len(data)))
|
||||
writeJSON(w, http.StatusCreated, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
func attachmentFilename(query url.Values) (string, bool) {
|
||||
if len(query) != 1 || len(query["filename"]) != 1 {
|
||||
return "", false
|
||||
}
|
||||
filename := query["filename"][0]
|
||||
length := utf8.RuneCountInString(filename)
|
||||
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
|
||||
return "", false
|
||||
}
|
||||
for _, char := range filename {
|
||||
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return filename, true
|
||||
}
|
||||
|
||||
func readImage(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
maxBytes int64,
|
||||
maxPixels int64,
|
||||
decodeMemory *semaphore.Weighted,
|
||||
) ([]byte, string, *apiError) {
|
||||
mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
|
||||
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
|
||||
}
|
||||
if r.ContentLength > maxBytes {
|
||||
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||
}
|
||||
return nil, "", invalidRequest()
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
|
||||
}
|
||||
detected := http.DetectContentType(data)
|
||||
if detected != mediaType {
|
||||
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "content type does not match image"}
|
||||
}
|
||||
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || imageContentType(format) != mediaType {
|
||||
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
|
||||
}
|
||||
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
|
||||
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > maxPixels {
|
||||
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image dimensions are too large"}
|
||||
}
|
||||
decodeBytes := pixels*8 + int64(len(data))
|
||||
if !decodeMemory.TryAcquire(decodeBytes) {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
return nil, "", &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
|
||||
}
|
||||
defer decodeMemory.Release(decodeBytes)
|
||||
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
|
||||
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
|
||||
}
|
||||
return data, mediaType, nil
|
||||
}
|
||||
|
||||
func imageContentType(format string) string {
|
||||
switch format {
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if r.Method != http.MethodGet {
|
||||
return methodNotAllowed(w, http.MethodGet)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
attachment, apiErr := s.attachmentForUser(r, user.ID)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
writeJSON(w, http.StatusOK, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if r.Method != http.MethodGet {
|
||||
return methodNotAllowed(w, http.MethodGet)
|
||||
}
|
||||
user, _, authErr := s.authenticate(r)
|
||||
if authErr != nil {
|
||||
return authErr
|
||||
}
|
||||
if !originalVariant(r.URL.Query()) {
|
||||
return invalidRequest()
|
||||
}
|
||||
attachment, apiErr := s.attachmentForUser(r, user.ID)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
content, err := s.objects.Get(r.Context(), attachment.ObjectKey)
|
||||
if errors.Is(err, objectstore.ErrNotFound) {
|
||||
return unavailable()
|
||||
}
|
||||
if err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
defer func() { _ = content.Close() }()
|
||||
w.Header().Set("Content-Type", attachment.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(attachment.SizeBytes, 10))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content); err != nil {
|
||||
s.logger.Warn("attachment download interrupted", "attachment_id", attachment.ID, "error", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func originalVariant(query url.Values) bool {
|
||||
if len(query) == 0 {
|
||||
return true
|
||||
}
|
||||
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
|
||||
}
|
||||
|
||||
func (s *Server) attachmentForUser(r *http.Request, userID string) (Attachment, *apiError) {
|
||||
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||
attachmentID, valid := canonicalID(parts[3])
|
||||
if !valid {
|
||||
return Attachment{}, invalidRequest()
|
||||
}
|
||||
attachment, err := s.messengerStore.AttachmentForUser(r.Context(), attachmentID, userID)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
return Attachment{}, notFound()
|
||||
}
|
||||
if err != nil {
|
||||
return Attachment{}, unavailable()
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
@@ -14,6 +14,7 @@ const (
|
||||
errorCodePayloadTooLarge = "payload_too_large"
|
||||
errorCodeUnavailable = "unavailable"
|
||||
errorCodeConflict = "conflict"
|
||||
errorCodeRateLimited = "rate_limited"
|
||||
)
|
||||
|
||||
type apiError struct {
|
||||
|
||||
@@ -38,6 +38,9 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
if err := s.sessions.Ping(ctx); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
if err := s.objects.Ping(ctx); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
return nil
|
||||
}
|
||||
@@ -207,16 +210,17 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
return invalidRequest()
|
||||
}
|
||||
var input struct {
|
||||
Text string `json:"text"`
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
if err := readJSON(w, r, &input); err != nil {
|
||||
return err
|
||||
}
|
||||
if !validText(input.Text, 4000) {
|
||||
if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) {
|
||||
return invalidRequest()
|
||||
}
|
||||
message, replay, err := s.messengerStore.CreateMessage(
|
||||
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
|
||||
r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL,
|
||||
)
|
||||
if errors.Is(err, postgres.ErrNotFound) {
|
||||
return notFound()
|
||||
@@ -234,6 +238,32 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validMessage(text string, attachmentIDs []string) bool {
|
||||
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 {
|
||||
return false
|
||||
}
|
||||
return len(attachmentIDs) > 0 || strings.TrimSpace(text) != ""
|
||||
}
|
||||
|
||||
func canonicalAttachmentIDs(attachmentIDs []string) bool {
|
||||
if len(attachmentIDs) > 10 {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(attachmentIDs))
|
||||
for index, attachmentID := range attachmentIDs {
|
||||
canonical, valid := canonicalID(attachmentID)
|
||||
if !valid {
|
||||
return false
|
||||
}
|
||||
if _, duplicate := seen[canonical]; duplicate {
|
||||
return false
|
||||
}
|
||||
seen[canonical] = struct{}{}
|
||||
attachmentIDs[index] = canonical
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
|
||||
limit, cursor, err := pageParams(r, "message", chatID)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package messenger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
@@ -15,8 +16,18 @@ type MessengerStore interface {
|
||||
CreateChat(context.Context, string, []string, string) (model.Chat, error)
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
|
||||
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
|
||||
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
|
||||
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
|
||||
CreateAttachment(context.Context, model.Attachment) (model.Attachment, error)
|
||||
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Ping(context.Context) error
|
||||
Put(context.Context, string, string, []byte) error
|
||||
Get(context.Context, string) (io.ReadCloser, error)
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type SessionStore interface {
|
||||
|
||||
@@ -12,11 +12,14 @@ import (
|
||||
)
|
||||
|
||||
type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
handler http.Handler
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
attachmentUploads *prometheus.CounterVec
|
||||
attachmentDuration prometheus.Histogram
|
||||
attachmentBytes prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
@@ -47,12 +50,31 @@ func newMetrics() *metrics {
|
||||
Name: "idempotency_replays_total",
|
||||
Help: "Message requests served from an idempotency record.",
|
||||
}),
|
||||
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_uploads_total",
|
||||
Help: "Completed attachment uploads by result.",
|
||||
}, []string{"result"}),
|
||||
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_duration_seconds",
|
||||
Help: "Attachment upload duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}),
|
||||
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_bytes_total",
|
||||
Help: "Bytes accepted by successful attachment uploads.",
|
||||
}),
|
||||
}
|
||||
registry.MustRegister(
|
||||
result.requests,
|
||||
result.duration,
|
||||
result.inFlight,
|
||||
result.replays,
|
||||
result.attachmentUploads,
|
||||
result.attachmentDuration,
|
||||
result.attachmentBytes,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
@@ -79,5 +101,14 @@ func routeTemplate(path string) string {
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||
return "/api/v1/chats/{chat_id}/messages"
|
||||
}
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||
return "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||
return "/api/v1/attachments/{attachment_id}"
|
||||
}
|
||||
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||
return "/api/v1/attachments/{attachment_id}/content"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
type User = model.User
|
||||
type Chat = model.Chat
|
||||
type Message = model.Message
|
||||
type Attachment = model.Attachment
|
||||
|
||||
func newID() (string, error) {
|
||||
id, err := uuid.NewRandom()
|
||||
|
||||
@@ -33,6 +33,12 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
handler = s.chat
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
|
||||
handler = s.messages
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
|
||||
handler = s.uploadAttachment
|
||||
} else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
|
||||
handler = s.attachmentMetadata
|
||||
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
|
||||
handler = s.attachmentContent
|
||||
} else {
|
||||
return notFound()
|
||||
}
|
||||
|
||||
@@ -6,18 +6,24 @@ import (
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
messengerStore MessengerStore
|
||||
sessions SessionStore
|
||||
passwords PasswordService
|
||||
objects ObjectStore
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
requestTTL time.Duration
|
||||
instanceID string
|
||||
logger *slog.Logger
|
||||
swaggerPort string
|
||||
maxUploadBytes int64
|
||||
maxImagePixels int64
|
||||
decodeMemory *semaphore.Weighted
|
||||
metrics *metrics
|
||||
ready atomic.Bool
|
||||
}
|
||||
@@ -37,16 +43,22 @@ func NewServer(
|
||||
messages MessengerStore,
|
||||
sessions SessionStore,
|
||||
passwords PasswordService,
|
||||
objects ObjectStore,
|
||||
sessionTTL time.Duration,
|
||||
idempotencyTTL time.Duration,
|
||||
requestTTL time.Duration,
|
||||
instanceID string,
|
||||
swaggerPort string,
|
||||
maxUploadBytes int64,
|
||||
maxImagePixels int64,
|
||||
decodeMemoryBytes int64,
|
||||
) *Server {
|
||||
s := &Server{
|
||||
messengerStore: messages, sessions: sessions, passwords: passwords,
|
||||
messengerStore: messages, sessions: sessions, passwords: passwords, objects: objects,
|
||||
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
|
||||
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
|
||||
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort,
|
||||
maxUploadBytes: maxUploadBytes, maxImagePixels: maxImagePixels,
|
||||
decodeMemory: semaphore.NewWeighted(decodeMemoryBytes), metrics: newMetrics(),
|
||||
}
|
||||
s.ready.Store(true)
|
||||
return s
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -13,6 +17,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
@@ -34,14 +39,19 @@ func (s *MessengerSuite) SetupTest() {
|
||||
}
|
||||
|
||||
type messengerStoreStub struct {
|
||||
pingErr error
|
||||
userByID model.User
|
||||
userByIDErr error
|
||||
credentials model.UserCredentials
|
||||
credentialsErr error
|
||||
message model.Message
|
||||
messageReplay bool
|
||||
messageErr error
|
||||
pingErr error
|
||||
userByID model.User
|
||||
userByIDErr error
|
||||
credentials model.UserCredentials
|
||||
credentialsErr error
|
||||
message model.Message
|
||||
messageReplay bool
|
||||
messageErr error
|
||||
attachment model.Attachment
|
||||
attachmentErr error
|
||||
storedAttachment model.Attachment
|
||||
storedAttachmentExists bool
|
||||
storedAttachmentErr error
|
||||
}
|
||||
|
||||
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
|
||||
@@ -63,12 +73,57 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
|
||||
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
|
||||
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) {
|
||||
return s.message, s.messageReplay, s.messageErr
|
||||
}
|
||||
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
func (s messengerStoreStub) CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) {
|
||||
return s.attachment, s.attachmentErr
|
||||
}
|
||||
func (s messengerStoreStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.storedAttachmentErr != nil {
|
||||
return model.Attachment{}, s.storedAttachmentErr
|
||||
}
|
||||
if !s.storedAttachmentExists {
|
||||
return model.Attachment{}, postgres.ErrNotFound
|
||||
}
|
||||
return s.storedAttachment, nil
|
||||
}
|
||||
func (s messengerStoreStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.attachment, s.attachmentErr
|
||||
}
|
||||
|
||||
type objectStoreStub struct {
|
||||
pingErr error
|
||||
putErr error
|
||||
getData []byte
|
||||
getErr error
|
||||
deleteErr error
|
||||
putKey *string
|
||||
deletedKey *string
|
||||
}
|
||||
|
||||
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
|
||||
func (s objectStoreStub) Put(_ context.Context, key string, _ string, _ []byte) error {
|
||||
if s.putKey != nil {
|
||||
*s.putKey = key
|
||||
}
|
||||
return s.putErr
|
||||
}
|
||||
func (s objectStoreStub) Get(context.Context, string) (io.ReadCloser, error) {
|
||||
if s.getErr != nil {
|
||||
return nil, s.getErr
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(s.getData)), nil
|
||||
}
|
||||
func (s objectStoreStub) Delete(_ context.Context, key string) error {
|
||||
if s.deletedKey != nil {
|
||||
*s.deletedKey = key
|
||||
}
|
||||
return s.deleteErr
|
||||
}
|
||||
|
||||
type sessionStub struct {
|
||||
pingErr error
|
||||
@@ -100,14 +155,19 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
|
||||
name string
|
||||
store messengerStoreStub
|
||||
sessions sessionStub
|
||||
objects objectStoreStub
|
||||
}{
|
||||
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
|
||||
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
|
||||
{name: "s3", objects: objectStoreStub{pingErr: errDependency}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
|
||||
server := NewServer(
|
||||
s.logger, tt.store, tt.sessions, passwordsStub{}, tt.objects, time.Hour, 24*time.Hour,
|
||||
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
|
||||
)
|
||||
response := serveRequest(server, http.MethodGet, "/health/ready", "")
|
||||
if response.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
|
||||
@@ -160,6 +220,18 @@ func (s *MessengerSuite) TestAuthenticationChallenges() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() {
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
response := serveRequest(
|
||||
server,
|
||||
http.MethodGet,
|
||||
"/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid",
|
||||
"",
|
||||
)
|
||||
s.Equal(http.StatusUnauthorized, response.Code)
|
||||
s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate"))
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
|
||||
t := s.T()
|
||||
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
|
||||
@@ -242,8 +314,135 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestImageValidation() {
|
||||
data := testPNG(s.T(), 2, 2)
|
||||
tests := []struct {
|
||||
name string
|
||||
contentType string
|
||||
data []byte
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "valid", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 4},
|
||||
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
|
||||
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, wantStatus: http.StatusRequestEntityTooLarge},
|
||||
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, wantStatus: http.StatusRequestEntityTooLarge},
|
||||
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(tt.data))
|
||||
request.Header.Set("Content-Type", tt.contentType)
|
||||
response := httptest.NewRecorder()
|
||||
actual, _, apiErr := readImage(
|
||||
response, request, tt.maxBytes, tt.maxPixels, semaphore.NewWeighted(384*1024*1024),
|
||||
)
|
||||
if tt.wantStatus == 0 {
|
||||
s.Require().Nil(apiErr)
|
||||
s.Equal(tt.data, actual)
|
||||
return
|
||||
}
|
||||
s.Require().NotNil(apiErr)
|
||||
s.Equal(tt.wantStatus, apiErr.status)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestImageValidationLimitsConcurrentDecodeMemory() {
|
||||
data := testPNG(s.T(), 2, 2)
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(data))
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
response := httptest.NewRecorder()
|
||||
_, _, apiErr := readImage(response, request, 1024, 4, semaphore.NewWeighted(1))
|
||||
s.Require().NotNil(apiErr)
|
||||
s.Equal(http.StatusTooManyRequests, apiErr.status)
|
||||
s.Equal("1", response.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
|
||||
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
|
||||
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
|
||||
s.False(valid, filename)
|
||||
}
|
||||
filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}})
|
||||
s.True(valid)
|
||||
s.Equal("фото.png", filename)
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestUploadCompensatesObjectWhenMetadataWriteFails() {
|
||||
var putKey, deletedKey string
|
||||
store := messengerStoreStub{
|
||||
userByID: model.User{ID: "00000000-0000-0000-0000-000000000001"},
|
||||
attachmentErr: errDependency,
|
||||
}
|
||||
objects := objectStoreStub{putKey: &putKey, deletedKey: &deletedKey}
|
||||
server := NewServer(
|
||||
s.logger, store, sessionStub{userID: store.userByID.ID}, passwordsStub{}, objects,
|
||||
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
|
||||
)
|
||||
data := testPNG(s.T(), 2, 2)
|
||||
request := httptest.NewRequest(
|
||||
http.MethodPost,
|
||||
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
|
||||
bytes.NewReader(data),
|
||||
)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
s.Equal(http.StatusServiceUnavailable, response.Code)
|
||||
s.NotEmpty(putKey)
|
||||
s.Equal(putKey, deletedKey)
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) TestUploadKeepsObjectWhenMetadataCommitSucceeded() {
|
||||
var deletedKey string
|
||||
stored := model.Attachment{
|
||||
ID: "00000000-0000-0000-0000-000000000003",
|
||||
ChatID: "00000000-0000-0000-0000-000000000002",
|
||||
OwnerID: "00000000-0000-0000-0000-000000000001",
|
||||
Filename: "photo.png", ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("0", 64),
|
||||
Status: "ready", Variants: []model.Variant{}, CreatedAt: time.Now(), UpdatedAt: time.Now(),
|
||||
}
|
||||
store := messengerStoreStub{
|
||||
userByID: model.User{ID: stored.OwnerID}, attachmentErr: errDependency,
|
||||
storedAttachment: stored, storedAttachmentExists: true,
|
||||
}
|
||||
server := NewServer(
|
||||
s.logger, store, sessionStub{userID: stored.OwnerID}, passwordsStub{}, objectStoreStub{deletedKey: &deletedKey},
|
||||
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
|
||||
)
|
||||
data := testPNG(s.T(), 2, 2)
|
||||
request := httptest.NewRequest(
|
||||
http.MethodPost,
|
||||
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
|
||||
bytes.NewReader(data),
|
||||
)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
response := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(response, request)
|
||||
s.Equal(http.StatusCreated, response.Code)
|
||||
s.Empty(deletedKey)
|
||||
}
|
||||
|
||||
func testPNG(t *testing.T, width, height int) []byte {
|
||||
t.Helper()
|
||||
var data bytes.Buffer
|
||||
imageData := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
imageData.Set(0, 0, color.White)
|
||||
if err := png.Encode(&data, imageData); err != nil {
|
||||
t.Fatalf("encode PNG: %v", err)
|
||||
}
|
||||
return data.Bytes()
|
||||
}
|
||||
|
||||
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
|
||||
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
|
||||
return NewServer(
|
||||
s.logger, messages, sessions, passwords, objectStoreStub{}, time.Hour, 24*time.Hour,
|
||||
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
|
||||
)
|
||||
}
|
||||
|
||||
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
|
||||
|
||||
+68
-11
@@ -27,11 +27,37 @@ type Chat struct {
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type Position struct {
|
||||
@@ -54,11 +80,27 @@ type chatJSON struct {
|
||||
}
|
||||
|
||||
type messageJSON struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type attachmentJSON struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (u User) MarshalJSON() ([]byte, error) {
|
||||
@@ -70,7 +112,22 @@ func (c Chat) MarshalJSON() ([]byte, error) {
|
||||
}
|
||||
|
||||
func (m Message) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt)})
|
||||
attachments := m.AttachmentIDs
|
||||
if attachments == nil {
|
||||
attachments = []string{}
|
||||
}
|
||||
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments})
|
||||
}
|
||||
|
||||
func (a Attachment) MarshalJSON() ([]byte, error) {
|
||||
variants := a.Variants
|
||||
if variants == nil {
|
||||
variants = []Variant{}
|
||||
}
|
||||
return json.Marshal(attachmentJSON{
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
|
||||
a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt),
|
||||
})
|
||||
}
|
||||
|
||||
func FormatTime(value time.Time) string {
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseTLS bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func New(config Config) (*Store, error) {
|
||||
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
|
||||
return nil, errors.New("invalid object store configuration")
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
|
||||
Secure: config.UseTLS,
|
||||
Region: config.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create S3 client: %w", err)
|
||||
}
|
||||
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
|
||||
}
|
||||
|
||||
func endpointHost(value string, useTLS bool) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if !strings.Contains(value, "://") {
|
||||
if value == "" {
|
||||
return "", errors.New("S3 endpoint is required")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
|
||||
return "", errors.New("invalid S3 endpoint")
|
||||
}
|
||||
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
|
||||
}
|
||||
return parsed.Host, nil
|
||||
}
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check S3 bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("S3 bucket does not exist")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
_, err := s.client.PutObject(
|
||||
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, objectError("get S3 object", err)
|
||||
}
|
||||
if _, err := object.Stat(); err != nil {
|
||||
cancel()
|
||||
_ = object.Close()
|
||||
return nil, objectError("stat S3 object", err)
|
||||
}
|
||||
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return Object{}, objectError("stat S3 object", err)
|
||||
}
|
||||
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
return objectError("delete S3 object", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]Object, string, error) {
|
||||
if limit < 1 || limit > 1000 {
|
||||
return nil, "", errors.New("object list limit must be between 1 and 1000")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
objects := make([]Object, 0, limit+1)
|
||||
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
|
||||
for info := range s.client.ListObjects(ctx, s.bucket, options) {
|
||||
if info.Err != nil {
|
||||
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
|
||||
}
|
||||
objects = append(objects, Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
|
||||
if len(objects) == limit+1 {
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
}
|
||||
var next string
|
||||
if len(objects) > limit {
|
||||
objects = objects[:limit]
|
||||
next = objects[len(objects)-1].Key
|
||||
}
|
||||
return objects, next, nil
|
||||
}
|
||||
|
||||
type timeoutReader struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r *timeoutReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
r.cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
func objectError(action string, err error) error {
|
||||
response := minio.ToErrorResponse(err)
|
||||
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
|
||||
return ErrNotFound
|
||||
}
|
||||
return fmt.Errorf("%s: %w", action, err)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) CreateAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
|
||||
const query = `
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key
|
||||
)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
|
||||
)
|
||||
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, created_at, updated_at`
|
||||
|
||||
err := s.pool.QueryRow(
|
||||
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
|
||||
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
|
||||
).Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.CreatedAt, &attachment.UpdatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("create attachment: %w", err)
|
||||
}
|
||||
return readyAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
|
||||
const query = `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
|
||||
a.sha256, a.object_key, a.created_at, a.updated_at
|
||||
FROM attachments a
|
||||
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
|
||||
WHERE a.id = $1`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := s.pool.QueryRow(ctx, query, attachmentID, userID).Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.CreatedAt, &attachment.UpdatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
|
||||
}
|
||||
return readyAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||
const query = `
|
||||
SELECT id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE id = $1 AND object_key = $2`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := s.pool.QueryRow(ctx, query, attachmentID, objectKey).Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.CreatedAt, &attachment.UpdatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
|
||||
}
|
||||
return readyAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
|
||||
if limit < 1 || limit > 1000 {
|
||||
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
|
||||
}
|
||||
const query = `
|
||||
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE object_key > $1
|
||||
ORDER BY object_key
|
||||
LIMIT $2`
|
||||
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("list attachments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
|
||||
var attachment model.Attachment
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.CreatedAt, &attachment.UpdatedAt,
|
||||
)
|
||||
return readyAttachment(attachment), err
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("scan attachments: %w", err)
|
||||
}
|
||||
var next string
|
||||
if len(attachments) > limit {
|
||||
attachments = attachments[:limit]
|
||||
next = attachments[len(attachments)-1].ObjectKey
|
||||
}
|
||||
return attachments, next, nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
|
||||
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
|
||||
var referenced bool
|
||||
if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil {
|
||||
return false, fmt.Errorf("check attachment references: %w", err)
|
||||
}
|
||||
return referenced, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAttachmentIfUnreferencedBefore(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
createdBefore time.Time,
|
||||
) (bool, error) {
|
||||
const query = `
|
||||
DELETE FROM attachments a
|
||||
WHERE a.id = $1
|
||||
AND a.created_at <= $2
|
||||
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
|
||||
result, err := s.pool.Exec(ctx, query, attachmentID, createdBefore)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("delete old unreferenced attachment: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func readyAttachment(attachment model.Attachment) model.Attachment {
|
||||
attachment.Status = "ready"
|
||||
attachment.Variants = []model.Variant{}
|
||||
return attachment
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
@@ -21,6 +22,7 @@ func (s *Store) CreateMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
idempotencyKey string,
|
||||
idempotencyTTL time.Duration,
|
||||
) (model.Message, bool, error) {
|
||||
@@ -46,13 +48,23 @@ func (s *Store) CreateMessage(
|
||||
if !visible {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !validAttachments {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
|
||||
requestHash = sha256.Sum256([]byte(text))
|
||||
requestHash, err = messageRequestHash(text, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
message, replay, err = createIdempotentMessage(
|
||||
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||
ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL,
|
||||
)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
@@ -63,6 +75,35 @@ func (s *Store) CreateMessage(
|
||||
return message, replay, nil
|
||||
}
|
||||
|
||||
func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) {
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = []string{}
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}{text, attachmentIDs})
|
||||
if err != nil {
|
||||
return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err)
|
||||
}
|
||||
return sha256.Sum256(payload), nil
|
||||
}
|
||||
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
|
||||
if len(attachmentIDs) == 0 {
|
||||
return true, nil
|
||||
}
|
||||
const query = `
|
||||
SELECT count(*)
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id::text = ANY($2)`
|
||||
var count int
|
||||
if err := tx.QueryRow(ctx, query, chatID, attachmentIDs).Scan(&count); err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
return count == len(attachmentIDs), nil
|
||||
}
|
||||
|
||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||
const query = `
|
||||
SELECT EXISTS (
|
||||
@@ -84,6 +125,7 @@ func createIdempotentMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
idempotencyKey string,
|
||||
requestHash []byte,
|
||||
idempotencyTTL time.Duration,
|
||||
@@ -126,7 +168,7 @@ func createIdempotentMessage(
|
||||
continue
|
||||
}
|
||||
|
||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
|
||||
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs)
|
||||
return message, false, err
|
||||
}
|
||||
}
|
||||
@@ -235,6 +277,7 @@ func insertMessage(
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
attachmentIDs []string,
|
||||
) (model.Message, error) {
|
||||
const query = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
@@ -255,14 +298,36 @@ func insertMessage(
|
||||
if err != nil {
|
||||
return model.Message{}, fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil {
|
||||
return model.Message{}, err
|
||||
}
|
||||
message.AttachmentIDs = append([]string(nil), attachmentIDs...)
|
||||
return message, nil
|
||||
}
|
||||
|
||||
func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error {
|
||||
const query = `
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES ($1, $2, $3, $4)`
|
||||
for position, attachmentID := range attachmentIDs {
|
||||
if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil {
|
||||
return fmt.Errorf("insert message attachment: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
|
||||
const query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE id = $1`
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(
|
||||
SELECT ma.attachment_id::text
|
||||
FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id
|
||||
ORDER BY ma.position
|
||||
)
|
||||
FROM messages m
|
||||
WHERE m.id = $1`
|
||||
|
||||
var (
|
||||
message model.Message
|
||||
@@ -274,6 +339,7 @@ func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Messag
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Message{}, fmt.Errorf("idempotency result message not found")
|
||||
|
||||
@@ -344,11 +344,17 @@ func (s *Store) ListMessages(
|
||||
}
|
||||
|
||||
const query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid))
|
||||
ORDER BY created_at, id
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(
|
||||
SELECT ma.attachment_id::text
|
||||
FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id
|
||||
ORDER BY ma.position
|
||||
)
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid))
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $4`
|
||||
|
||||
var afterTime any
|
||||
@@ -372,6 +378,7 @@ func (s *Store) ListMessages(
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
)
|
||||
return message, err
|
||||
}))
|
||||
|
||||
@@ -100,18 +100,16 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
||||
owner, chatID := createOwnerAndChat(t, store)
|
||||
key := uuid.NewString()
|
||||
|
||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour)
|
||||
if err != nil || replay {
|
||||
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
|
||||
}
|
||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
|
||||
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour)
|
||||
if err != nil || !replay {
|
||||
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("replay message = %v, want %v", second, first)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
s.Equal(first, second)
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
|
||||
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
|
||||
}
|
||||
if count := messageCount(t, pool, chatID); count != 1 {
|
||||
@@ -119,6 +117,62 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash")
|
||||
s.Require().NoError(err)
|
||||
attachment := model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
}
|
||||
created, err := s.store.CreateAttachment(context.Background(), attachment)
|
||||
s.Require().NoError(err)
|
||||
s.Equal("ready", created.Status)
|
||||
s.Empty(created.Variants)
|
||||
s.Nil(created.ErrorCode)
|
||||
|
||||
visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(created, visible)
|
||||
_, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID)
|
||||
s.ErrorIs(err, postgres.ErrNotFound)
|
||||
|
||||
message, replay, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.False(replay)
|
||||
s.Equal([]string{created.ID}, message.AttachmentIDs)
|
||||
|
||||
messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10)
|
||||
s.Require().NoError(err)
|
||||
s.Len(messages, 1)
|
||||
s.Equal([]string{created.ID}, messages[0].AttachmentIDs)
|
||||
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(referenced)
|
||||
deleted, err := s.store.DeleteAttachmentIfUnreferencedBefore(context.Background(), created.ID, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.False(deleted)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
|
||||
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
|
||||
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
attachment, err := s.store.CreateAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.ErrorIs(err, postgres.ErrNotFound)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||
t := s.T()
|
||||
pool := s.pool
|
||||
@@ -139,7 +193,7 @@ func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
|
||||
go func() {
|
||||
defer group.Done()
|
||||
message, replay, err := stores[worker%len(stores)].CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
|
||||
context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour,
|
||||
)
|
||||
results <- result{message: message, replay: replay, err: err}
|
||||
}()
|
||||
@@ -186,13 +240,13 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
||||
}
|
||||
key := uuid.NewString()
|
||||
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("owner CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("second user CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
|
||||
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil {
|
||||
t.Fatalf("other chat CreateMessage() error = %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(context.Background(), `
|
||||
@@ -202,7 +256,7 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
|
||||
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
|
||||
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour)
|
||||
if err != nil || replay || message.Text != "after expiry" {
|
||||
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
|
||||
}
|
||||
@@ -212,7 +266,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
||||
t := s.T()
|
||||
owner, chatID := createOwnerAndChat(t, s.store)
|
||||
expiredKey := uuid.NewString()
|
||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
|
||||
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil {
|
||||
t.Fatalf("create expired message: %v", err)
|
||||
}
|
||||
if _, err := s.pool.Exec(context.Background(), `
|
||||
@@ -223,7 +277,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
|
||||
t.Fatalf("expire idempotency key: %v", err)
|
||||
}
|
||||
if _, _, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
|
||||
context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour,
|
||||
); err != nil {
|
||||
t.Fatalf("create new message: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
)
|
||||
|
||||
type MetadataStore interface {
|
||||
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
|
||||
AttachmentReferenced(context.Context, string) (bool, error)
|
||||
DeleteAttachmentIfUnreferencedBefore(context.Context, string, time.Time) (bool, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
List(context.Context, string, string, int) ([]objectstore.Object, string, error)
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Prefix string
|
||||
PageSize int
|
||||
GraceTime time.Duration
|
||||
Apply bool
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
MetadataScanned int `json:"metadata_scanned"`
|
||||
ObjectsScanned int `json:"objects_scanned"`
|
||||
Consistent int `json:"consistent"`
|
||||
FreshOrphanObjects int `json:"fresh_orphan_objects"`
|
||||
OldOrphanObjects int `json:"old_orphan_objects"`
|
||||
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
|
||||
FreshMissingObjects int `json:"fresh_missing_objects"`
|
||||
ReferencedMissingObjects int `json:"referenced_missing_objects"`
|
||||
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
|
||||
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
metadata MetadataStore
|
||||
objects ObjectStore
|
||||
config Config
|
||||
}
|
||||
|
||||
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
|
||||
if config.PageSize < 1 || config.PageSize > 1000 {
|
||||
return nil, fmt.Errorf("page size must be between 1 and 1000")
|
||||
}
|
||||
if config.GraceTime < 0 {
|
||||
return nil, fmt.Errorf("grace time must not be negative")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &Reconciler{metadata: metadata, objects: objects, config: config}, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) Run(ctx context.Context) (Report, error) {
|
||||
metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
cutoff := r.config.Now().Add(-r.config.GraceTime)
|
||||
var report Report
|
||||
for metadata.current != nil || objects.current != nil {
|
||||
switch {
|
||||
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
|
||||
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
|
||||
if err := r.handleMissingObject(ctx, *metadata.current, cutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := metadata.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
default:
|
||||
report.MetadataScanned++
|
||||
report.ObjectsScanned++
|
||||
report.Consistent++
|
||||
if err := metadata.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return report, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.Object, cutoff time.Time, report *Report) error {
|
||||
report.ObjectsScanned++
|
||||
if object.LastModified.After(cutoff) {
|
||||
report.FreshOrphanObjects++
|
||||
return nil
|
||||
}
|
||||
report.OldOrphanObjects++
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
if err := r.objects.Delete(ctx, object.Key); err != nil {
|
||||
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
report.DeletedOrphanObjects++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Attachment, cutoff time.Time, report *Report) error {
|
||||
report.MetadataScanned++
|
||||
if metadata.CreatedAt.After(cutoff) {
|
||||
report.FreshMissingObjects++
|
||||
return nil
|
||||
}
|
||||
referenced, err := r.metadata.AttachmentReferenced(ctx, metadata.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check attachment metadata %q: %w", metadata.ID, err)
|
||||
}
|
||||
if referenced {
|
||||
report.ReferencedMissingObjects++
|
||||
return nil
|
||||
}
|
||||
report.OldUnreferencedMetadata++
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
deleted, err := r.metadata.DeleteAttachmentIfUnreferencedBefore(ctx, metadata.ID, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete attachment metadata %q: %w", metadata.ID, err)
|
||||
}
|
||||
if deleted {
|
||||
report.DeletedUnreferencedRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type metadataIterator struct {
|
||||
store MetadataStore
|
||||
pageSize int
|
||||
after string
|
||||
page []model.Attachment
|
||||
index int
|
||||
current *model.Attachment
|
||||
done bool
|
||||
}
|
||||
|
||||
func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) {
|
||||
iterator := &metadataIterator{store: store, pageSize: pageSize}
|
||||
if err := iterator.advance(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return iterator, nil
|
||||
}
|
||||
|
||||
func (i *metadataIterator) advance(ctx context.Context) error {
|
||||
for {
|
||||
if i.index < len(i.page) {
|
||||
i.current = &i.page[i.index]
|
||||
i.index++
|
||||
return nil
|
||||
}
|
||||
if i.done {
|
||||
i.current = nil
|
||||
return nil
|
||||
}
|
||||
page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list attachment metadata: %w", err)
|
||||
}
|
||||
i.page = page
|
||||
i.index = 0
|
||||
i.after = next
|
||||
i.done = next == ""
|
||||
if len(page) == 0 && !i.done {
|
||||
return fmt.Errorf("metadata page is empty but has continuation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type objectIterator struct {
|
||||
store ObjectStore
|
||||
prefix string
|
||||
pageSize int
|
||||
after string
|
||||
page []objectstore.Object
|
||||
index int
|
||||
current *objectstore.Object
|
||||
done bool
|
||||
}
|
||||
|
||||
func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) {
|
||||
iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize}
|
||||
if err := iterator.advance(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return iterator, nil
|
||||
}
|
||||
|
||||
func (i *objectIterator) advance(ctx context.Context) error {
|
||||
for {
|
||||
if i.index < len(i.page) {
|
||||
i.current = &i.page[i.index]
|
||||
i.index++
|
||||
return nil
|
||||
}
|
||||
if i.done {
|
||||
i.current = nil
|
||||
return nil
|
||||
}
|
||||
page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
i.page = page
|
||||
i.index = 0
|
||||
i.after = next
|
||||
i.done = next == ""
|
||||
if len(page) == 0 && !i.done {
|
||||
return fmt.Errorf("object page is empty but has continuation")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_length,
|
||||
DROP CONSTRAINT messages_text_not_blank;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000),
|
||||
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL,
|
||||
owner_id UUID NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
|
||||
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
|
||||
CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
|
||||
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL,
|
||||
position SMALLINT NOT NULL,
|
||||
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments (id, chat_id) ON DELETE RESTRICT
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);
|
||||
Reference in New Issue
Block a user