feat: add S3 attachments and reconciliation

This commit is contained in:
2026-10-02 19:55:52 +03:00
parent 7e60f099d3
commit efaeb593e8
21 changed files with 1623 additions and 85 deletions
+84 -1
View File
@@ -2,20 +2,26 @@ package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
@@ -34,7 +40,10 @@ func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 1 && arguments[0] == "migrate" {
return runMigrations()
}
return fmt.Errorf("usage: messenger [migrate]")
if len(arguments) > 0 && arguments[0] == "reconcile-attachments" {
return runAttachmentReconciliation(arguments[1:], os.Stdout)
}
return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]")
}
func runServer(logger *slog.Logger) error {
@@ -68,16 +77,28 @@ func runServer(logger *slog.Logger) error {
if err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
app := messenger.NewServer(
logger,
postgres.New(pool),
sessions,
passwords,
objects,
settings.SessionTTL,
settings.IdempotencyTTL,
settings.RequestTimeout,
instanceID,
settings.SwaggerPort,
settings.AttachmentMaxBytes,
settings.AttachmentMaxPixels,
settings.AttachmentDecodeMemory,
)
server := &http.Server{
Addr: settings.Address,
@@ -130,3 +151,65 @@ func runMigrations() error {
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
type reconciliationOutput struct {
Mode string `json:"mode"`
GracePeriod string `json:"grace_period"`
PageSize int `json:"page_size"`
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func runAttachmentReconciliation(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
flags.SetOutput(io.Discard)
apply := flags.Bool("apply", false, "apply safe repairs")
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]")
}
settings, err := config.Load()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply,
})
if err != nil {
return err
}
started := time.Now()
report, err := reconciler.Run(ctx)
if err != nil {
return err
}
mode := "dry-run"
if *apply {
mode = "apply"
}
return json.NewEncoder(output).Encode(reconciliationOutput{
Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize,
DurationMillis: time.Since(started).Milliseconds(), Report: report,
})
}
+13 -1
View File
@@ -5,37 +5,49 @@ go 1.26.0
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.0.99
github.com/prometheus/client_golang v1.23.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
+23 -2
View File
@@ -5,10 +5,14 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -23,20 +27,33 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
@@ -49,6 +66,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
@@ -68,6 +87,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+93 -24
View File
@@ -12,29 +12,39 @@ import (
)
type Config struct {
Address string
DatabaseURL string
CacheURL string
SessionTTL time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
RequestTimeout time.Duration
HTTPReadHeaderTimeout time.Duration
HTTPReadTimeout time.Duration
HTTPWriteTimeout time.Duration
HTTPIdleTimeout time.Duration
HTTPShutdownTimeout time.Duration
PasswordMemoryKiB uint32
PasswordIterations uint32
PasswordParallelism uint8
PasswordSaltLength uint32
PasswordKeyLength uint32
SwaggerPort string
Address string
DatabaseURL string
CacheURL string
SessionTTL time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
RequestTimeout time.Duration
HTTPReadHeaderTimeout time.Duration
HTTPReadTimeout time.Duration
HTTPWriteTimeout time.Duration
HTTPIdleTimeout time.Duration
HTTPShutdownTimeout time.Duration
PasswordMemoryKiB uint32
PasswordIterations uint32
PasswordParallelism uint8
PasswordSaltLength uint32
PasswordKeyLength uint32
SwaggerPort string
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
AttachmentMaxBytes int64
AttachmentMaxPixels int64
AttachmentDecodeMemory int64
}
type MigrationConfig struct {
@@ -142,6 +152,50 @@ func Load() (Config, error) {
if err != nil {
return Config{}, err
}
s3Endpoint, err := requiredString(values, "S3_ENDPOINT")
if err != nil {
return Config{}, err
}
s3Region, err := requiredString(values, "S3_REGION")
if err != nil {
return Config{}, err
}
s3Bucket, err := requiredString(values, "S3_BUCKET")
if err != nil {
return Config{}, err
}
s3AccessKey, err := requiredString(values, "S3_ACCESS_KEY")
if err != nil {
return Config{}, err
}
s3SecretKey, err := requiredString(values, "S3_SECRET_KEY")
if err != nil {
return Config{}, err
}
s3UseTLS, err := boolean(values, "S3_USE_TLS")
if err != nil {
return Config{}, err
}
s3Timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
if err != nil {
return Config{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
if err != nil {
return Config{}, err
}
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
if err != nil {
return Config{}, err
}
minimumDecodeMemory := int64(attachmentMaxBytes) + int64(attachmentMaxPixels)*8
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
return Config{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
}
return Config{
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
@@ -155,7 +209,10 @@ func Load() (Config, error) {
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
SwaggerPort: swaggerPort,
SwaggerPort: swaggerPort, S3Endpoint: s3Endpoint, S3Region: s3Region, S3Bucket: s3Bucket,
S3AccessKey: s3AccessKey, S3SecretKey: s3SecretKey, S3UseTLS: s3UseTLS, S3Timeout: s3Timeout,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
}, nil
}
@@ -260,3 +317,15 @@ func integerValue(value string, minimum, maximum int) (int, error) {
}
return parsed, nil
}
func boolean(values *viper.Viper, name string) (bool, error) {
value, err := requiredString(values, name)
if err != nil {
return false, err
}
parsed, err := strconv.ParseBool(value)
if err != nil {
return false, fmt.Errorf("%s: must be true or false", name)
}
return parsed, nil
}
+260
View File
@@ -0,0 +1,260 @@
package messenger
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
}()
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
filename, valid := attachmentFilename(r.URL.Query())
if !valid {
return invalidRequest()
}
if _, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID); errors.Is(err, postgres.ErrNotFound) {
return notFound()
} else if err != nil {
result = "dependency_error"
return unavailable()
}
data, contentType, readErr := readImage(w, r, s.maxUploadBytes, s.maxImagePixels, s.decodeMemory)
if readErr != nil {
return readErr
}
attachmentID, err := newID()
if err != nil {
result = "dependency_error"
return internalError()
}
objectID, err := newID()
if err != nil {
result = "dependency_error"
return internalError()
}
objectKey := "attachments/" + objectID + "/original"
if err := s.objects.Put(r.Context(), objectKey, contentType, data); err != nil {
result = "dependency_error"
return unavailable()
}
digest := sha256.Sum256(data)
attachment, err := s.messengerStore.CreateAttachment(r.Context(), model.Attachment{
ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: objectKey,
})
if err != nil {
result = "dependency_error"
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.requestTTL)
defer cancel()
stored, lookupErr := s.messengerStore.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
if lookupErr == nil {
result = "success"
s.metrics.attachmentBytes.Add(float64(len(data)))
writeJSON(w, http.StatusCreated, stored)
return nil
}
if !errors.Is(lookupErr, postgres.ErrNotFound) {
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", lookupErr)
return unavailable()
}
if cleanupErr := s.objects.Delete(cleanupCtx, objectKey); cleanupErr != nil {
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", cleanupErr)
}
if errors.Is(err, postgres.ErrNotFound) {
return notFound()
}
return unavailable()
}
result = "success"
s.metrics.attachmentBytes.Add(float64(len(data)))
writeJSON(w, http.StatusCreated, attachment)
return nil
}
func attachmentFilename(query url.Values) (string, bool) {
if len(query) != 1 || len(query["filename"]) != 1 {
return "", false
}
filename := query["filename"][0]
length := utf8.RuneCountInString(filename)
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
return "", false
}
for _, char := range filename {
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
return "", false
}
}
return filename, true
}
func readImage(
w http.ResponseWriter,
r *http.Request,
maxBytes int64,
maxPixels int64,
decodeMemory *semaphore.Weighted,
) ([]byte, string, *apiError) {
mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
}
if r.ContentLength > maxBytes {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
}
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
}
return nil, "", invalidRequest()
}
if len(data) == 0 {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
detected := http.DetectContentType(data)
if detected != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "content type does not match image"}
}
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || imageContentType(format) != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > maxPixels {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image dimensions are too large"}
}
decodeBytes := pixels*8 + int64(len(data))
if !decodeMemory.TryAcquire(decodeBytes) {
w.Header().Set("Retry-After", "1")
return nil, "", &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
}
defer decodeMemory.Release(decodeBytes)
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
return data, mediaType, nil
}
func imageContentType(format string) string {
switch format {
case "png":
return "image/png"
case "jpeg":
return "image/jpeg"
default:
return ""
}
}
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
attachment, apiErr := s.attachmentForUser(r, user.ID)
if apiErr != nil {
return apiErr
}
writeJSON(w, http.StatusOK, attachment)
return nil
}
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
if !originalVariant(r.URL.Query()) {
return invalidRequest()
}
attachment, apiErr := s.attachmentForUser(r, user.ID)
if apiErr != nil {
return apiErr
}
content, err := s.objects.Get(r.Context(), attachment.ObjectKey)
if errors.Is(err, objectstore.ErrNotFound) {
return unavailable()
}
if err != nil {
return unavailable()
}
defer func() { _ = content.Close() }()
w.Header().Set("Content-Type", attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", attachment.ID, "error", err)
}
return nil
}
func originalVariant(query url.Values) bool {
if len(query) == 0 {
return true
}
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
}
func (s *Server) attachmentForUser(r *http.Request, userID string) (Attachment, *apiError) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
attachmentID, valid := canonicalID(parts[3])
if !valid {
return Attachment{}, invalidRequest()
}
attachment, err := s.messengerStore.AttachmentForUser(r.Context(), attachmentID, userID)
if errors.Is(err, postgres.ErrNotFound) {
return Attachment{}, notFound()
}
if err != nil {
return Attachment{}, unavailable()
}
return attachment, nil
}
+1
View File
@@ -14,6 +14,7 @@ const (
errorCodePayloadTooLarge = "payload_too_large"
errorCodeUnavailable = "unavailable"
errorCodeConflict = "conflict"
errorCodeRateLimited = "rate_limited"
)
type apiError struct {
+33 -3
View File
@@ -38,6 +38,9 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
if err := s.sessions.Ping(ctx); err != nil {
return unavailable()
}
if err := s.objects.Ping(ctx); err != nil {
return unavailable()
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
return nil
}
@@ -207,16 +210,17 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
return invalidRequest()
}
var input struct {
Text string `json:"text"`
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}
if err := readJSON(w, r, &input); err != nil {
return err
}
if !validText(input.Text, 4000) {
if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) {
return invalidRequest()
}
message, replay, err := s.messengerStore.CreateMessage(
r.Context(), chatID, user.ID, input.Text, key, s.idempotencyTTL,
r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL,
)
if errors.Is(err, postgres.ErrNotFound) {
return notFound()
@@ -234,6 +238,32 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
return nil
}
func validMessage(text string, attachmentIDs []string) bool {
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 {
return false
}
return len(attachmentIDs) > 0 || strings.TrimSpace(text) != ""
}
func canonicalAttachmentIDs(attachmentIDs []string) bool {
if len(attachmentIDs) > 10 {
return false
}
seen := make(map[string]struct{}, len(attachmentIDs))
for index, attachmentID := range attachmentIDs {
canonical, valid := canonicalID(attachmentID)
if !valid {
return false
}
if _, duplicate := seen[canonical]; duplicate {
return false
}
seen[canonical] = struct{}{}
attachmentIDs[index] = canonical
}
return true
}
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError {
limit, cursor, err := pageParams(r, "message", chatID)
if err != nil {
+12 -1
View File
@@ -2,6 +2,7 @@ package messenger
import (
"context"
"io"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
@@ -15,8 +16,18 @@ type MessengerStore interface {
CreateChat(context.Context, string, []string, string) (model.Chat, error)
ChatForUser(context.Context, string, string) (model.Chat, error)
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error)
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
CreateAttachment(context.Context, model.Attachment) (model.Attachment, error)
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
}
type ObjectStore interface {
Ping(context.Context) error
Put(context.Context, string, string, []byte) error
Get(context.Context, string) (io.ReadCloser, error)
Delete(context.Context, string) error
}
type SessionStore interface {
+36 -5
View File
@@ -12,11 +12,14 @@ import (
)
type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
handler http.Handler
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
attachmentUploads *prometheus.CounterVec
attachmentDuration prometheus.Histogram
attachmentBytes prometheus.Counter
handler http.Handler
}
func newMetrics() *metrics {
@@ -47,12 +50,31 @@ func newMetrics() *metrics {
Name: "idempotency_replays_total",
Help: "Message requests served from an idempotency record.",
}),
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_uploads_total",
Help: "Completed attachment uploads by result.",
}, []string{"result"}),
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "messenger",
Name: "attachment_upload_duration_seconds",
Help: "Attachment upload duration in seconds.",
Buckets: prometheus.DefBuckets,
}),
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_upload_bytes_total",
Help: "Bytes accepted by successful attachment uploads.",
}),
}
registry.MustRegister(
result.requests,
result.duration,
result.inFlight,
result.replays,
result.attachmentUploads,
result.attachmentDuration,
result.attachmentBytes,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
@@ -79,5 +101,14 @@ func routeTemplate(path string) string {
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
return "/api/v1/chats/{chat_id}/messages"
}
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
return "/api/v1/chats/{chat_id}/attachments"
}
if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
return "/api/v1/attachments/{attachment_id}"
}
if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
return "/api/v1/attachments/{attachment_id}/content"
}
return "unknown"
}
+1
View File
@@ -8,6 +8,7 @@ import (
type User = model.User
type Chat = model.Chat
type Message = model.Message
type Attachment = model.Attachment
func newID() (string, error) {
id, err := uuid.NewRandom()
+6
View File
@@ -33,6 +33,12 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError {
handler = s.chat
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" {
handler = s.messages
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" {
handler = s.uploadAttachment
} else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" {
handler = s.attachmentMetadata
} else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" {
handler = s.attachmentContent
} else {
return notFound()
}
+14 -2
View File
@@ -6,18 +6,24 @@ import (
"net/http"
"sync/atomic"
"time"
"golang.org/x/sync/semaphore"
)
type Server struct {
messengerStore MessengerStore
sessions SessionStore
passwords PasswordService
objects ObjectStore
sessionTTL time.Duration
idempotencyTTL time.Duration
requestTTL time.Duration
instanceID string
logger *slog.Logger
swaggerPort string
maxUploadBytes int64
maxImagePixels int64
decodeMemory *semaphore.Weighted
metrics *metrics
ready atomic.Bool
}
@@ -37,16 +43,22 @@ func NewServer(
messages MessengerStore,
sessions SessionStore,
passwords PasswordService,
objects ObjectStore,
sessionTTL time.Duration,
idempotencyTTL time.Duration,
requestTTL time.Duration,
instanceID string,
swaggerPort string,
maxUploadBytes int64,
maxImagePixels int64,
decodeMemoryBytes int64,
) *Server {
s := &Server{
messengerStore: messages, sessions: sessions, passwords: passwords,
messengerStore: messages, sessions: sessions, passwords: passwords, objects: objects,
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, metrics: newMetrics(),
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort,
maxUploadBytes: maxUploadBytes, maxImagePixels: maxImagePixels,
decodeMemory: semaphore.NewWeighted(decodeMemoryBytes), metrics: newMetrics(),
}
s.ready.Store(true)
return s
+210 -11
View File
@@ -1,9 +1,13 @@
package messenger
import (
"bytes"
"context"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"net/http"
@@ -13,6 +17,7 @@ import (
"time"
"github.com/stretchr/testify/suite"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
@@ -34,14 +39,19 @@ func (s *MessengerSuite) SetupTest() {
}
type messengerStoreStub struct {
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
attachment model.Attachment
attachmentErr error
storedAttachment model.Attachment
storedAttachmentExists bool
storedAttachmentErr error
}
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
@@ -63,12 +73,57 @@ func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Ch
func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) {
return nil, false, nil
}
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, string, time.Duration) (model.Message, bool, error) {
func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) {
return s.message, s.messageReplay, s.messageErr
}
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
return nil, false, nil
}
func (s messengerStoreStub) CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) {
return s.attachment, s.attachmentErr
}
func (s messengerStoreStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
if s.storedAttachmentErr != nil {
return model.Attachment{}, s.storedAttachmentErr
}
if !s.storedAttachmentExists {
return model.Attachment{}, postgres.ErrNotFound
}
return s.storedAttachment, nil
}
func (s messengerStoreStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
return s.attachment, s.attachmentErr
}
type objectStoreStub struct {
pingErr error
putErr error
getData []byte
getErr error
deleteErr error
putKey *string
deletedKey *string
}
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
func (s objectStoreStub) Put(_ context.Context, key string, _ string, _ []byte) error {
if s.putKey != nil {
*s.putKey = key
}
return s.putErr
}
func (s objectStoreStub) Get(context.Context, string) (io.ReadCloser, error) {
if s.getErr != nil {
return nil, s.getErr
}
return io.NopCloser(bytes.NewReader(s.getData)), nil
}
func (s objectStoreStub) Delete(_ context.Context, key string) error {
if s.deletedKey != nil {
*s.deletedKey = key
}
return s.deleteErr
}
type sessionStub struct {
pingErr error
@@ -100,14 +155,19 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
name string
store messengerStoreStub
sessions sessionStub
objects objectStoreStub
}{
{name: "postgres", store: messengerStoreStub{pingErr: errDependency}},
{name: "valkey", sessions: sessionStub{pingErr: errDependency}},
{name: "s3", objects: objectStoreStub{pingErr: errDependency}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := s.newTestServer(tt.store, tt.sessions, passwordsStub{})
server := NewServer(
s.logger, tt.store, tt.sessions, passwordsStub{}, tt.objects, time.Hour, 24*time.Hour,
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
response := serveRequest(server, http.MethodGet, "/health/ready", "")
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
@@ -160,6 +220,18 @@ func (s *MessengerSuite) TestAuthenticationChallenges() {
}
}
func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() {
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
response := serveRequest(
server,
http.MethodGet,
"/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid",
"",
)
s.Equal(http.StatusUnauthorized, response.Code)
s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate"))
}
func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() {
t := s.T()
server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{})
@@ -242,8 +314,135 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
}
}
func (s *MessengerSuite) TestImageValidation() {
data := testPNG(s.T(), 2, 2)
tests := []struct {
name string
contentType string
data []byte
maxBytes int64
maxPixels int64
wantStatus int
}{
{name: "valid", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 4},
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, wantStatus: http.StatusRequestEntityTooLarge},
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, wantStatus: http.StatusRequestEntityTooLarge},
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
}
for _, tt := range tests {
s.Run(tt.name, func() {
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(tt.data))
request.Header.Set("Content-Type", tt.contentType)
response := httptest.NewRecorder()
actual, _, apiErr := readImage(
response, request, tt.maxBytes, tt.maxPixels, semaphore.NewWeighted(384*1024*1024),
)
if tt.wantStatus == 0 {
s.Require().Nil(apiErr)
s.Equal(tt.data, actual)
return
}
s.Require().NotNil(apiErr)
s.Equal(tt.wantStatus, apiErr.status)
})
}
}
func (s *MessengerSuite) TestImageValidationLimitsConcurrentDecodeMemory() {
data := testPNG(s.T(), 2, 2)
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(data))
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
_, _, apiErr := readImage(response, request, 1024, 4, semaphore.NewWeighted(1))
s.Require().NotNil(apiErr)
s.Equal(http.StatusTooManyRequests, apiErr.status)
s.Equal("1", response.Header().Get("Retry-After"))
}
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
s.False(valid, filename)
}
filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}})
s.True(valid)
s.Equal("фото.png", filename)
}
func (s *MessengerSuite) TestUploadCompensatesObjectWhenMetadataWriteFails() {
var putKey, deletedKey string
store := messengerStoreStub{
userByID: model.User{ID: "00000000-0000-0000-0000-000000000001"},
attachmentErr: errDependency,
}
objects := objectStoreStub{putKey: &putKey, deletedKey: &deletedKey}
server := NewServer(
s.logger, store, sessionStub{userID: store.userByID.ID}, passwordsStub{}, objects,
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
data := testPNG(s.T(), 2, 2)
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
bytes.NewReader(data),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
s.Equal(http.StatusServiceUnavailable, response.Code)
s.NotEmpty(putKey)
s.Equal(putKey, deletedKey)
}
func (s *MessengerSuite) TestUploadKeepsObjectWhenMetadataCommitSucceeded() {
var deletedKey string
stored := model.Attachment{
ID: "00000000-0000-0000-0000-000000000003",
ChatID: "00000000-0000-0000-0000-000000000002",
OwnerID: "00000000-0000-0000-0000-000000000001",
Filename: "photo.png", ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("0", 64),
Status: "ready", Variants: []model.Variant{}, CreatedAt: time.Now(), UpdatedAt: time.Now(),
}
store := messengerStoreStub{
userByID: model.User{ID: stored.OwnerID}, attachmentErr: errDependency,
storedAttachment: stored, storedAttachmentExists: true,
}
server := NewServer(
s.logger, store, sessionStub{userID: stored.OwnerID}, passwordsStub{}, objectStoreStub{deletedKey: &deletedKey},
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
data := testPNG(s.T(), 2, 2)
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
bytes.NewReader(data),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
s.Equal(http.StatusCreated, response.Code)
s.Empty(deletedKey)
}
func testPNG(t *testing.T, width, height int) []byte {
t.Helper()
var data bytes.Buffer
imageData := image.NewRGBA(image.Rect(0, 0, width, height))
imageData.Set(0, 0, color.White)
if err := png.Encode(&data, imageData); err != nil {
t.Fatalf("encode PNG: %v", err)
}
return data.Bytes()
}
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
return NewServer(s.logger, messages, sessions, passwords, time.Hour, 24*time.Hour, time.Second, "test-instance", "8081")
return NewServer(
s.logger, messages, sessions, passwords, objectStoreStub{}, time.Hour, 24*time.Hour,
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
}
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
+68 -11
View File
@@ -27,11 +27,37 @@ type Chat struct {
}
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Width int `json:"width"`
Height int `json:"height"`
}
type Position struct {
@@ -54,11 +80,27 @@ type chatJSON struct {
}
type messageJSON struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt string `json:"created_at"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt string `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type attachmentJSON struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
func (u User) MarshalJSON() ([]byte, error) {
@@ -70,7 +112,22 @@ func (c Chat) MarshalJSON() ([]byte, error) {
}
func (m Message) MarshalJSON() ([]byte, error) {
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt)})
attachments := m.AttachmentIDs
if attachments == nil {
attachments = []string{}
}
return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments})
}
func (a Attachment) MarshalJSON() ([]byte, error) {
variants := a.Variants
if variants == nil {
variants = []Variant{}
}
return json.Marshal(attachmentJSON{
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt),
})
}
func FormatTime(value time.Time) string {
+180
View File
@@ -0,0 +1,180 @@
package objectstore
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
var ErrNotFound = errors.New("object not found")
type Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
UseTLS bool
Timeout time.Duration
}
type Object struct {
Key string
Size int64
LastModified time.Time
}
type Store struct {
client *minio.Client
bucket string
timeout time.Duration
}
func New(config Config) (*Store, error) {
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
if err != nil {
return nil, err
}
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
return nil, errors.New("invalid object store configuration")
}
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
Secure: config.UseTLS,
Region: config.Region,
})
if err != nil {
return nil, fmt.Errorf("create S3 client: %w", err)
}
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
}
func endpointHost(value string, useTLS bool) (string, error) {
value = strings.TrimSpace(value)
if !strings.Contains(value, "://") {
if value == "" {
return "", errors.New("S3 endpoint is required")
}
return value, nil
}
parsed, err := url.Parse(value)
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
return "", errors.New("invalid S3 endpoint")
}
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
}
return parsed.Host, nil
}
func (s *Store) Ping(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
exists, err := s.client.BucketExists(ctx, s.bucket)
if err != nil {
return fmt.Errorf("check S3 bucket: %w", err)
}
if !exists {
return errors.New("S3 bucket does not exist")
}
return nil
}
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
_, err := s.client.PutObject(
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
)
if err != nil {
return fmt.Errorf("put S3 object: %w", err)
}
return nil
}
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
cancel()
return nil, objectError("get S3 object", err)
}
if _, err := object.Stat(); err != nil {
cancel()
_ = object.Close()
return nil, objectError("stat S3 object", err)
}
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
}
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return Object{}, objectError("stat S3 object", err)
}
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
}
func (s *Store) Delete(ctx context.Context, key string) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
return objectError("delete S3 object", err)
}
return nil
}
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]Object, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", errors.New("object list limit must be between 1 and 1000")
}
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
objects := make([]Object, 0, limit+1)
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
for info := range s.client.ListObjects(ctx, s.bucket, options) {
if info.Err != nil {
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
}
objects = append(objects, Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
if len(objects) == limit+1 {
cancel()
break
}
}
var next string
if len(objects) > limit {
objects = objects[:limit]
next = objects[len(objects)-1].Key
}
return objects, next, nil
}
type timeoutReader struct {
io.ReadCloser
cancel context.CancelFunc
}
func (r *timeoutReader) Close() error {
err := r.ReadCloser.Close()
r.cancel()
return err
}
func objectError(action string, err error) error {
response := minio.ToErrorResponse(err)
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
return ErrNotFound
}
return fmt.Errorf("%s: %w", action, err)
}
+155
View File
@@ -0,0 +1,155 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func (s *Store) CreateAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
const query = `
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key
)
SELECT $1, $2, $3, $4, $5, $6, $7, $8
WHERE EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
)
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, created_at, updated_at`
err := s.pool.QueryRow(
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("create attachment: %w", err)
}
return readyAttachment(attachment), nil
}
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
const query = `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
a.sha256, a.object_key, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
WHERE a.id = $1`
var attachment model.Attachment
err := s.pool.QueryRow(ctx, query, attachmentID, userID).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
}
return readyAttachment(attachment), nil
}
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
const query = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, created_at, updated_at
FROM attachments
WHERE id = $1 AND object_key = $2`
var attachment model.Attachment
err := s.pool.QueryRow(ctx, query, attachmentID, objectKey).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
}
return readyAttachment(attachment), nil
}
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
}
const query = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, created_at, updated_at
FROM attachments
WHERE object_key > $1
ORDER BY object_key
LIMIT $2`
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
if err != nil {
return nil, "", fmt.Errorf("list attachments: %w", err)
}
defer rows.Close()
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
var attachment model.Attachment
err := row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
return readyAttachment(attachment), err
}))
if err != nil {
return nil, "", fmt.Errorf("scan attachments: %w", err)
}
var next string
if len(attachments) > limit {
attachments = attachments[:limit]
next = attachments[len(attachments)-1].ObjectKey
}
return attachments, next, nil
}
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
var referenced bool
if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil {
return false, fmt.Errorf("check attachment references: %w", err)
}
return referenced, nil
}
func (s *Store) DeleteAttachmentIfUnreferencedBefore(
ctx context.Context,
attachmentID string,
createdBefore time.Time,
) (bool, error) {
const query = `
DELETE FROM attachments a
WHERE a.id = $1
AND a.created_at <= $2
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
result, err := s.pool.Exec(ctx, query, attachmentID, createdBefore)
if err != nil {
return false, fmt.Errorf("delete old unreferenced attachment: %w", err)
}
return result.RowsAffected() == 1, nil
}
func readyAttachment(attachment model.Attachment) model.Attachment {
attachment.Status = "ready"
attachment.Variants = []model.Variant{}
return attachment
}
+72 -6
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"time"
@@ -21,6 +22,7 @@ func (s *Store) CreateMessage(
chatID string,
senderID string,
text string,
attachmentIDs []string,
idempotencyKey string,
idempotencyTTL time.Duration,
) (model.Message, bool, error) {
@@ -46,13 +48,23 @@ func (s *Store) CreateMessage(
if !visible {
return model.Message{}, false, ErrNotFound
}
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
if err != nil {
return model.Message{}, false, err
}
if !validAttachments {
return model.Message{}, false, ErrNotFound
}
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
return model.Message{}, false, err
}
requestHash = sha256.Sum256([]byte(text))
requestHash, err = messageRequestHash(text, attachmentIDs)
if err != nil {
return model.Message{}, false, err
}
message, replay, err = createIdempotentMessage(
ctx, tx, chatID, senderID, text, idempotencyKey, requestHash[:], idempotencyTTL,
ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL,
)
if err != nil {
return model.Message{}, false, err
@@ -63,6 +75,35 @@ func (s *Store) CreateMessage(
return message, replay, nil
}
func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) {
if attachmentIDs == nil {
attachmentIDs = []string{}
}
payload, err := json.Marshal(struct {
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}{text, attachmentIDs})
if err != nil {
return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err)
}
return sha256.Sum256(payload), nil
}
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
if len(attachmentIDs) == 0 {
return true, nil
}
const query = `
SELECT count(*)
FROM attachments
WHERE chat_id = $1 AND id::text = ANY($2)`
var count int
if err := tx.QueryRow(ctx, query, chatID, attachmentIDs).Scan(&count); err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
}
return count == len(attachmentIDs), nil
}
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
const query = `
SELECT EXISTS (
@@ -84,6 +125,7 @@ func createIdempotentMessage(
chatID string,
senderID string,
text string,
attachmentIDs []string,
idempotencyKey string,
requestHash []byte,
idempotencyTTL time.Duration,
@@ -126,7 +168,7 @@ func createIdempotentMessage(
continue
}
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text)
message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs)
return message, false, err
}
}
@@ -235,6 +277,7 @@ func insertMessage(
chatID string,
senderID string,
text string,
attachmentIDs []string,
) (model.Message, error) {
const query = `
INSERT INTO messages (id, chat_id, sender_id, text)
@@ -255,14 +298,36 @@ func insertMessage(
if err != nil {
return model.Message{}, fmt.Errorf("insert message: %w", err)
}
if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil {
return model.Message{}, err
}
message.AttachmentIDs = append([]string(nil), attachmentIDs...)
return message, nil
}
func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error {
const query = `
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES ($1, $2, $3, $4)`
for position, attachmentID := range attachmentIDs {
if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil {
return fmt.Errorf("insert message attachment: %w", err)
}
}
return nil
}
func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) {
const query = `
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE id = $1`
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id::text
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
)
FROM messages m
WHERE m.id = $1`
var (
message model.Message
@@ -274,6 +339,7 @@ func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Messag
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, fmt.Errorf("idempotency result message not found")
+12 -5
View File
@@ -344,11 +344,17 @@ func (s *Store) ListMessages(
}
const query = `
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid))
ORDER BY created_at, id
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id::text
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
)
FROM messages m
WHERE m.chat_id = $1
AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid))
ORDER BY m.created_at, m.id
LIMIT $4`
var afterTime any
@@ -372,6 +378,7 @@ func (s *Store) ListMessages(
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
)
return message, err
}))
+67 -13
View File
@@ -100,18 +100,16 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
owner, chatID := createOwnerAndChat(t, store)
key := uuid.NewString()
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour)
if err != nil || replay {
t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err)
}
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", key, 24*time.Hour)
second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour)
if err != nil || !replay {
t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err)
}
if first != second {
t.Fatalf("replay message = %v, want %v", second, first)
}
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
s.Equal(first, second)
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) {
t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err)
}
if count := messageCount(t, pool, chatID); count != 1 {
@@ -119,6 +117,62 @@ func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() {
}
}
func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash")
s.Require().NoError(err)
attachment := model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
}
created, err := s.store.CreateAttachment(context.Background(), attachment)
s.Require().NoError(err)
s.Equal("ready", created.Status)
s.Empty(created.Variants)
s.Nil(created.ErrorCode)
visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID)
s.Require().NoError(err)
s.Equal(created, visible)
_, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID)
s.ErrorIs(err, postgres.ErrNotFound)
message, replay, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour,
)
s.Require().NoError(err)
s.False(replay)
s.Equal([]string{created.ID}, message.AttachmentIDs)
messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10)
s.Require().NoError(err)
s.Len(messages, 1)
s.Equal([]string{created.ID}, messages[0].AttachmentIDs)
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
s.Require().NoError(err)
s.True(referenced)
deleted, err := s.store.DeleteAttachmentIfUnreferencedBefore(context.Background(), created.ID, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(deleted)
}
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
s.Require().NoError(err)
attachment, err := s.store.CreateAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
_, _, err = s.store.CreateMessage(
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
)
s.ErrorIs(err, postgres.ErrNotFound)
}
func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
t := s.T()
pool := s.pool
@@ -139,7 +193,7 @@ func (s *PostgresSuite) TestConcurrentMessageIdempotency() {
go func() {
defer group.Done()
message, replay, err := stores[worker%len(stores)].CreateMessage(
context.Background(), chatID, owner.ID, "concurrent body", key, 24*time.Hour,
context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour,
)
results <- result{message: message, replay: replay, err: err}
}()
@@ -186,13 +240,13 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
}
key := uuid.NewString()
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", key, 24*time.Hour); err != nil {
if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil {
t.Fatalf("owner CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", key, 24*time.Hour); err != nil {
if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil {
t.Fatalf("second user CreateMessage() error = %v", err)
}
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", key, 24*time.Hour); err != nil {
if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil {
t.Fatalf("other chat CreateMessage() error = %v", err)
}
if _, err := pool.Exec(context.Background(), `
@@ -202,7 +256,7 @@ func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() {
WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil {
t.Fatalf("expire idempotency key: %v", err)
}
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", key, 24*time.Hour)
message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour)
if err != nil || replay || message.Text != "after expiry" {
t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err)
}
@@ -212,7 +266,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
t := s.T()
owner, chatID := createOwnerAndChat(t, s.store)
expiredKey := uuid.NewString()
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", expiredKey, 24*time.Hour); err != nil {
if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil {
t.Fatalf("create expired message: %v", err)
}
if _, err := s.pool.Exec(context.Background(), `
@@ -223,7 +277,7 @@ func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() {
t.Fatalf("expire idempotency key: %v", err)
}
if _, _, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "new message", uuid.NewString(), 24*time.Hour,
context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour,
); err != nil {
t.Fatalf("create new message: %v", err)
}
+236
View File
@@ -0,0 +1,236 @@
package reconcile
import (
"context"
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
)
type MetadataStore interface {
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
AttachmentReferenced(context.Context, string) (bool, error)
DeleteAttachmentIfUnreferencedBefore(context.Context, string, time.Time) (bool, error)
}
type ObjectStore interface {
List(context.Context, string, string, int) ([]objectstore.Object, string, error)
Delete(context.Context, string) error
}
type Config struct {
Prefix string
PageSize int
GraceTime time.Duration
Apply bool
Now func() time.Time
}
type Report struct {
MetadataScanned int `json:"metadata_scanned"`
ObjectsScanned int `json:"objects_scanned"`
Consistent int `json:"consistent"`
FreshOrphanObjects int `json:"fresh_orphan_objects"`
OldOrphanObjects int `json:"old_orphan_objects"`
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
FreshMissingObjects int `json:"fresh_missing_objects"`
ReferencedMissingObjects int `json:"referenced_missing_objects"`
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
}
type Reconciler struct {
metadata MetadataStore
objects ObjectStore
config Config
}
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
if config.PageSize < 1 || config.PageSize > 1000 {
return nil, fmt.Errorf("page size must be between 1 and 1000")
}
if config.GraceTime < 0 {
return nil, fmt.Errorf("grace time must not be negative")
}
if config.Now == nil {
config.Now = time.Now
}
return &Reconciler{metadata: metadata, objects: objects, config: config}, nil
}
func (r *Reconciler) Run(ctx context.Context) (Report, error) {
metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize)
if err != nil {
return Report{}, err
}
objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize)
if err != nil {
return Report{}, err
}
cutoff := r.config.Now().Add(-r.config.GraceTime)
var report Report
for metadata.current != nil || objects.current != nil {
switch {
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, &report); err != nil {
return report, err
}
if err := objects.advance(ctx); err != nil {
return report, err
}
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
if err := r.handleMissingObject(ctx, *metadata.current, cutoff, &report); err != nil {
return report, err
}
if err := metadata.advance(ctx); err != nil {
return report, err
}
default:
report.MetadataScanned++
report.ObjectsScanned++
report.Consistent++
if err := metadata.advance(ctx); err != nil {
return report, err
}
if err := objects.advance(ctx); err != nil {
return report, err
}
}
}
return report, nil
}
func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.Object, cutoff time.Time, report *Report) error {
report.ObjectsScanned++
if object.LastModified.After(cutoff) {
report.FreshOrphanObjects++
return nil
}
report.OldOrphanObjects++
if !r.config.Apply {
return nil
}
if err := r.objects.Delete(ctx, object.Key); err != nil {
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
}
report.DeletedOrphanObjects++
return nil
}
func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Attachment, cutoff time.Time, report *Report) error {
report.MetadataScanned++
if metadata.CreatedAt.After(cutoff) {
report.FreshMissingObjects++
return nil
}
referenced, err := r.metadata.AttachmentReferenced(ctx, metadata.ID)
if err != nil {
return fmt.Errorf("check attachment metadata %q: %w", metadata.ID, err)
}
if referenced {
report.ReferencedMissingObjects++
return nil
}
report.OldUnreferencedMetadata++
if !r.config.Apply {
return nil
}
deleted, err := r.metadata.DeleteAttachmentIfUnreferencedBefore(ctx, metadata.ID, cutoff)
if err != nil {
return fmt.Errorf("delete attachment metadata %q: %w", metadata.ID, err)
}
if deleted {
report.DeletedUnreferencedRows++
}
return nil
}
type metadataIterator struct {
store MetadataStore
pageSize int
after string
page []model.Attachment
index int
current *model.Attachment
done bool
}
func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) {
iterator := &metadataIterator{store: store, pageSize: pageSize}
if err := iterator.advance(ctx); err != nil {
return nil, err
}
return iterator, nil
}
func (i *metadataIterator) advance(ctx context.Context) error {
for {
if i.index < len(i.page) {
i.current = &i.page[i.index]
i.index++
return nil
}
if i.done {
i.current = nil
return nil
}
page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize)
if err != nil {
return fmt.Errorf("list attachment metadata: %w", err)
}
i.page = page
i.index = 0
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return fmt.Errorf("metadata page is empty but has continuation")
}
}
}
type objectIterator struct {
store ObjectStore
prefix string
pageSize int
after string
page []objectstore.Object
index int
current *objectstore.Object
done bool
}
func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) {
iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize}
if err := iterator.advance(ctx); err != nil {
return nil, err
}
return iterator, nil
}
func (i *objectIterator) advance(ctx context.Context) error {
for {
if i.index < len(i.page) {
i.current = &i.page[i.index]
i.index++
return nil
}
if i.done {
i.current = nil
return nil
}
page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize)
if err != nil {
return fmt.Errorf("list objects: %w", err)
}
i.page = page
i.index = 0
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return fmt.Errorf("object page is empty but has continuation")
}
}
}
+47
View File
@@ -0,0 +1,47 @@
ALTER TABLE messages
DROP CONSTRAINT messages_text_length,
DROP CONSTRAINT messages_text_not_blank;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000),
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
owner_id UUID NOT NULL,
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
);
CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id UUID NOT NULL,
chat_id UUID NOT NULL,
attachment_id UUID NOT NULL,
position SMALLINT NOT NULL,
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id)
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
REFERENCES attachments (id, chat_id) ON DELETE RESTRICT
);
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);