From cc4ac1ddd722985bac2cf73de81525bb628f8e6b Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 9 Oct 2026 13:13:59 +0300 Subject: [PATCH] refactor: isolate attachment lifecycle --- cmd/lab4experiment/main.go | 38 ++- cmd/messenger/main.go | 200 +------------- internal/bootstrap/migration.go | 25 ++ internal/bootstrap/reconciliation.go | 85 ++++++ internal/bootstrap/reconciliation_test.go | 13 + internal/bootstrap/server.go | 113 ++++++++ internal/config/config.go | 181 ++++++++---- internal/config/config_test.go | 29 ++ internal/messenger/attachment/service.go | 252 +++++++++++++++++ internal/messenger/attachment/service_test.go | 195 +++++++++++++ internal/messenger/attachments.go | 260 ------------------ internal/messenger/httpapi/attachments.go | 164 +++++++++++ internal/messenger/{ => httpapi}/auth.go | 16 +- internal/messenger/{ => httpapi}/errors.go | 2 +- internal/messenger/{ => httpapi}/handlers.go | 33 ++- .../{ => httpapi}/messenger_interface.go | 33 ++- internal/messenger/{ => httpapi}/metrics.go | 2 +- internal/messenger/{ => httpapi}/model.go | 2 +- internal/messenger/{ => httpapi}/request.go | 2 +- internal/messenger/{ => httpapi}/routes.go | 2 +- internal/messenger/{ => httpapi}/server.go | 79 +++--- .../messenger/{ => httpapi}/server_test.go | 217 ++++----------- internal/model/errors.go | 18 ++ internal/model/model.go | 40 ++- internal/objectstore/store.go | 20 +- internal/postgres/attachments.go | 190 +++++++++---- internal/postgres/errors.go | 12 +- internal/postgres/idempotency.go | 17 +- internal/postgres/store_integration_test.go | 10 +- internal/reconcile/reconcile.go | 183 +++++++++--- internal/reconcile/reconcile_test.go | 197 +++++++------ internal/session/session.go | 8 +- migrations/0004_attachment_lifecycle.sql | 16 ++ 33 files changed, 1672 insertions(+), 982 deletions(-) create mode 100644 internal/bootstrap/migration.go create mode 100644 internal/bootstrap/reconciliation.go create mode 100644 internal/bootstrap/reconciliation_test.go create mode 100644 internal/bootstrap/server.go create mode 100644 internal/config/config_test.go create mode 100644 internal/messenger/attachment/service.go create mode 100644 internal/messenger/attachment/service_test.go delete mode 100644 internal/messenger/attachments.go create mode 100644 internal/messenger/httpapi/attachments.go rename internal/messenger/{ => httpapi}/auth.go (86%) rename internal/messenger/{ => httpapi}/errors.go (99%) rename internal/messenger/{ => httpapi}/handlers.go (88%) rename internal/messenger/{ => httpapi}/messenger_interface.go (67%) rename internal/messenger/{ => httpapi}/metrics.go (99%) rename internal/messenger/{ => httpapi}/model.go (96%) rename internal/messenger/{ => httpapi}/request.go (99%) rename internal/messenger/{ => httpapi}/routes.go (98%) rename internal/messenger/{ => httpapi}/server.go (63%) rename internal/messenger/{ => httpapi}/server_test.go (60%) create mode 100644 internal/model/errors.go create mode 100644 migrations/0004_attachment_lifecycle.sql diff --git a/cmd/lab4experiment/main.go b/cmd/lab4experiment/main.go index ed24f21..e8128b7 100644 --- a/cmd/lab4experiment/main.go +++ b/cmd/lab4experiment/main.go @@ -240,6 +240,7 @@ type consistencyResult struct { OrphanDeleted bool `json:"orphan_deleted"` UnusedDeleted bool `json:"unused_metadata_deleted"` InUsePreserved bool `json:"in_use_metadata_preserved"` + AbandonedDeleted bool `json:"abandoned_attachment_deleted"` TotalDurationMillis int64 `json:"total_duration_ms"` } @@ -259,7 +260,7 @@ func runConsistency(arguments []string, output io.Writer) error { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - settings, err := config.Load() + settings, err := config.LoadReconciliation() if err != nil { return err } @@ -281,7 +282,9 @@ func runConsistency(arguments []string, output io.Writer) error { return err } metadata := postgres.New(pool) - clean, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0}) + clean, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour, + }) if err != nil { return err } @@ -320,7 +323,11 @@ func runConsistency(arguments []string, output io.Writer) error { if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil { return err } - keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID}) + abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture))) + if err != nil { + return err + } + keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) if err != nil { return err } @@ -336,7 +343,8 @@ func runConsistency(arguments []string, output io.Writer) error { } fresh, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, Apply: true, + Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, + UnreferencedRetention: time.Hour, Apply: true, }) if err != nil { return err @@ -346,9 +354,19 @@ func runConsistency(arguments []string, output io.Writer) error { return err } freshPreserved := objectExists(ctx, objects, orphanKey) && - attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) + attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) && + attachmentExists(ctx, pool, abandoned.ID) + if _, err := pool.Exec(ctx, ` + UPDATE attachments + SET created_at = clock_timestamp() - INTERVAL '2 hours', + ready_at = clock_timestamp() - INTERVAL '2 hours' + WHERE id = $1`, abandoned.ID); err != nil { + return fmt.Errorf("age abandoned attachment: %w", err) + } - dryRun, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize}) + dryRun, err := reconcile.New(metadata, objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, + }) if err != nil { return err } @@ -357,7 +375,7 @@ func runConsistency(arguments []string, output io.Writer) error { return err } apply, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, Apply: true, + Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true, }) if err != nil { return err @@ -376,9 +394,10 @@ func runConsistency(arguments []string, output io.Writer) error { FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), + AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), TotalDurationMillis: time.Since(started).Milliseconds(), } - if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || + if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 { return fmt.Errorf("unexpected reconciliation result: %+v", result) } @@ -394,7 +413,8 @@ func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconcile func hasAnomalies(report reconcile.Report) bool { return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || - report.OldUnreferencedMetadata != 0 + report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || + report.OldUnreferencedAttachments != 0 } func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 90f25b6..036c498 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -1,215 +1,31 @@ package main import ( - "context" - "encoding/json" - "flag" - "fmt" - "io" + "errors" "log/slog" - "net" - "net/http" "os" - "os/signal" - "syscall" - "time" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session" ) func main() { appLogger := logger.New() if err := run(appLogger, os.Args[1:]); err != nil { - appLogger.Error("server stopped", "error", err) + appLogger.Error("application stopped", "error", err) os.Exit(1) } } func run(logger *slog.Logger, arguments []string) error { if len(arguments) == 0 { - return runServer(logger) + return bootstrap.RunServer(logger) } if len(arguments) == 1 && arguments[0] == "migrate" { - return runMigrations() + return bootstrap.RunMigrations() } - if len(arguments) > 0 && arguments[0] == "reconcile-attachments" { - return runAttachmentReconciliation(arguments[1:], os.Stdout) + if arguments[0] == "reconcile-attachments" { + return bootstrap.RunReconciliation(arguments[1:], os.Stdout) } - return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]") -} - -func runServer(logger *slog.Logger) error { - settings, err := config.Load() - if err != nil { - return err - } - instanceID, err := os.Hostname() - if err != nil { - return err - } - pool, err := database.Open(context.Background(), database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - sessions, err := session.New(settings.CacheURL) - if err != nil { - return err - } - defer func() { _ = sessions.Close() }() - passwords, err := password.New(password.Config{ - MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, - Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, - KeyLength: settings.PasswordKeyLength, - }) - if err != nil { - return err - } - objects, err := objectstore.New(objectstore.Config{ - Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, - AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, - }) - if err != nil { - return err - } - app := messenger.NewServer( - logger, - postgres.New(pool), - sessions, - passwords, - objects, - settings.SessionTTL, - settings.IdempotencyTTL, - settings.RequestTimeout, - instanceID, - settings.SwaggerPort, - settings.AttachmentMaxBytes, - settings.AttachmentMaxPixels, - settings.AttachmentDecodeMemory, - ) - server := &http.Server{ - Addr: settings.Address, - Handler: app.Handler(), - ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, - ReadTimeout: settings.HTTPReadTimeout, - WriteTimeout: settings.HTTPWriteTimeout, - IdleTimeout: settings.HTTPIdleTimeout, - } - listener, err := net.Listen("tcp", server.Addr) - if err != nil { - return err - } - serveResult := make(chan error, 1) - go func() { - serveResult <- server.Serve(listener) - }() - logger.Info("server started", "address", server.Addr) - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - select { - case err := <-serveResult: - return err - case <-ctx.Done(): - logger.Info("shutdown requested") - app.SetReady(false) - shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) - defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil { - _ = server.Close() - return err - } - return nil - } -} - -func runMigrations() error { - settings, err := config.LoadMigration() - if err != nil { - return err - } - pool, err := database.Open(context.Background(), database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - return database.Migrate(context.Background(), pool, settings.MigrationTimeout) -} - -type reconciliationOutput struct { - Mode string `json:"mode"` - GracePeriod string `json:"grace_period"` - PageSize int `json:"page_size"` - DurationMillis int64 `json:"duration_ms"` - Report reconcile.Report `json:"report"` -} - -func runAttachmentReconciliation(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) - flags.SetOutput(io.Discard) - apply := flags.Bool("apply", false, "apply safe repairs") - gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") - pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]") - } - - settings, err := config.Load() - if err != nil { - return err - } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - pool, err := database.Open(ctx, database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - objects, err := objectstore.New(objectstore.Config{ - Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, - AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, - }) - if err != nil { - return err - } - reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply, - }) - if err != nil { - return err - } - started := time.Now() - report, err := reconciler.Run(ctx) - if err != nil { - return err - } - mode := "dry-run" - if *apply { - mode = "apply" - } - return json.NewEncoder(output).Encode(reconciliationOutput{ - Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize, - DurationMillis: time.Since(started).Milliseconds(), Report: report, - }) + return errors.New("usage: messenger [migrate|reconcile-attachments]") } diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go new file mode 100644 index 0000000..3fa1d8f --- /dev/null +++ b/internal/bootstrap/migration.go @@ -0,0 +1,25 @@ +package bootstrap + +import ( + "context" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database" +) + +func RunMigrations() error { + settings, err := config.LoadMigration() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + return database.Migrate(context.Background(), pool, settings.MigrationTimeout) +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go new file mode 100644 index 0000000..6e6f2d3 --- /dev/null +++ b/internal/bootstrap/reconciliation.go @@ -0,0 +1,85 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "flag" + "io" + "os/signal" + "syscall" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile" +) + +type reconciliationOutput struct { + Mode string `json:"mode"` + GracePeriod string `json:"grace_period"` + UnreferencedRetention string `json:"unreferenced_retention"` + PageSize int `json:"page_size"` + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func RunReconciliation(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) + flags.SetOutput(io.Discard) + apply := flags.Bool("apply", false, "apply safe repairs") + gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") + retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments") + pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]") + } + if *apply && *gracePeriod <= 0 { + return errors.New("grace-period must be positive in apply mode") + } + settings, err := config.LoadReconciliation() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + UnreferencedRetention: *retention, Apply: *apply, + }) + if err != nil { + return err + } + started := time.Now() + report, err := reconciler.Run(ctx) + if err != nil { + return err + } + mode := "dry-run" + if *apply { + mode = "apply" + } + return json.NewEncoder(output).Encode(reconciliationOutput{ + Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(), + PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report, + }) +} diff --git a/internal/bootstrap/reconciliation_test.go b/internal/bootstrap/reconciliation_test.go new file mode 100644 index 0000000..9a9d3b0 --- /dev/null +++ b/internal/bootstrap/reconciliation_test.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) { + err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard) + require.EqualError(t, err, "grace-period must be positive in apply mode") +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go new file mode 100644 index 0000000..e21e571 --- /dev/null +++ b/internal/bootstrap/server.go @@ -0,0 +1,113 @@ +package bootstrap + +import ( + "context" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session" +) + +func RunServer(logger *slog.Logger) error { + settings, err := config.LoadServer() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), databaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + sessions, err := session.New(settings.CacheURL) + if err != nil { + return err + } + defer func() { _ = sessions.Close() }() + passwords, err := password.New(password.Config{ + MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, + Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, + KeyLength: settings.PasswordKeyLength, + }) + if err != nil { + return err + } + objects, err := objectstore.New(objectStoreConfig(settings)) + if err != nil { + return err + } + repository := postgres.New(pool) + attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, + CleanupTimeout: settings.RequestTimeout, + }) + if err != nil { + return err + } + app := httpapi.NewServer(logger, httpapi.Dependencies{ + Users: repository, Chats: repository, Messages: repository, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + }, httpapi.Config{ + SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, + RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, + }) + server := &http.Server{ + Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, + ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout, + IdleTimeout: settings.HTTPIdleTimeout, + } + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + logger.Info("server started", "address", server.Addr) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + select { + case err := <-serveResult: + return err + case <-ctx.Done(): + logger.Info("shutdown requested") + app.SetReady(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + _ = server.Close() + return err + } + return nil + } +} + +func databaseConfig(settings config.ServerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func objectStoreConfig(settings config.ServerConfig) objectstore.Config { + return objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 2fcfab3..dcf5b9a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/viper" ) -type Config struct { +type ServerConfig struct { Address string DatabaseURL string CacheURL string @@ -44,6 +44,7 @@ type Config struct { S3Timeout time.Duration AttachmentMaxBytes int64 AttachmentMaxPixels int64 + AttachmentBodyMemory int64 AttachmentDecodeMemory int64 } @@ -58,6 +59,23 @@ type MigrationConfig struct { MigrationTimeout time.Duration } +type ReconciliationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration +} + type databaseConfig struct { url string maxConns int @@ -68,136 +86,129 @@ type databaseConfig struct { dependencyTimeout time.Duration } -func Load() (Config, error) { +type objectStorageConfig struct { + endpoint string + region string + bucket string + accessKey string + secretKey string + useTLS bool + timeout time.Duration +} + +func LoadServer() (ServerConfig, error) { values := viper.New() values.AutomaticEnv() host, err := requiredString(values, "APP_HOST") if err != nil { - return Config{}, err + return ServerConfig{}, err } port, err := requiredPort(values, "APP_PORT") if err != nil { - return Config{}, err + return ServerConfig{}, err } database, err := loadDatabaseConfig(values) if err != nil { - return Config{}, err + return ServerConfig{}, err } cacheURL, err := requiredString(values, "CACHE_URL") if err != nil { - return Config{}, err + return ServerConfig{}, err } store, err := requiredString(values, "SESSION_STORE") if err != nil { - return Config{}, err + return ServerConfig{}, err } if store != "valkey" { - return Config{}, errors.New("SESSION_STORE must be valkey") + return ServerConfig{}, errors.New("SESSION_STORE must be valkey") } sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30) if err != nil { - return Config{}, err + return ServerConfig{}, err } idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) if err != nil { - return Config{}, err + return ServerConfig{}, err } requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300) if err != nil { - return Config{}, err + return ServerConfig{}, err } readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300) if err != nil { - return Config{}, err + return ServerConfig{}, err } readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300) if err != nil { - return Config{}, err + return ServerConfig{}, err } writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300) if err != nil { - return Config{}, err + return ServerConfig{}, err } idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600) if err != nil { - return Config{}, err + return ServerConfig{}, err } shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300) if err != nil { - return Config{}, err + return ServerConfig{}, err } passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024) if err != nil { - return Config{}, err + return ServerConfig{}, err } passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20) if err != nil { - return Config{}, err + return ServerConfig{}, err } passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255) if err != nil { - return Config{}, err + return ServerConfig{}, err } passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024) if err != nil { - return Config{}, err + return ServerConfig{}, err } passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024) if err != nil { - return Config{}, err + return ServerConfig{}, err } swaggerPort, err := requiredPort(values, "SWAGGER_PORT") if err != nil { - return Config{}, err + return ServerConfig{}, err } - s3Endpoint, err := requiredString(values, "S3_ENDPOINT") + storage, err := loadObjectStorageConfig(values) if err != nil { - return Config{}, err - } - s3Region, err := requiredString(values, "S3_REGION") - if err != nil { - return Config{}, err - } - s3Bucket, err := requiredString(values, "S3_BUCKET") - if err != nil { - return Config{}, err - } - s3AccessKey, err := requiredString(values, "S3_ACCESS_KEY") - if err != nil { - return Config{}, err - } - s3SecretKey, err := requiredString(values, "S3_SECRET_KEY") - if err != nil { - return Config{}, err - } - s3UseTLS, err := boolean(values, "S3_USE_TLS") - if err != nil { - return Config{}, err - } - s3Timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) - if err != nil { - return Config{}, err + return ServerConfig{}, err } attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) if err != nil { - return Config{}, err + return ServerConfig{}, err } attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) if err != nil { - return Config{}, err + return ServerConfig{}, err + } + attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { + return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes) } attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) if err != nil { - return Config{}, err + return ServerConfig{}, err } - minimumDecodeMemory := int64(attachmentMaxBytes) + int64(attachmentMaxPixels)*8 + minimumDecodeMemory := int64(attachmentMaxPixels) * 8 if int64(attachmentDecodeMemory) < minimumDecodeMemory { - return Config{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) } - return Config{ + return ServerConfig{ Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), @@ -209,9 +220,11 @@ func Load() (Config, error) { HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), - SwaggerPort: swaggerPort, S3Endpoint: s3Endpoint, S3Region: s3Region, S3Bucket: s3Bucket, - S3AccessKey: s3AccessKey, S3SecretKey: s3SecretKey, S3UseTLS: s3UseTLS, S3Timeout: s3Timeout, + SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + AttachmentBodyMemory: int64(attachmentBodyMemory), AttachmentDecodeMemory: int64(attachmentDecodeMemory), }, nil } @@ -235,6 +248,62 @@ func LoadMigration() (MigrationConfig, error) { }, nil } +func LoadReconciliation() (ReconciliationConfig, error) { + values := viper.New() + values.AutomaticEnv() + database, err := loadDatabaseConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + return ReconciliationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + }, nil +} + +func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { + endpoint, err := requiredString(values, "S3_ENDPOINT") + if err != nil { + return objectStorageConfig{}, err + } + region, err := requiredString(values, "S3_REGION") + if err != nil { + return objectStorageConfig{}, err + } + bucket, err := requiredString(values, "S3_BUCKET") + if err != nil { + return objectStorageConfig{}, err + } + accessKey, err := requiredString(values, "S3_ACCESS_KEY") + if err != nil { + return objectStorageConfig{}, err + } + secretKey, err := requiredString(values, "S3_SECRET_KEY") + if err != nil { + return objectStorageConfig{}, err + } + useTLS, err := boolean(values, "S3_USE_TLS") + if err != nil { + return objectStorageConfig{}, err + } + timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + if err != nil { + return objectStorageConfig{}, err + } + return objectStorageConfig{ + endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey, + secretKey: secretKey, useTLS: useTLS, timeout: timeout, + }, nil +} + func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { databaseURL, err := requiredString(values, "DATABASE_URL") if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..863746c --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,29 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4", + "DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60", + "DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + "S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5", + } + for name, value := range values { + t.Setenv(name, value) + } + t.Setenv("APP_HOST", "") + t.Setenv("CACHE_URL", "") + t.Setenv("SWAGGER_PORT", "") + + settings, err := LoadReconciliation() + require.NoError(t, err) + require.Equal(t, "attachments", settings.S3Bucket) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go new file mode 100644 index 0000000..3724d56 --- /dev/null +++ b/internal/messenger/attachment/service.go @@ -0,0 +1,252 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "mime" + "net/http" + "time" + + "github.com/google/uuid" + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" +) + +type Repository interface { + ChatForUser(context.Context, string, string) (model.Chat, error) + CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) + MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) + DeletePendingAttachment(context.Context, string, string) error + AttachmentByStorage(context.Context, string, string) (model.Attachment, error) + AttachmentForUser(context.Context, string, string) (model.Attachment, error) +} + +type ObjectStore interface { + Put(context.Context, string, string, []byte) error + Get(context.Context, string) (io.ReadCloser, error) + Delete(context.Context, string) error +} + +type Config struct { + MaxBytes int64 + MaxPixels int64 + BodyMemoryBytes int64 + DecodeMemoryBytes int64 + CleanupTimeout time.Duration +} + +type Upload struct { + ChatID string + OwnerID string + Filename string + ContentType string + ContentLength int64 + Body io.Reader +} + +type Content struct { + Attachment model.Attachment + Body io.ReadCloser +} + +type Service struct { + repository Repository + objects ObjectStore + config Config + bodyMemory *semaphore.Weighted + decodeMemory *semaphore.Weighted + logger *slog.Logger +} + +func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.CleanupTimeout <= 0 { + return nil, errors.New("invalid attachment service configuration") + } + if config.BodyMemoryBytes < config.MaxBytes { + return nil, errors.New("attachment body memory must fit one maximum upload") + } + if config.DecodeMemoryBytes < config.MaxPixels*8 { + return nil, errors.New("attachment decode memory must fit one maximum image") + } + return &Service{ + repository: repository, objects: objects, config: config, + bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes), + decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger, + }, nil +} + +func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) { + if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil { + return model.Attachment{}, mapDependencyError(err) + } + data, contentType, release, err := s.readImage(input) + if err != nil { + return model.Attachment{}, err + } + defer release() + + attachmentID := uuid.NewString() + objectKey := "attachments/" + uuid.NewString() + "/original" + digest := sha256.Sum256(data) + pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: objectKey, Status: model.AttachmentPending, + }) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil { + s.deletePending(pending.ID, objectKey) + return model.Attachment{}, model.ErrUnavailable + } + attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey) + if err == nil { + return attachment, nil + } + return s.resolveReadyOutcome(pending.ID, objectKey) +} + +func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) + defer cancel() + stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) + if err == nil && stored.Status == model.AttachmentReady { + return stored, nil + } + if err == nil && stored.Status == model.AttachmentPending { + if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { + s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr) + return model.Attachment{}, model.ErrUnavailable + } + if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil { + s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr) + } + return model.Attachment{}, model.ErrUnavailable + } + if errors.Is(err, model.ErrNotFound) { + if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { + s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr) + } + return model.Attachment{}, model.ErrUnavailable + } + s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err) + return model.Attachment{}, model.ErrUnavailable +} + +func (s *Service) deletePending(attachmentID, objectKey string) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) + defer cancel() + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil { + s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + } +} + +func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + return attachment, nil +} + +func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) { + attachment, err := s.Metadata(ctx, attachmentID, userID) + if err != nil { + return Content{}, err + } + body, err := s.objects.Get(ctx, attachment.ObjectKey) + if err != nil { + return Content{}, model.ErrUnavailable + } + return Content{Attachment: attachment, Body: body}, nil +} + +func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { + mediaType, parameters, err := mime.ParseMediaType(input.ContentType) + if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { + return nil, "", nil, model.ErrUnsupportedMedia + } + if input.ContentLength > s.config.MaxBytes { + return nil, "", nil, model.ErrPayloadTooLarge + } + reservation := s.config.MaxBytes + if !s.bodyMemory.TryAcquire(reservation) { + return nil, "", nil, model.ErrCapacity + } + releaseBody := func() { s.bodyMemory.Release(reservation) } + data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1)) + if err != nil { + releaseBody() + return nil, "", nil, model.ErrInvalidInput + } + if int64(len(data)) > s.config.MaxBytes { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + if len(data) == 0 { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + detected := http.DetectContentType(data) + if detected != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || imageContentType(format) != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) + if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > s.config.MaxPixels { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + decodeBytes := pixels * 8 + if !s.decodeMemory.TryAcquire(decodeBytes) { + releaseBody() + return nil, "", nil, model.ErrCapacity + } + if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { + s.decodeMemory.Release(decodeBytes) + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + release := func() { + s.decodeMemory.Release(decodeBytes) + releaseBody() + } + return data, mediaType, release, nil +} + +func imageContentType(format string) string { + switch format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + default: + return "" + } +} + +func mapDependencyError(err error) error { + switch { + case errors.Is(err, model.ErrNotFound): + return model.ErrNotFound + case errors.Is(err, model.ErrConflict): + return model.ErrConflict + default: + return model.ErrUnavailable + } +} diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go new file mode 100644 index 0000000..3c5bb9b --- /dev/null +++ b/internal/messenger/attachment/service_test.go @@ -0,0 +1,195 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" +) + +func TestUploadLifecycle(t *testing.T) { + data := pngFixture(t, 2, 2) + events := make([]string, 0, 3) + repository := &repositoryStub{events: &events} + objects := &objectStub{events: &events} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentReady, result.Status) + require.Equal(t, []string{"pending", "put", "ready"}, events) +} + +func TestUploadValidation(t *testing.T) { + data := pngFixture(t, 2, 2) + tests := []struct { + name string + contentType string + data []byte + maxBytes int64 + maxPixels int64 + want error + }{ + {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge}, + {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge}, + {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels) + input := upload(test.data, test.contentType) + _, err := service.Upload(context.Background(), input) + require.ErrorIs(t, err, test.want) + }) + } +} + +func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) { + data := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + require.True(t, service.bodyMemory.TryAcquire(1024)) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.bodyMemory.Release(1024) + + require.True(t, service.decodeMemory.TryAcquire(32)) + _, err = service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.decodeMemory.Release(32) +} + +func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{} + objects := &objectStub{putErr: errors.New("s3 unavailable")} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.True(t, repository.pendingDeleted) +} + +func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true} + objects := &objectStub{} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentReady, result.Status) + require.False(t, objects.deleted) +} + +func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { + t.Helper() + service, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, + Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, + }, + ) + require.NoError(t, err) + return service +} + +func upload(data []byte, contentType string) Upload { + return Upload{ + ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType, + ContentLength: int64(len(data)), Body: bytes.NewReader(data), + } +} + +type repositoryStub struct { + events *[]string + markErr error + lookupReady bool + pendingDeleted bool + pending model.Attachment +} + +func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} + +func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) { + attachment.Status = model.AttachmentPending + s.pending = attachment + s.record("pending") + return attachment, nil +} + +func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) { + if s.markErr != nil { + return model.Attachment{}, s.markErr + } + s.pending.Status = model.AttachmentReady + s.record("ready") + return s.pending, nil +} + +func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error { + s.pendingDeleted = true + return nil +} + +func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { + if s.lookupReady { + s.pending.Status = model.AttachmentReady + return s.pending, nil + } + return model.Attachment{}, model.ErrNotFound +} + +func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { + return s.pending, nil +} + +func (s *repositoryStub) record(event string) { + if s.events != nil { + *s.events = append(*s.events, event) + } +} + +type objectStub struct { + events *[]string + putErr error + deleted bool +} + +func (s *objectStub) Put(context.Context, string, string, []byte) error { + if s.events != nil { + *s.events = append(*s.events, "put") + } + return s.putErr +} + +func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (s *objectStub) Delete(context.Context, string) error { + s.deleted = true + return nil +} + +func pngFixture(t *testing.T, width, height int) []byte { + t.Helper() + var data bytes.Buffer + value := image.NewRGBA(image.Rect(0, 0, width, height)) + value.Set(0, 0, color.White) + require.NoError(t, png.Encode(&data, value)) + return data.Bytes() +} diff --git a/internal/messenger/attachments.go b/internal/messenger/attachments.go deleted file mode 100644 index f69a506..0000000 --- a/internal/messenger/attachments.go +++ /dev/null @@ -1,260 +0,0 @@ -package messenger - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "image" - _ "image/jpeg" - _ "image/png" - "io" - "mime" - "net/http" - "net/url" - "strconv" - "strings" - "time" - "unicode" - "unicode/utf8" - - "golang.org/x/sync/semaphore" - - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" -) - -func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { - started := time.Now() - result := "client_error" - defer func() { - s.metrics.attachmentUploads.WithLabelValues(result).Inc() - s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) - }() - - if r.Method != http.MethodPost { - return methodNotAllowed(w, http.MethodPost) - } - user, _, authErr := s.authenticate(r) - if authErr != nil { - return authErr - } - chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) - if !valid { - return invalidRequest() - } - filename, valid := attachmentFilename(r.URL.Query()) - if !valid { - return invalidRequest() - } - if _, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID); errors.Is(err, postgres.ErrNotFound) { - return notFound() - } else if err != nil { - result = "dependency_error" - return unavailable() - } - data, contentType, readErr := readImage(w, r, s.maxUploadBytes, s.maxImagePixels, s.decodeMemory) - if readErr != nil { - return readErr - } - - attachmentID, err := newID() - if err != nil { - result = "dependency_error" - return internalError() - } - objectID, err := newID() - if err != nil { - result = "dependency_error" - return internalError() - } - objectKey := "attachments/" + objectID + "/original" - if err := s.objects.Put(r.Context(), objectKey, contentType, data); err != nil { - result = "dependency_error" - return unavailable() - } - digest := sha256.Sum256(data) - attachment, err := s.messengerStore.CreateAttachment(r.Context(), model.Attachment{ - ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: filename, - ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: objectKey, - }) - if err != nil { - result = "dependency_error" - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.requestTTL) - defer cancel() - stored, lookupErr := s.messengerStore.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) - if lookupErr == nil { - result = "success" - s.metrics.attachmentBytes.Add(float64(len(data))) - writeJSON(w, http.StatusCreated, stored) - return nil - } - if !errors.Is(lookupErr, postgres.ErrNotFound) { - s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", lookupErr) - return unavailable() - } - if cleanupErr := s.objects.Delete(cleanupCtx, objectKey); cleanupErr != nil { - s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", cleanupErr) - } - if errors.Is(err, postgres.ErrNotFound) { - return notFound() - } - return unavailable() - } - result = "success" - s.metrics.attachmentBytes.Add(float64(len(data))) - writeJSON(w, http.StatusCreated, attachment) - return nil -} - -func attachmentFilename(query url.Values) (string, bool) { - if len(query) != 1 || len(query["filename"]) != 1 { - return "", false - } - filename := query["filename"][0] - length := utf8.RuneCountInString(filename) - if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { - return "", false - } - for _, char := range filename { - if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { - return "", false - } - } - return filename, true -} - -func readImage( - w http.ResponseWriter, - r *http.Request, - maxBytes int64, - maxPixels int64, - decodeMemory *semaphore.Weighted, -) ([]byte, string, *apiError) { - mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { - return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} - } - if r.ContentLength > maxBytes { - return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} - } - r.Body = http.MaxBytesReader(w, r.Body, maxBytes) - data, err := io.ReadAll(r.Body) - if err != nil { - var tooLarge *http.MaxBytesError - if errors.As(err, &tooLarge) { - return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} - } - return nil, "", invalidRequest() - } - if len(data) == 0 { - return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} - } - detected := http.DetectContentType(data) - if detected != mediaType { - return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "content type does not match image"} - } - imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil || imageContentType(format) != mediaType { - return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} - } - pixels := int64(imageConfig.Width) * int64(imageConfig.Height) - if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > maxPixels { - return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image dimensions are too large"} - } - decodeBytes := pixels*8 + int64(len(data)) - if !decodeMemory.TryAcquire(decodeBytes) { - w.Header().Set("Retry-After", "1") - return nil, "", &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} - } - defer decodeMemory.Release(decodeBytes) - if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { - return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"} - } - return data, mediaType, nil -} - -func imageContentType(format string) string { - switch format { - case "png": - return "image/png" - case "jpeg": - return "image/jpeg" - default: - return "" - } -} - -func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { - if r.Method != http.MethodGet { - return methodNotAllowed(w, http.MethodGet) - } - user, _, authErr := s.authenticate(r) - if authErr != nil { - return authErr - } - attachment, apiErr := s.attachmentForUser(r, user.ID) - if apiErr != nil { - return apiErr - } - writeJSON(w, http.StatusOK, attachment) - return nil -} - -func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { - if r.Method != http.MethodGet { - return methodNotAllowed(w, http.MethodGet) - } - user, _, authErr := s.authenticate(r) - if authErr != nil { - return authErr - } - if !originalVariant(r.URL.Query()) { - return invalidRequest() - } - attachment, apiErr := s.attachmentForUser(r, user.ID) - if apiErr != nil { - return apiErr - } - content, err := s.objects.Get(r.Context(), attachment.ObjectKey) - if errors.Is(err, objectstore.ErrNotFound) { - return unavailable() - } - if err != nil { - return unavailable() - } - defer func() { _ = content.Close() }() - w.Header().Set("Content-Type", attachment.ContentType) - w.Header().Set("Content-Length", strconv.FormatInt(attachment.SizeBytes, 10)) - w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename})) - w.WriteHeader(http.StatusOK) - if _, err := io.Copy(w, content); err != nil { - s.logger.Warn("attachment download interrupted", "attachment_id", attachment.ID, "error", err) - } - return nil -} - -func originalVariant(query url.Values) bool { - if len(query) == 0 { - return true - } - return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" -} - -func (s *Server) attachmentForUser(r *http.Request, userID string) (Attachment, *apiError) { - parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") - attachmentID, valid := canonicalID(parts[3]) - if !valid { - return Attachment{}, invalidRequest() - } - attachment, err := s.messengerStore.AttachmentForUser(r.Context(), attachmentID, userID) - if errors.Is(err, postgres.ErrNotFound) { - return Attachment{}, notFound() - } - if err != nil { - return Attachment{}, unavailable() - } - return attachment, nil -} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go new file mode 100644 index 0000000..d579de4 --- /dev/null +++ b/internal/messenger/httpapi/attachments.go @@ -0,0 +1,164 @@ +package httpapi + +import ( + "errors" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" +) + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { + started := time.Now() + result := "client_error" + defer func() { + s.metrics.attachmentUploads.WithLabelValues(result).Inc() + s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) + }() + + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return invalidRequest() + } + filename, valid := attachmentFilename(query) + if !valid { + return invalidRequest() + } + uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ + ChatID: chatID, OwnerID: user.ID, Filename: filename, + ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + }) + if err != nil { + if errors.Is(err, model.ErrCapacity) { + result = "capacity_rejected" + w.Header().Set("Retry-After", "1") + } else if errors.Is(err, model.ErrUnavailable) { + result = "dependency_error" + } + return attachmentError(err) + } + result = "success" + s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) + writeJSON(w, http.StatusCreated, uploaded) + return nil +} + +func attachmentFilename(query url.Values) (string, bool) { + if len(query) != 1 || len(query["filename"]) != 1 { + return "", false + } + filename := query["filename"][0] + length := utf8.RuneCountInString(filename) + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { + return "", false + } + for _, char := range filename { + if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { + return "", false + } + } + return filename, true +} + +func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + writeJSON(w, http.StatusOK, value) + return nil +} + +func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || !originalVariant(query) { + return invalidRequest() + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + defer func() { _ = content.Body.Close() }() + w.Header().Set("Content-Type", content.Attachment.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, content.Body); err != nil { + s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err) + } + return nil +} + +func originalVariant(query url.Values) bool { + if len(query) == 0 { + return true + } + return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" +} + +func attachmentIDFromRequest(r *http.Request) (string, *apiError) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + attachmentID, valid := canonicalID(parts[3]) + if !valid { + return "", invalidRequest() + } + return attachmentID, nil +} + +func attachmentError(err error) *apiError { + switch { + case errors.Is(err, model.ErrNotFound): + return notFound() + case errors.Is(err, model.ErrInvalidInput): + return invalidRequest() + case errors.Is(err, model.ErrUnsupportedMedia): + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} + case errors.Is(err, model.ErrPayloadTooLarge): + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + case errors.Is(err, model.ErrCapacity): + return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + default: + return unavailable() + } +} diff --git a/internal/messenger/auth.go b/internal/messenger/httpapi/auth.go similarity index 86% rename from internal/messenger/auth.go rename to internal/messenger/httpapi/auth.go index 7413222..0f74555 100644 --- a/internal/messenger/auth.go +++ b/internal/messenger/httpapi/auth.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "errors" @@ -7,8 +7,6 @@ import ( "strings" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session" ) func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { @@ -23,8 +21,8 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError if !ok { return unauthorized() } - credentials, err := s.messengerStore.UserByUsername(r.Context(), username) - if errors.Is(err, postgres.ErrNotFound) { + credentials, err := s.userRepository.UserByUsername(r.Context(), username) + if errors.Is(err, model.ErrNotFound) { if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { return internalError() } @@ -69,7 +67,7 @@ func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError if authErr != nil { return authErr } - if err := s.sessions.Delete(r.Context(), token); errors.Is(err, session.ErrNotFound) { + if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) { return unauthorized() } else if err != nil { return unavailable() @@ -88,14 +86,14 @@ func (s *Server) authenticate(r *http.Request) (User, string, *apiError) { return User{}, "", unauthorized() } userID, err := s.sessions.UserID(r.Context(), token) - if errors.Is(err, session.ErrNotFound) || errors.Is(err, session.ErrInvalidToken) { + if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) { return User{}, "", unauthorized() } if err != nil { return User{}, "", unavailable() } - user, err := s.messengerStore.UserByID(r.Context(), userID) - if errors.Is(err, postgres.ErrNotFound) { + user, err := s.userRepository.UserByID(r.Context(), userID) + if errors.Is(err, model.ErrNotFound) { _ = s.sessions.Delete(r.Context(), token) return User{}, "", unauthorized() } diff --git a/internal/messenger/errors.go b/internal/messenger/httpapi/errors.go similarity index 99% rename from internal/messenger/errors.go rename to internal/messenger/httpapi/errors.go index 43923b7..95ffd1c 100644 --- a/internal/messenger/errors.go +++ b/internal/messenger/httpapi/errors.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "encoding/json" diff --git a/internal/messenger/handlers.go b/internal/messenger/httpapi/handlers.go similarity index 88% rename from internal/messenger/handlers.go rename to internal/messenger/httpapi/handlers.go index 29ec395..72d307c 100644 --- a/internal/messenger/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "context" @@ -10,7 +10,6 @@ import ( "unicode/utf8" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" ) var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) @@ -32,7 +31,7 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { } ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) defer cancel() - if err := s.messengerStore.Ping(ctx); err != nil { + if err := s.database.Ping(ctx); err != nil { return unavailable() } if err := s.sessions.Ping(ctx); err != nil { @@ -64,8 +63,8 @@ func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError { if err != nil { return internalError() } - user, err := s.messengerStore.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) - if errors.Is(err, postgres.ErrConflict) { + user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) + if errors.Is(err, model.ErrConflict) { return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"} } if err != nil { @@ -128,8 +127,8 @@ func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError { if _, included := members[user.ID]; !included { return invalidRequest() } - chat, err := s.messengerStore.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID) - if errors.Is(err, postgres.ErrInvalidMembers) { + chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID) + if errors.Is(err, model.ErrInvalidMembers) { return invalidRequest() } if err != nil { @@ -144,8 +143,8 @@ func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *a if err != nil { return err } - items, hasNext, listErr := s.messengerStore.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit) - if errors.Is(listErr, postgres.ErrInvalidCursor) { + items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrInvalidCursor) { return invalidRequest() } if listErr != nil { @@ -179,8 +178,8 @@ func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError { if !valid { return invalidRequest() } - chat, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID) - if errors.Is(err, postgres.ErrNotFound) { + chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID) + if errors.Is(err, model.ErrNotFound) { return notFound() } if err != nil { @@ -219,13 +218,13 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) { return invalidRequest() } - message, replay, err := s.messengerStore.CreateMessage( + message, replay, err := s.messageRepository.CreateMessage( r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL, ) - if errors.Is(err, postgres.ErrNotFound) { + if errors.Is(err, model.ErrNotFound) { return notFound() } - if errors.Is(err, postgres.ErrIdempotencyConflict) { + if errors.Is(err, model.ErrIdempotencyConflict) { return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"} } if err != nil { @@ -269,11 +268,11 @@ func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, if err != nil { return err } - items, hasNext, listErr := s.messengerStore.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit) - if errors.Is(listErr, postgres.ErrNotFound) { + items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrNotFound) { return notFound() } - if errors.Is(listErr, postgres.ErrInvalidCursor) { + if errors.Is(listErr, model.ErrInvalidCursor) { return invalidRequest() } if listErr != nil { diff --git a/internal/messenger/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go similarity index 67% rename from internal/messenger/messenger_interface.go rename to internal/messenger/httpapi/messenger_interface.go index 682a10f..73dbb4a 100644 --- a/internal/messenger/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -1,37 +1,42 @@ -package messenger +package httpapi import ( "context" - "io" "time" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" ) -type MessengerStore interface { - Ping(context.Context) error +type UserRepository interface { CreateUser(context.Context, string, string, string) (model.User, error) UserByID(context.Context, string) (model.User, error) UserByUsername(context.Context, string) (model.UserCredentials, error) +} + +type ChatRepository interface { CreateChat(context.Context, string, []string, string) (model.Chat, error) ChatForUser(context.Context, string, string) (model.Chat, error) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) - CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) - ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) - CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) - AttachmentByStorage(context.Context, string, string) (model.Attachment, error) - AttachmentForUser(context.Context, string, string) (model.Attachment, error) } -type ObjectStore interface { +type MessageRepository interface { + CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) + ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) +} + +type AttachmentService interface { + Upload(context.Context, attachment.Upload) (model.Attachment, error) + Metadata(context.Context, string, string) (model.Attachment, error) + Content(context.Context, string, string) (attachment.Content, error) +} + +type HealthChecker interface { Ping(context.Context) error - Put(context.Context, string, string, []byte) error - Get(context.Context, string) (io.ReadCloser, error) - Delete(context.Context, string) error } type SessionStore interface { - Ping(context.Context) error + HealthChecker Create(context.Context, string, time.Duration) (string, time.Time, error) UserID(context.Context, string) (string, error) Delete(context.Context, string) error diff --git a/internal/messenger/metrics.go b/internal/messenger/httpapi/metrics.go similarity index 99% rename from internal/messenger/metrics.go rename to internal/messenger/httpapi/metrics.go index 267d4b7..1dae064 100644 --- a/internal/messenger/metrics.go +++ b/internal/messenger/httpapi/metrics.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "net/http" diff --git a/internal/messenger/model.go b/internal/messenger/httpapi/model.go similarity index 96% rename from internal/messenger/model.go rename to internal/messenger/httpapi/model.go index f1509b2..c1707ce 100644 --- a/internal/messenger/model.go +++ b/internal/messenger/httpapi/model.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" diff --git a/internal/messenger/request.go b/internal/messenger/httpapi/request.go similarity index 99% rename from internal/messenger/request.go rename to internal/messenger/httpapi/request.go index da264ec..c0a5159 100644 --- a/internal/messenger/request.go +++ b/internal/messenger/httpapi/request.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "bytes" diff --git a/internal/messenger/routes.go b/internal/messenger/httpapi/routes.go similarity index 98% rename from internal/messenger/routes.go rename to internal/messenger/httpapi/routes.go index 25821e7..f272767 100644 --- a/internal/messenger/routes.go +++ b/internal/messenger/httpapi/routes.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "net/http" diff --git a/internal/messenger/server.go b/internal/messenger/httpapi/server.go similarity index 63% rename from internal/messenger/server.go rename to internal/messenger/httpapi/server.go index 7d65445..9af19cf 100644 --- a/internal/messenger/server.go +++ b/internal/messenger/httpapi/server.go @@ -1,4 +1,4 @@ -package messenger +package httpapi import ( "context" @@ -6,26 +6,44 @@ import ( "net/http" "sync/atomic" "time" - - "golang.org/x/sync/semaphore" ) +type Dependencies struct { + Users UserRepository + Chats ChatRepository + Messages MessageRepository + Attachments AttachmentService + Sessions SessionStore + Passwords PasswordService + Database HealthChecker + Objects HealthChecker +} + +type Config struct { + SessionTTL time.Duration + IdempotencyTTL time.Duration + RequestTTL time.Duration + InstanceID string + SwaggerPort string +} + type Server struct { - messengerStore MessengerStore - sessions SessionStore - passwords PasswordService - objects ObjectStore - sessionTTL time.Duration - idempotencyTTL time.Duration - requestTTL time.Duration - instanceID string - logger *slog.Logger - swaggerPort string - maxUploadBytes int64 - maxImagePixels int64 - decodeMemory *semaphore.Weighted - metrics *metrics - ready atomic.Bool + userRepository UserRepository + chatRepository ChatRepository + messageRepository MessageRepository + attachments AttachmentService + sessions SessionStore + passwords PasswordService + database HealthChecker + objects HealthChecker + sessionTTL time.Duration + idempotencyTTL time.Duration + requestTTL time.Duration + instanceID string + logger *slog.Logger + swaggerPort string + metrics *metrics + ready atomic.Bool } type statusWriter struct { @@ -40,25 +58,16 @@ func (w *statusWriter) WriteHeader(status int) { func NewServer( logger *slog.Logger, - messages MessengerStore, - sessions SessionStore, - passwords PasswordService, - objects ObjectStore, - sessionTTL time.Duration, - idempotencyTTL time.Duration, - requestTTL time.Duration, - instanceID string, - swaggerPort string, - maxUploadBytes int64, - maxImagePixels int64, - decodeMemoryBytes int64, + dependencies Dependencies, + config Config, ) *Server { s := &Server{ - messengerStore: messages, sessions: sessions, passwords: passwords, objects: objects, - sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL, - instanceID: instanceID, logger: logger, swaggerPort: swaggerPort, - maxUploadBytes: maxUploadBytes, maxImagePixels: maxImagePixels, - decodeMemory: semaphore.NewWeighted(decodeMemoryBytes), metrics: newMetrics(), + userRepository: dependencies.Users, chatRepository: dependencies.Chats, + messageRepository: dependencies.Messages, + attachments: dependencies.Attachments, sessions: dependencies.Sessions, + passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, + sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(), } s.ready.Store(true) return s diff --git a/internal/messenger/server_test.go b/internal/messenger/httpapi/server_test.go similarity index 60% rename from internal/messenger/server_test.go rename to internal/messenger/httpapi/server_test.go index 3440240..d11d1b2 100644 --- a/internal/messenger/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -1,13 +1,9 @@ -package messenger +package httpapi import ( - "bytes" "context" "errors" "fmt" - "image" - "image/color" - "image/png" "io" "log/slog" "net/http" @@ -17,10 +13,9 @@ import ( "time" "github.com/stretchr/testify/suite" - "golang.org/x/sync/semaphore" + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres" ) var errDependency = errors.New("dependency unavailable") @@ -39,19 +34,14 @@ func (s *MessengerSuite) SetupTest() { } type messengerStoreStub struct { - pingErr error - userByID model.User - userByIDErr error - credentials model.UserCredentials - credentialsErr error - message model.Message - messageReplay bool - messageErr error - attachment model.Attachment - attachmentErr error - storedAttachment model.Attachment - storedAttachmentExists bool - storedAttachmentErr error + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error } func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } @@ -79,50 +69,26 @@ func (s messengerStoreStub) CreateMessage(context.Context, string, string, strin func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { return nil, false, nil } -func (s messengerStoreStub) CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) { - return s.attachment, s.attachmentErr -} -func (s messengerStoreStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { - if s.storedAttachmentErr != nil { - return model.Attachment{}, s.storedAttachmentErr - } - if !s.storedAttachmentExists { - return model.Attachment{}, postgres.ErrNotFound - } - return s.storedAttachment, nil -} -func (s messengerStoreStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { - return s.attachment, s.attachmentErr -} type objectStoreStub struct { - pingErr error - putErr error - getData []byte - getErr error - deleteErr error - putKey *string - deletedKey *string + pingErr error } func (s objectStoreStub) Ping(context.Context) error { return s.pingErr } -func (s objectStoreStub) Put(_ context.Context, key string, _ string, _ []byte) error { - if s.putKey != nil { - *s.putKey = key - } - return s.putErr + +type attachmentServiceStub struct { + uploaded model.Attachment + err error } -func (s objectStoreStub) Get(context.Context, string) (io.ReadCloser, error) { - if s.getErr != nil { - return nil, s.getErr - } - return io.NopCloser(bytes.NewReader(s.getData)), nil + +func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) { + return s.uploaded, s.err } -func (s objectStoreStub) Delete(_ context.Context, key string) error { - if s.deletedKey != nil { - *s.deletedKey = key - } - return s.deleteErr +func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) { + return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err } type sessionStub struct { @@ -164,10 +130,7 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - server := NewServer( - s.logger, tt.store, tt.sessions, passwordsStub{}, tt.objects, time.Hour, 24*time.Hour, - time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, - ) + server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{}) response := serveRequest(server, http.MethodGet, "/health/ready", "") if response.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) @@ -300,7 +263,7 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe t := s.T() store := messengerStoreStub{ userByID: model.User{ID: "user-id"}, - messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict), + messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict), } server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`)) @@ -314,52 +277,6 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe } } -func (s *MessengerSuite) TestImageValidation() { - data := testPNG(s.T(), 2, 2) - tests := []struct { - name string - contentType string - data []byte - maxBytes int64 - maxPixels int64 - wantStatus int - }{ - {name: "valid", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 4}, - {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType}, - {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, wantStatus: http.StatusRequestEntityTooLarge}, - {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, wantStatus: http.StatusRequestEntityTooLarge}, - {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType}, - } - for _, tt := range tests { - s.Run(tt.name, func() { - request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(tt.data)) - request.Header.Set("Content-Type", tt.contentType) - response := httptest.NewRecorder() - actual, _, apiErr := readImage( - response, request, tt.maxBytes, tt.maxPixels, semaphore.NewWeighted(384*1024*1024), - ) - if tt.wantStatus == 0 { - s.Require().Nil(apiErr) - s.Equal(tt.data, actual) - return - } - s.Require().NotNil(apiErr) - s.Equal(tt.wantStatus, apiErr.status) - }) - } -} - -func (s *MessengerSuite) TestImageValidationLimitsConcurrentDecodeMemory() { - data := testPNG(s.T(), 2, 2) - request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(data)) - request.Header.Set("Content-Type", "image/png") - response := httptest.NewRecorder() - _, _, apiErr := readImage(response, request, 1024, 4, semaphore.NewWeighted(1)) - s.Require().NotNil(apiErr) - s.Equal(http.StatusTooManyRequests, apiErr.status) - s.Equal("1", response.Header().Get("Retry-After")) -} - func (s *MessengerSuite) TestAttachmentFilenameValidation() { for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} { _, valid := attachmentFilename(map[string][]string{"filename": {filename}}) @@ -370,79 +287,39 @@ func (s *MessengerSuite) TestAttachmentFilenameValidation() { s.Equal("фото.png", filename) } -func (s *MessengerSuite) TestUploadCompensatesObjectWhenMetadataWriteFails() { - var putKey, deletedKey string - store := messengerStoreStub{ - userByID: model.User{ID: "00000000-0000-0000-0000-000000000001"}, - attachmentErr: errDependency, - } - objects := objectStoreStub{putKey: &putKey, deletedKey: &deletedKey} - server := NewServer( - s.logger, store, sessionStub{userID: store.userByID.ID}, passwordsStub{}, objects, - time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, - ) - data := testPNG(s.T(), 2, 2) +func (s *MessengerSuite) TestUploadRejectsMalformedQuery() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) request := httptest.NewRequest( http.MethodPost, - "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png", - bytes.NewReader(data), + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG", + strings.NewReader("image"), ) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", "image/png") response := httptest.NewRecorder() server.Handler().ServeHTTP(response, request) - s.Equal(http.StatusServiceUnavailable, response.Code) - s.NotEmpty(putKey) - s.Equal(putKey, deletedKey) + s.Equal(http.StatusBadRequest, response.Code) } -func (s *MessengerSuite) TestUploadKeepsObjectWhenMetadataCommitSucceeded() { - var deletedKey string - stored := model.Attachment{ - ID: "00000000-0000-0000-0000-000000000003", - ChatID: "00000000-0000-0000-0000-000000000002", - OwnerID: "00000000-0000-0000-0000-000000000001", - Filename: "photo.png", ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("0", 64), - Status: "ready", Variants: []model.Variant{}, CreatedAt: time.Now(), UpdatedAt: time.Now(), - } - store := messengerStoreStub{ - userByID: model.User{ID: stored.OwnerID}, attachmentErr: errDependency, - storedAttachment: stored, storedAttachmentExists: true, - } - server := NewServer( - s.logger, store, sessionStub{userID: stored.OwnerID}, passwordsStub{}, objectStoreStub{deletedKey: &deletedKey}, - time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, - ) - data := testPNG(s.T(), 2, 2) - request := httptest.NewRequest( - http.MethodPost, - "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png", - bytes.NewReader(data), - ) - request.Header.Set("Authorization", "Bearer valid-token") - request.Header.Set("Content-Type", "image/png") - response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) - s.Equal(http.StatusCreated, response.Code) - s.Empty(deletedKey) +func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server { + return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{}) } -func testPNG(t *testing.T, width, height int) []byte { - t.Helper() - var data bytes.Buffer - imageData := image.NewRGBA(image.Rect(0, 0, width, height)) - imageData.Set(0, 0, color.White) - if err := png.Encode(&data, imageData); err != nil { - t.Fatalf("encode PNG: %v", err) - } - return data.Bytes() -} - -func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server { - return NewServer( - s.logger, messages, sessions, passwords, objectStoreStub{}, time.Hour, 24*time.Hour, - time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024, - ) +func (s *MessengerSuite) server( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + objects objectStoreStub, + attachments AttachmentService, +) *Server { + return NewServer(s.logger, Dependencies{ + Users: store, Chats: store, Messages: store, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: store, Objects: objects, + }, Config{ + SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second, + InstanceID: "test-instance", SwaggerPort: "8081", + }) } func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { diff --git a/internal/model/errors.go b/internal/model/errors.go new file mode 100644 index 0000000..dbe09f0 --- /dev/null +++ b/internal/model/errors.go @@ -0,0 +1,18 @@ +package model + +import "errors" + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrInvalidMembers = errors.New("invalid members") + ErrInvalidCursor = errors.New("invalid cursor") + ErrIdempotencyConflict = errors.New("idempotency conflict") + ErrInvalidToken = errors.New("invalid token") + ErrSessionNotFound = errors.New("session not found") + ErrInvalidInput = errors.New("invalid input") + ErrUnsupportedMedia = errors.New("unsupported media") + ErrPayloadTooLarge = errors.New("payload too large") + ErrCapacity = errors.New("capacity exhausted") + ErrUnavailable = errors.New("dependency unavailable") +) diff --git a/internal/model/model.go b/internal/model/model.go index 1a73c6b..bc71b97 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -5,6 +5,12 @@ import ( "time" ) +const ( + AttachmentPending = "pending" + AttachmentReady = "ready" + AttachmentDeleting = "deleting" +) + const timestampLayout = "2006-01-02T15:04:05.000000Z" type User struct { @@ -36,19 +42,27 @@ type Message struct { } type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []Variant `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ObjectKey string `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` +} + +type StoredObject struct { + Key string + Size int64 + LastModified time.Time } type Variant struct { diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go index 6cf73bf..dfcd33b 100644 --- a/internal/objectstore/store.go +++ b/internal/objectstore/store.go @@ -12,6 +12,8 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" ) var ErrNotFound = errors.New("object not found") @@ -26,12 +28,6 @@ type Config struct { Timeout time.Duration } -type Object struct { - Key string - Size int64 - LastModified time.Time -} - type Store struct { client *minio.Client bucket string @@ -115,14 +111,14 @@ func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { return &timeoutReader{ReadCloser: object, cancel: cancel}, nil } -func (s *Store) Stat(ctx context.Context, key string) (Object, error) { +func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) if err != nil { - return Object{}, objectError("stat S3 object", err) + return model.StoredObject{}, objectError("stat S3 object", err) } - return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil + return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil } func (s *Store) Delete(ctx context.Context, key string) error { @@ -134,19 +130,19 @@ func (s *Store) Delete(ctx context.Context, key string) error { return nil } -func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]Object, string, error) { +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { if limit < 1 || limit > 1000 { return nil, "", errors.New("object list limit must be between 1 and 1000") } ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() - objects := make([]Object, 0, limit+1) + objects := make([]model.StoredObject, 0, limit+1) options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1} for info := range s.client.ListObjects(ctx, s.bucket, options) { if info.Err != nil { return nil, "", fmt.Errorf("list S3 objects: %w", info.Err) } - objects = append(objects, Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) + objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) if len(objects) == limit+1 { cancel() break diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 59a73ee..ae92171 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -11,90 +11,115 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" ) -func (s *Store) CreateAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { - const query = ` +const attachmentColumns = ` + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, created_at, updated_at, ready_at` + +type rowScanner interface { + Scan(...any) error +} + +func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { + query := ` INSERT INTO attachments ( - id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at ) - SELECT $1, $2, $3, $4, $5, $6, $7, $8 + SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL WHERE EXISTS ( SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 ) - RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, created_at, updated_at` + RETURNING ` + attachmentColumns - err := s.pool.QueryRow( + err := scanAttachment(s.pool.QueryRow( ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename, attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey, - ).Scan( - &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, - &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, - &attachment.CreatedAt, &attachment.UpdatedAt, - ) + ), &attachment) if errors.Is(err, pgx.ErrNoRows) { return model.Attachment{}, ErrNotFound } if err != nil { - return model.Attachment{}, fmt.Errorf("create attachment: %w", err) + return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err) } - return readyAttachment(attachment), nil + return publicAttachment(attachment), nil +} + +func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + query := ` + UPDATE attachments + SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error { + const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'` + if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil { + return fmt.Errorf("delete pending attachment: %w", err) + } + return nil } func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { - const query = ` - SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, - a.sha256, a.object_key, a.created_at, a.updated_at + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` FROM attachments a JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 - WHERE a.id = $1` + WHERE a.id = $1 AND a.status = 'ready'` var attachment model.Attachment - err := s.pool.QueryRow(ctx, query, attachmentID, userID).Scan( - &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, - &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, - &attachment.CreatedAt, &attachment.UpdatedAt, - ) + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) if errors.Is(err, pgx.ErrNoRows) { return model.Attachment{}, ErrNotFound } if err != nil { return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) } - return readyAttachment(attachment), nil + return publicAttachment(attachment), nil } func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { - const query = ` - SELECT id, chat_id, owner_id, filename, content_type, size_bytes, - sha256, object_key, created_at, updated_at - FROM attachments - WHERE id = $1 AND object_key = $2` + const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` var attachment model.Attachment - err := s.pool.QueryRow(ctx, query, attachmentID, objectKey).Scan( - &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, - &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, - &attachment.CreatedAt, &attachment.UpdatedAt, - ) + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) if errors.Is(err, pgx.ErrNoRows) { return model.Attachment{}, ErrNotFound } if err != nil { return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err) } - return readyAttachment(attachment), nil + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)` + var exists bool + if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { + return false, fmt.Errorf("check attachment by object key: %w", err) + } + return exists, nil } func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { if limit < 1 || limit > 1000 { return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") } - const query = ` - SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, created_at, updated_at - FROM attachments - WHERE object_key > $1 - ORDER BY object_key + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + `, + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) + FROM attachments a + WHERE a.object_key > $1 + ORDER BY a.object_key LIMIT $2` rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) if err != nil { @@ -104,12 +129,8 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { var attachment model.Attachment - err := row.Scan( - &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, - &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, - &attachment.CreatedAt, &attachment.UpdatedAt, - ) - return readyAttachment(attachment), err + err := scanAttachmentWithReference(row, &attachment) + return publicAttachment(attachment), err })) if err != nil { return nil, "", fmt.Errorf("scan attachments: %w", err) @@ -122,6 +143,15 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi return attachments, next, nil } +func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { + return row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.Referenced, + ) +} + func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` var referenced bool @@ -131,25 +161,71 @@ func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) ( return referenced, nil } -func (s *Store) DeleteAttachmentIfUnreferencedBefore( - ctx context.Context, - attachmentID string, - createdBefore time.Time, -) (bool, error) { +func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, attachmentID string, createdBefore time.Time) (bool, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return false, fmt.Errorf("begin attachment deletion claim: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + var createdAt time.Time + err = tx.QueryRow(ctx, `SELECT created_at FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID).Scan(&createdAt) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("lock attachment deletion candidate: %w", err) + } + if createdAt.After(createdBefore) { + return false, nil + } + var referenced bool + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, attachmentID).Scan(&referenced); err != nil { + return false, fmt.Errorf("recheck attachment references: %w", err) + } + if referenced { + return false, nil + } + if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1`, attachmentID); err != nil { + return false, fmt.Errorf("mark attachment deleting: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return false, fmt.Errorf("commit attachment deletion claim: %w", err) + } + return true, nil +} + +func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) { const query = ` DELETE FROM attachments a WHERE a.id = $1 - AND a.created_at <= $2 + AND a.status = 'deleting' AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)` - result, err := s.pool.Exec(ctx, query, attachmentID, createdBefore) + result, err := s.pool.Exec(ctx, query, attachmentID) if err != nil { - return false, fmt.Errorf("delete old unreferenced attachment: %w", err) + return false, fmt.Errorf("delete claimed attachment: %w", err) } return result.RowsAffected() == 1, nil } -func readyAttachment(attachment model.Attachment) model.Attachment { - attachment.Status = "ready" +func scanAttachment(row rowScanner, attachment *model.Attachment) error { + return row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + ) +} + +func attachmentColumnsWithAlias(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ + "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at", + alias, + ) +} + +func publicAttachment(attachment model.Attachment) model.Attachment { attachment.Variants = []model.Variant{} return attachment } diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index 4e0c378..a20bdb3 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -1,11 +1,11 @@ package postgres -import "errors" +import "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" var ( - ErrConflict = errors.New("conflict") - ErrNotFound = errors.New("not found") - ErrInvalidMembers = errors.New("invalid members") - ErrInvalidCursor = errors.New("invalid cursor") - ErrIdempotencyConflict = errors.New("idempotency conflict") + ErrConflict = model.ErrConflict + ErrNotFound = model.ErrNotFound + ErrInvalidMembers = model.ErrInvalidMembers + ErrInvalidCursor = model.ErrInvalidCursor + ErrIdempotencyConflict = model.ErrIdempotencyConflict ) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index 5daea35..5e04f87 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -97,11 +97,20 @@ func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, atta return true, nil } const query = ` - SELECT count(*) + SELECT id FROM attachments - WHERE chat_id = $1 AND id::text = ANY($2)` - var count int - if err := tx.QueryRow(ctx, query, chatID, attachmentIDs).Scan(&count); err != nil { + WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready' + FOR UPDATE` + rows, err := tx.Query(ctx, query, chatID, attachmentIDs) + if err != nil { + return false, fmt.Errorf("validate message attachments: %w", err) + } + defer rows.Close() + count := 0 + for rows.Next() { + count++ + } + if err := rows.Err(); err != nil { return false, fmt.Errorf("validate message attachments: %w", err) } return count == len(attachmentIDs), nil diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 08a9dbc..71943ea 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -126,7 +126,9 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), ObjectKey: "attachments/" + uuid.NewString() + "/original", } - created, err := s.store.CreateAttachment(context.Background(), attachment) + pending, err := s.store.CreatePendingAttachment(context.Background(), attachment) + s.Require().NoError(err) + created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) s.Require().NoError(err) s.Equal("ready", created.Status) s.Empty(created.Variants) @@ -152,7 +154,7 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) s.Require().NoError(err) s.True(referenced) - deleted, err := s.store.DeleteAttachmentIfUnreferencedBefore(context.Background(), created.ID, time.Now().Add(time.Hour)) + deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created.ID, time.Now().Add(time.Hour)) s.Require().NoError(err) s.False(deleted) } @@ -161,12 +163,14 @@ func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { owner, firstChatID := createOwnerAndChat(s.T(), s.store) secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) s.Require().NoError(err) - attachment, err := s.store.CreateAttachment(context.Background(), model.Attachment{ + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png", ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64), ObjectKey: "attachments/" + uuid.NewString() + "/original", }) s.Require().NoError(err) + attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) _, _, err = s.store.CreateMessage( context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour, ) diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 34c7444..d12a138 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -2,43 +2,50 @@ package reconcile import ( "context" + "errors" "fmt" "time" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" ) type MetadataStore interface { ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) - AttachmentReferenced(context.Context, string) (bool, error) - DeleteAttachmentIfUnreferencedBefore(context.Context, string, time.Time) (bool, error) + AttachmentExistsByObjectKey(context.Context, string) (bool, error) + ClaimAttachmentForDeletion(context.Context, string, time.Time) (bool, error) + DeleteAttachmentIfDeleting(context.Context, string) (bool, error) } type ObjectStore interface { - List(context.Context, string, string, int) ([]objectstore.Object, string, error) + List(context.Context, string, string, int) ([]model.StoredObject, string, error) Delete(context.Context, string) error } type Config struct { - Prefix string - PageSize int - GraceTime time.Duration - Apply bool - Now func() time.Time + Prefix string + PageSize int + GraceTime time.Duration + UnreferencedRetention time.Duration + Apply bool + Now func() time.Time } type Report struct { - MetadataScanned int `json:"metadata_scanned"` - ObjectsScanned int `json:"objects_scanned"` - Consistent int `json:"consistent"` - FreshOrphanObjects int `json:"fresh_orphan_objects"` - OldOrphanObjects int `json:"old_orphan_objects"` - DeletedOrphanObjects int `json:"deleted_orphan_objects"` - FreshMissingObjects int `json:"fresh_missing_objects"` - ReferencedMissingObjects int `json:"referenced_missing_objects"` - OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` - DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` + MetadataScanned int `json:"metadata_scanned"` + ObjectsScanned int `json:"objects_scanned"` + Consistent int `json:"consistent"` + FreshPendingAttachments int `json:"fresh_pending_attachments"` + OldPendingAttachments int `json:"old_pending_attachments"` + FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"` + OldUnreferencedAttachments int `json:"old_unreferenced_attachments"` + DeletedAttachmentObjects int `json:"deleted_attachment_objects"` + FreshOrphanObjects int `json:"fresh_orphan_objects"` + OldOrphanObjects int `json:"old_orphan_objects"` + DeletedOrphanObjects int `json:"deleted_orphan_objects"` + FreshMissingObjects int `json:"fresh_missing_objects"` + ReferencedMissingObjects int `json:"referenced_missing_objects"` + OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` + DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` } type Reconciler struct { @@ -49,10 +56,13 @@ type Reconciler struct { func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) { if config.PageSize < 1 || config.PageSize > 1000 { - return nil, fmt.Errorf("page size must be between 1 and 1000") + return nil, errors.New("page size must be between 1 and 1000") } if config.GraceTime < 0 { - return nil, fmt.Errorf("grace time must not be negative") + return nil, errors.New("grace time must not be negative") + } + if config.UnreferencedRetention <= 0 { + return nil, errors.New("unreferenced retention must be positive") } if config.Now == nil { config.Now = time.Now @@ -70,28 +80,30 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) { return Report{}, err } - cutoff := r.config.Now().Add(-r.config.GraceTime) + now := r.config.Now() + anomalyCutoff := now.Add(-r.config.GraceTime) + retentionCutoff := now.Add(-r.config.UnreferencedRetention) var report Report for metadata.current != nil || objects.current != nil { switch { case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey: - if err := r.handleOrphanObject(ctx, *objects.current, cutoff, &report); err != nil { + if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil { return report, err } if err := objects.advance(ctx); err != nil { return report, err } case objects.current == nil || metadata.current.ObjectKey < objects.current.Key: - if err := r.handleMissingObject(ctx, *metadata.current, cutoff, &report); err != nil { + if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil { return report, err } if err := metadata.advance(ctx); err != nil { return report, err } default: - report.MetadataScanned++ - report.ObjectsScanned++ - report.Consistent++ + if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil { + return report, err + } if err := metadata.advance(ctx); err != nil { return report, err } @@ -103,7 +115,83 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) { return report, nil } -func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.Object, cutoff time.Time, report *Report) error { +func (r *Reconciler) handlePair( + ctx context.Context, + metadata model.Attachment, + retentionCutoff time.Time, + anomalyCutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + report.ObjectsScanned++ + report.Consistent++ + switch metadata.Status { + case model.AttachmentPending: + if metadata.CreatedAt.After(anomalyCutoff) { + report.FreshPendingAttachments++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, anomalyCutoff, report) + case model.AttachmentDeleting: + return r.deleteClaimedPair(ctx, metadata, report) + case model.AttachmentReady: + if metadata.Referenced { + return nil + } + age := metadata.CreatedAt + if metadata.ReadyAt != nil { + age = *metadata.ReadyAt + } + if age.After(retentionCutoff) { + report.FreshUnreferencedAttachments++ + return nil + } + report.OldUnreferencedAttachments++ + return r.deletePair(ctx, metadata, retentionCutoff, report) + default: + return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status) + } +} + +func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error { + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment.ID, cutoff) + if err != nil { + return fmt.Errorf("claim attachment %q: %w", attachment.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedPair(ctx, attachment, report) +} + +func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err) + } + report.DeletedAttachmentObjects++ + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +func (r *Reconciler) handleOrphanObject( + ctx context.Context, + object model.StoredObject, + cutoff time.Time, + report *Report, +) error { report.ObjectsScanned++ if object.LastModified.After(cutoff) { report.FreshOrphanObjects++ @@ -113,6 +201,13 @@ func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore. if !r.config.Apply { return nil } + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + return nil + } if err := r.objects.Delete(ctx, object.Key); err != nil { return fmt.Errorf("delete orphan object %q: %w", object.Key, err) } @@ -120,17 +215,18 @@ func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore. return nil } -func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Attachment, cutoff time.Time, report *Report) error { +func (r *Reconciler) handleMissingObject( + ctx context.Context, + metadata model.Attachment, + cutoff time.Time, + report *Report, +) error { report.MetadataScanned++ if metadata.CreatedAt.After(cutoff) { report.FreshMissingObjects++ return nil } - referenced, err := r.metadata.AttachmentReferenced(ctx, metadata.ID) - if err != nil { - return fmt.Errorf("check attachment metadata %q: %w", metadata.ID, err) - } - if referenced { + if metadata.Referenced { report.ReferencedMissingObjects++ return nil } @@ -138,9 +234,16 @@ func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Att if !r.config.Apply { return nil } - deleted, err := r.metadata.DeleteAttachmentIfUnreferencedBefore(ctx, metadata.ID, cutoff) + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata.ID, cutoff) if err != nil { - return fmt.Errorf("delete attachment metadata %q: %w", metadata.ID, err) + return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err) + } + if !claimed { + return nil + } + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) + if err != nil { + return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) } if deleted { report.DeletedUnreferencedRows++ @@ -186,7 +289,7 @@ func (i *metadataIterator) advance(ctx context.Context) error { i.after = next i.done = next == "" if len(page) == 0 && !i.done { - return fmt.Errorf("metadata page is empty but has continuation") + return errors.New("metadata page is empty but has continuation") } } } @@ -196,9 +299,9 @@ type objectIterator struct { prefix string pageSize int after string - page []objectstore.Object + page []model.StoredObject index int - current *objectstore.Object + current *model.StoredObject done bool } @@ -230,7 +333,7 @@ func (i *objectIterator) advance(ctx context.Context) error { i.after = next i.done = next == "" if len(page) == 0 && !i.done { - return fmt.Errorf("object page is empty but has continuation") + return errors.New("object page is empty but has continuation") } } } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index defe3cd..221ac0c 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -9,88 +9,133 @@ import ( "github.com/stretchr/testify/require" "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore" ) -func TestReconcilerDryRunAndApply(t *testing.T) { - now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC) - metadata := &metadataStub{items: []model.Attachment{ - {ID: "consistent", ObjectKey: "attachments/consistent", CreatedAt: now.Add(-time.Hour)}, - {ID: "fresh-missing", ObjectKey: "attachments/fresh-missing", CreatedAt: now.Add(-time.Minute)}, - {ID: "old-missing", ObjectKey: "attachments/old-missing", CreatedAt: now.Add(-time.Hour)}, - {ID: "referenced-missing", ObjectKey: "attachments/referenced-missing", CreatedAt: now.Add(-time.Hour)}, - }, referenced: map[string]bool{"referenced-missing": true}} - objects := &objectStub{items: []objectstore.Object{ - {Key: "attachments/consistent", LastModified: now.Add(-time.Hour)}, - {Key: "attachments/fresh-orphan", LastModified: now.Add(-time.Minute)}, - {Key: "attachments/old-orphan", LastModified: now.Add(-time.Hour)}, +func TestReconcilerLifecycleAndRepeat(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + fresh := now.Add(-10 * time.Minute) + metadata := &metadataStub{items: map[string]model.Attachment{ + "attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false), + "attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true), + "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), + "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), + "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), }} - - dryRun, err := New(metadata, objects, Config{ - Prefix: "attachments/", PageSize: 2, GraceTime: 10 * time.Minute, Now: func() time.Time { return now }, + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, + "attachments/b/original": {Key: "attachments/b/original", LastModified: old}, + "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, + "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, + "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, }) require.NoError(t, err) - report, err := dryRun.Run(context.Background()) - require.NoError(t, err) - require.Equal(t, Report{ - MetadataScanned: 4, ObjectsScanned: 3, Consistent: 1, - FreshOrphanObjects: 1, OldOrphanObjects: 1, FreshMissingObjects: 1, - ReferencedMissingObjects: 1, OldUnreferencedMetadata: 1, - }, report) - require.Empty(t, metadata.deleted) - require.Empty(t, objects.deleted) - apply, err := New(metadata, objects, Config{ - Prefix: "attachments/", PageSize: 2, GraceTime: 10 * time.Minute, Apply: true, - Now: func() time.Time { return now }, - }) - require.NoError(t, err) - report, err = apply.Run(context.Background()) + report, err := reconciler.Run(context.Background()) require.NoError(t, err) + require.Equal(t, 1, report.OldPendingAttachments) + require.Equal(t, 1, report.OldUnreferencedAttachments) + require.Equal(t, 1, report.FreshUnreferencedAttachments) require.Equal(t, 1, report.DeletedOrphanObjects) - require.Equal(t, 1, report.DeletedUnreferencedRows) - require.Equal(t, []string{"old-missing"}, metadata.deleted) - require.Equal(t, []string{"attachments/old-orphan"}, objects.deleted) + require.Equal(t, 2, report.DeletedAttachmentObjects) + require.Equal(t, 3, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, "attachments/b/original") + require.Contains(t, metadata.items, "attachments/d/original") + require.NotContains(t, metadata.items, "attachments/a/original") + require.NotContains(t, metadata.items, "attachments/c/original") + require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, objects.items, "attachments/orphan/original") - report, err = apply.Run(context.Background()) + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedOrphanObjects) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { + now := time.Now().UTC() + metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) require.NoError(t, err) require.Zero(t, report.DeletedOrphanObjects) - require.Zero(t, report.DeletedUnreferencedRows) - require.Equal(t, 1, report.FreshOrphanObjects) - require.Equal(t, 1, report.FreshMissingObjects) - require.Equal(t, 1, report.ReferencedMissingObjects) + require.Contains(t, objects.items, "attachments/a/original") +} + +func TestNewRejectsInvalidRetention(t *testing.T) { + _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) + require.Error(t, err) +} + +func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment { + readyAt := createdAt + return model.Attachment{ + ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt, + ReadyAt: &readyAt, Referenced: referenced, + } } type metadataStub struct { - items []model.Attachment - referenced map[string]bool - deleted []string + items map[string]model.Attachment + appeared map[string]bool } func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { - items := append([]model.Attachment(nil), s.items...) - sort.Slice(items, func(i, j int) bool { return items[i].ObjectKey < items[j].ObjectKey }) - start := 0 - for start < len(items) && items[start].ObjectKey <= after { - start++ + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after { + keys = append(keys, key) + } } - end := min(start+limit, len(items)) + sort.Strings(keys) var next string - if end < len(items) { - next = items[end-1].ObjectKey + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] } - return items[start:end], next, nil + items := make([]model.Attachment, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil } -func (s *metadataStub) AttachmentReferenced(_ context.Context, id string) (bool, error) { - return s.referenced[id], nil +func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) { + if s.appeared[key] { + return true, nil + } + _, exists := s.items[key] + return exists, nil } -func (s *metadataStub) DeleteAttachmentIfUnreferencedBefore(_ context.Context, id string, _ time.Time) (bool, error) { - for index, item := range s.items { - if item.ID == id && !s.referenced[id] { - s.items = append(s.items[:index], s.items[index+1:]...) - s.deleted = append(s.deleted, id) +func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, id string, cutoff time.Time) (bool, error) { + for key, item := range s.items { + if item.ID == id && !item.CreatedAt.After(cutoff) && !item.Referenced { + item.Status = model.AttachmentDeleting + s.items[key] = item + return true, nil + } + } + return false, nil +} + +func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + for key, item := range s.items { + if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { + delete(s.items, key) return true, nil } } @@ -98,32 +143,30 @@ func (s *metadataStub) DeleteAttachmentIfUnreferencedBefore(_ context.Context, i } type objectStub struct { - items []objectstore.Object - deleted []string + items map[string]model.StoredObject } -func (s *objectStub) List(_ context.Context, _ string, after string, limit int) ([]objectstore.Object, string, error) { - items := append([]objectstore.Object(nil), s.items...) - sort.Slice(items, func(i, j int) bool { return items[i].Key < items[j].Key }) - start := 0 - for start < len(items) && items[start].Key <= after { - start++ +func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix { + keys = append(keys, key) + } } - end := min(start+limit, len(items)) + sort.Strings(keys) var next string - if end < len(items) { - next = items[end-1].Key + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] } - return items[start:end], next, nil + items := make([]model.StoredObject, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil } func (s *objectStub) Delete(_ context.Context, key string) error { - for index, item := range s.items { - if item.Key == key { - s.items = append(s.items[:index], s.items[index+1:]...) - s.deleted = append(s.deleted, key) - return nil - } - } + delete(s.items, key) return nil } diff --git a/internal/session/session.go b/internal/session/session.go index 4f40fc5..35fa628 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -13,6 +13,8 @@ import ( "time" "github.com/valkey-io/valkey-go" + + "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model" ) const ( @@ -21,12 +23,12 @@ const ( ) var ( - ErrNotFound = errors.New("session not found") - ErrInvalidToken = errors.New("invalid session token") + ErrNotFound = model.ErrSessionNotFound + ErrInvalidToken = model.ErrInvalidToken ErrInvalidRecord = errors.New("invalid session record") ErrInvalidTTL = errors.New("invalid session TTL") ErrTokenGeneration = errors.New("session token generation failed") - ErrStore = errors.New("session store failure") + ErrStore = model.ErrUnavailable ) type record struct { diff --git a/migrations/0004_attachment_lifecycle.sql b/migrations/0004_attachment_lifecycle.sql new file mode 100644 index 0000000..1596c99 --- /dev/null +++ b/migrations/0004_attachment_lifecycle.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachments + ADD COLUMN status TEXT NOT NULL DEFAULT 'ready', + ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp(); + +UPDATE attachments +SET ready_at = created_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status IN ('ready', 'deleting') AND ready_at IS NOT NULL) + ); + +CREATE INDEX attachments_status_created_at_object_key_idx + ON attachments (status, created_at, object_key);