refactor: isolate attachment lifecycle

This commit is contained in:
2026-10-09 13:13:59 +03:00
parent ad36ab8a92
commit cc4ac1ddd7
33 changed files with 1672 additions and 982 deletions
+29 -9
View File
@@ -240,6 +240,7 @@ type consistencyResult struct {
OrphanDeleted bool `json:"orphan_deleted"`
UnusedDeleted bool `json:"unused_metadata_deleted"`
InUsePreserved bool `json:"in_use_metadata_preserved"`
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
TotalDurationMillis int64 `json:"total_duration_ms"`
}
@@ -259,7 +260,7 @@ func runConsistency(arguments []string, output io.Writer) error {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
settings, err := config.Load()
settings, err := config.LoadReconciliation()
if err != nil {
return err
}
@@ -281,7 +282,9 @@ func runConsistency(arguments []string, output io.Writer) error {
return err
}
metadata := postgres.New(pool)
clean, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0})
clean, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
@@ -320,7 +323,11 @@ func runConsistency(arguments []string, output io.Writer) error {
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
return err
}
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID})
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
if err != nil {
return err
}
@@ -336,7 +343,8 @@ func runConsistency(arguments []string, output io.Writer) error {
}
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, Apply: true,
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
@@ -346,9 +354,19 @@ func runConsistency(arguments []string, output io.Writer) error {
return err
}
freshPreserved := objectExists(ctx, objects, orphanKey) &&
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID)
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
attachmentExists(ctx, pool, abandoned.ID)
if _, err := pool.Exec(ctx, `
UPDATE attachments
SET created_at = clock_timestamp() - INTERVAL '2 hours',
ready_at = clock_timestamp() - INTERVAL '2 hours'
WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("age abandoned attachment: %w", err)
}
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize})
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
@@ -357,7 +375,7 @@ func runConsistency(arguments []string, output io.Writer) error {
return err
}
apply, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, Apply: true,
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
@@ -376,9 +394,10 @@ func runConsistency(arguments []string, output io.Writer) error {
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
TotalDurationMillis: time.Since(started).Milliseconds(),
}
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved ||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
return fmt.Errorf("unexpected reconciliation result: %+v", result)
}
@@ -394,7 +413,8 @@ func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconcile
func hasAnomalies(report reconcile.Report) bool {
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
report.OldUnreferencedMetadata != 0
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
report.OldUnreferencedAttachments != 0
}
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
+8 -192
View File
@@ -1,215 +1,31 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"errors"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
func main() {
appLogger := logger.New()
if err := run(appLogger, os.Args[1:]); err != nil {
appLogger.Error("server stopped", "error", err)
appLogger.Error("application stopped", "error", err)
os.Exit(1)
}
}
func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 0 {
return runServer(logger)
return bootstrap.RunServer(logger)
}
if len(arguments) == 1 && arguments[0] == "migrate" {
return runMigrations()
return bootstrap.RunMigrations()
}
if len(arguments) > 0 && arguments[0] == "reconcile-attachments" {
return runAttachmentReconciliation(arguments[1:], os.Stdout)
if arguments[0] == "reconcile-attachments" {
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
}
return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]")
}
func runServer(logger *slog.Logger) error {
settings, err := config.Load()
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
sessions, err := session.New(settings.CacheURL)
if err != nil {
return err
}
defer func() { _ = sessions.Close() }()
passwords, err := password.New(password.Config{
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
KeyLength: settings.PasswordKeyLength,
})
if err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
app := messenger.NewServer(
logger,
postgres.New(pool),
sessions,
passwords,
objects,
settings.SessionTTL,
settings.IdempotencyTTL,
settings.RequestTimeout,
instanceID,
settings.SwaggerPort,
settings.AttachmentMaxBytes,
settings.AttachmentMaxPixels,
settings.AttachmentDecodeMemory,
)
server := &http.Server{
Addr: settings.Address,
Handler: app.Handler(),
ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
ReadTimeout: settings.HTTPReadTimeout,
WriteTimeout: settings.HTTPWriteTimeout,
IdleTimeout: settings.HTTPIdleTimeout,
}
listener, err := net.Listen("tcp", server.Addr)
if err != nil {
return err
}
serveResult := make(chan error, 1)
go func() {
serveResult <- server.Serve(listener)
}()
logger.Info("server started", "address", server.Addr)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
select {
case err := <-serveResult:
return err
case <-ctx.Done():
logger.Info("shutdown requested")
app.SetReady(false)
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
_ = server.Close()
return err
}
return nil
}
}
func runMigrations() error {
settings, err := config.LoadMigration()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
type reconciliationOutput struct {
Mode string `json:"mode"`
GracePeriod string `json:"grace_period"`
PageSize int `json:"page_size"`
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func runAttachmentReconciliation(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
flags.SetOutput(io.Discard)
apply := flags.Bool("apply", false, "apply safe repairs")
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]")
}
settings, err := config.Load()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply,
})
if err != nil {
return err
}
started := time.Now()
report, err := reconciler.Run(ctx)
if err != nil {
return err
}
mode := "dry-run"
if *apply {
mode = "apply"
}
return json.NewEncoder(output).Encode(reconciliationOutput{
Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize,
DurationMillis: time.Since(started).Milliseconds(), Report: report,
})
return errors.New("usage: messenger [migrate|reconcile-attachments]")
}
+25
View File
@@ -0,0 +1,25 @@
package bootstrap
import (
"context"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
)
func RunMigrations() error {
settings, err := config.LoadMigration()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
+85
View File
@@ -0,0 +1,85 @@
package bootstrap
import (
"context"
"encoding/json"
"errors"
"flag"
"io"
"os/signal"
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
)
type reconciliationOutput struct {
Mode string `json:"mode"`
GracePeriod string `json:"grace_period"`
UnreferencedRetention string `json:"unreferenced_retention"`
PageSize int `json:"page_size"`
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func RunReconciliation(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
flags.SetOutput(io.Discard)
apply := flags.Bool("apply", false, "apply safe repairs")
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments")
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]")
}
if *apply && *gracePeriod <= 0 {
return errors.New("grace-period must be positive in apply mode")
}
settings, err := config.LoadReconciliation()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
UnreferencedRetention: *retention, Apply: *apply,
})
if err != nil {
return err
}
started := time.Now()
report, err := reconciler.Run(ctx)
if err != nil {
return err
}
mode := "dry-run"
if *apply {
mode = "apply"
}
return json.NewEncoder(output).Encode(reconciliationOutput{
Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(),
PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report,
})
}
+13
View File
@@ -0,0 +1,13 @@
package bootstrap
import (
"io"
"testing"
"github.com/stretchr/testify/require"
)
func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) {
err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard)
require.EqualError(t, err, "grace-period must be positive in apply mode")
}
+113
View File
@@ -0,0 +1,113 @@
package bootstrap
import (
"context"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
func RunServer(logger *slog.Logger) error {
settings, err := config.LoadServer()
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), databaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
sessions, err := session.New(settings.CacheURL)
if err != nil {
return err
}
defer func() { _ = sessions.Close() }()
passwords, err := password.New(password.Config{
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
KeyLength: settings.PasswordKeyLength,
})
if err != nil {
return err
}
objects, err := objectstore.New(objectStoreConfig(settings))
if err != nil {
return err
}
repository := postgres.New(pool)
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
CleanupTimeout: settings.RequestTimeout,
})
if err != nil {
return err
}
app := httpapi.NewServer(logger, httpapi.Dependencies{
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
}, httpapi.Config{
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
})
server := &http.Server{
Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout,
IdleTimeout: settings.HTTPIdleTimeout,
}
listener, err := net.Listen("tcp", server.Addr)
if err != nil {
return err
}
serveResult := make(chan error, 1)
go func() { serveResult <- server.Serve(listener) }()
logger.Info("server started", "address", server.Addr)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
select {
case err := <-serveResult:
return err
case <-ctx.Done():
logger.Info("shutdown requested")
app.SetReady(false)
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
_ = server.Close()
return err
}
return nil
}
}
func databaseConfig(settings config.ServerConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}
func objectStoreConfig(settings config.ServerConfig) objectstore.Config {
return objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
}
}
+125 -56
View File
@@ -11,7 +11,7 @@ import (
"github.com/spf13/viper"
)
type Config struct {
type ServerConfig struct {
Address string
DatabaseURL string
CacheURL string
@@ -44,6 +44,7 @@ type Config struct {
S3Timeout time.Duration
AttachmentMaxBytes int64
AttachmentMaxPixels int64
AttachmentBodyMemory int64
AttachmentDecodeMemory int64
}
@@ -58,6 +59,23 @@ type MigrationConfig struct {
MigrationTimeout time.Duration
}
type ReconciliationConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
}
type databaseConfig struct {
url string
maxConns int
@@ -68,136 +86,129 @@ type databaseConfig struct {
dependencyTimeout time.Duration
}
func Load() (Config, error) {
type objectStorageConfig struct {
endpoint string
region string
bucket string
accessKey string
secretKey string
useTLS bool
timeout time.Duration
}
func LoadServer() (ServerConfig, error) {
values := viper.New()
values.AutomaticEnv()
host, err := requiredString(values, "APP_HOST")
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
port, err := requiredPort(values, "APP_PORT")
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
cacheURL, err := requiredString(values, "CACHE_URL")
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
store, err := requiredString(values, "SESSION_STORE")
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
if store != "valkey" {
return Config{}, errors.New("SESSION_STORE must be valkey")
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
}
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
s3Endpoint, err := requiredString(values, "S3_ENDPOINT")
storage, err := loadObjectStorageConfig(values)
if err != nil {
return Config{}, err
}
s3Region, err := requiredString(values, "S3_REGION")
if err != nil {
return Config{}, err
}
s3Bucket, err := requiredString(values, "S3_BUCKET")
if err != nil {
return Config{}, err
}
s3AccessKey, err := requiredString(values, "S3_ACCESS_KEY")
if err != nil {
return Config{}, err
}
s3SecretKey, err := requiredString(values, "S3_SECRET_KEY")
if err != nil {
return Config{}, err
}
s3UseTLS, err := boolean(values, "S3_USE_TLS")
if err != nil {
return Config{}, err
}
s3Timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
if err != nil {
return ServerConfig{}, err
}
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
}
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
if err != nil {
return Config{}, err
return ServerConfig{}, err
}
minimumDecodeMemory := int64(attachmentMaxBytes) + int64(attachmentMaxPixels)*8
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
return Config{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
}
return Config{
return ServerConfig{
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL,
DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
@@ -209,9 +220,11 @@ func Load() (Config, error) {
HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory),
PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism),
PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength),
SwaggerPort: swaggerPort, S3Endpoint: s3Endpoint, S3Region: s3Region, S3Bucket: s3Bucket,
S3AccessKey: s3AccessKey, S3SecretKey: s3SecretKey, S3UseTLS: s3UseTLS, S3Timeout: s3Timeout,
SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
AttachmentBodyMemory: int64(attachmentBodyMemory),
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
}, nil
}
@@ -235,6 +248,62 @@ func LoadMigration() (MigrationConfig, error) {
}, nil
}
func LoadReconciliation() (ReconciliationConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return ReconciliationConfig{}, err
}
storage, err := loadObjectStorageConfig(values)
if err != nil {
return ReconciliationConfig{}, err
}
return ReconciliationConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey,
S3UseTLS: storage.useTLS, S3Timeout: storage.timeout,
}, nil
}
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
endpoint, err := requiredString(values, "S3_ENDPOINT")
if err != nil {
return objectStorageConfig{}, err
}
region, err := requiredString(values, "S3_REGION")
if err != nil {
return objectStorageConfig{}, err
}
bucket, err := requiredString(values, "S3_BUCKET")
if err != nil {
return objectStorageConfig{}, err
}
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
if err != nil {
return objectStorageConfig{}, err
}
secretKey, err := requiredString(values, "S3_SECRET_KEY")
if err != nil {
return objectStorageConfig{}, err
}
useTLS, err := boolean(values, "S3_USE_TLS")
if err != nil {
return objectStorageConfig{}, err
}
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
if err != nil {
return objectStorageConfig{}, err
}
return objectStorageConfig{
endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey,
secretKey: secretKey, useTLS: useTLS, timeout: timeout,
}, nil
}
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
databaseURL, err := requiredString(values, "DATABASE_URL")
if err != nil {
+29
View File
@@ -0,0 +1,29 @@
package config
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
values := map[string]string{
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
}
for name, value := range values {
t.Setenv(name, value)
}
t.Setenv("APP_HOST", "")
t.Setenv("CACHE_URL", "")
t.Setenv("SWAGGER_PORT", "")
settings, err := LoadReconciliation()
require.NoError(t, err)
require.Equal(t, "attachments", settings.S3Bucket)
}
+252
View File
@@ -0,0 +1,252 @@
package attachment
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"mime"
"net/http"
"time"
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type Repository interface {
ChatForUser(context.Context, string, string) (model.Chat, error)
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
DeletePendingAttachment(context.Context, string, string) error
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
}
type ObjectStore interface {
Put(context.Context, string, string, []byte) error
Get(context.Context, string) (io.ReadCloser, error)
Delete(context.Context, string) error
}
type Config struct {
MaxBytes int64
MaxPixels int64
BodyMemoryBytes int64
DecodeMemoryBytes int64
CleanupTimeout time.Duration
}
type Upload struct {
ChatID string
OwnerID string
Filename string
ContentType string
ContentLength int64
Body io.Reader
}
type Content struct {
Attachment model.Attachment
Body io.ReadCloser
}
type Service struct {
repository Repository
objects ObjectStore
config Config
bodyMemory *semaphore.Weighted
decodeMemory *semaphore.Weighted
logger *slog.Logger
}
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.CleanupTimeout <= 0 {
return nil, errors.New("invalid attachment service configuration")
}
if config.BodyMemoryBytes < config.MaxBytes {
return nil, errors.New("attachment body memory must fit one maximum upload")
}
if config.DecodeMemoryBytes < config.MaxPixels*8 {
return nil, errors.New("attachment decode memory must fit one maximum image")
}
return &Service{
repository: repository, objects: objects, config: config,
bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes),
decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger,
}, nil
}
func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) {
if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil {
return model.Attachment{}, mapDependencyError(err)
}
data, contentType, release, err := s.readImage(input)
if err != nil {
return model.Attachment{}, err
}
defer release()
attachmentID := uuid.NewString()
objectKey := "attachments/" + uuid.NewString() + "/original"
digest := sha256.Sum256(data)
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
ObjectKey: objectKey, Status: model.AttachmentPending,
})
if err != nil {
return model.Attachment{}, mapDependencyError(err)
}
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
s.deletePending(pending.ID, objectKey)
return model.Attachment{}, model.ErrUnavailable
}
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
if err == nil {
return attachment, nil
}
return s.resolveReadyOutcome(pending.ID, objectKey)
}
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
if err == nil && stored.Status == model.AttachmentReady {
return stored, nil
}
if err == nil && stored.Status == model.AttachmentPending {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
return model.Attachment{}, model.ErrUnavailable
}
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
if errors.Is(err, model.ErrNotFound) {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
return model.Attachment{}, model.ErrUnavailable
}
func (s *Service) deletePending(attachmentID, objectKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
}
}
func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID)
if err != nil {
return model.Attachment{}, mapDependencyError(err)
}
return attachment, nil
}
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
attachment, err := s.Metadata(ctx, attachmentID, userID)
if err != nil {
return Content{}, err
}
body, err := s.objects.Get(ctx, attachment.ObjectKey)
if err != nil {
return Content{}, model.ErrUnavailable
}
return Content{Attachment: attachment, Body: body}, nil
}
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
mediaType, parameters, err := mime.ParseMediaType(input.ContentType)
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
return nil, "", nil, model.ErrUnsupportedMedia
}
if input.ContentLength > s.config.MaxBytes {
return nil, "", nil, model.ErrPayloadTooLarge
}
reservation := s.config.MaxBytes
if !s.bodyMemory.TryAcquire(reservation) {
return nil, "", nil, model.ErrCapacity
}
releaseBody := func() { s.bodyMemory.Release(reservation) }
data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1))
if err != nil {
releaseBody()
return nil, "", nil, model.ErrInvalidInput
}
if int64(len(data)) > s.config.MaxBytes {
releaseBody()
return nil, "", nil, model.ErrPayloadTooLarge
}
if len(data) == 0 {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
detected := http.DetectContentType(data)
if detected != mediaType {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || imageContentType(format) != mediaType {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > s.config.MaxPixels {
releaseBody()
return nil, "", nil, model.ErrPayloadTooLarge
}
decodeBytes := pixels * 8
if !s.decodeMemory.TryAcquire(decodeBytes) {
releaseBody()
return nil, "", nil, model.ErrCapacity
}
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
s.decodeMemory.Release(decodeBytes)
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
release := func() {
s.decodeMemory.Release(decodeBytes)
releaseBody()
}
return data, mediaType, release, nil
}
func imageContentType(format string) string {
switch format {
case "png":
return "image/png"
case "jpeg":
return "image/jpeg"
default:
return ""
}
}
func mapDependencyError(err error) error {
switch {
case errors.Is(err, model.ErrNotFound):
return model.ErrNotFound
case errors.Is(err, model.ErrConflict):
return model.ErrConflict
default:
return model.ErrUnavailable
}
}
@@ -0,0 +1,195 @@
package attachment
import (
"bytes"
"context"
"errors"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func TestUploadLifecycle(t *testing.T) {
data := pngFixture(t, 2, 2)
events := make([]string, 0, 3)
repository := &repositoryStub{events: &events}
objects := &objectStub{events: &events}
service := newTestService(t, repository, objects, 1024, 4)
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
require.Equal(t, []string{"pending", "put", "ready"}, events)
}
func TestUploadValidation(t *testing.T) {
data := pngFixture(t, 2, 2)
tests := []struct {
name string
contentType string
data []byte
maxBytes int64
maxPixels int64
want error
}{
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge},
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge},
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels)
input := upload(test.data, test.contentType)
_, err := service.Upload(context.Background(), input)
require.ErrorIs(t, err, test.want)
})
}
}
func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
data := pngFixture(t, 2, 2)
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
require.True(t, service.bodyMemory.TryAcquire(1024))
_, err := service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrCapacity)
service.bodyMemory.Release(1024)
require.True(t, service.decodeMemory.TryAcquire(32))
_, err = service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrCapacity)
service.decodeMemory.Release(32)
}
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{}
objects := &objectStub{putErr: errors.New("s3 unavailable")}
service := newTestService(t, repository, objects, 1024, 4)
_, err := service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrUnavailable)
require.True(t, repository.pendingDeleted)
}
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
objects := &objectStub{}
service := newTestService(t, repository, objects, 1024, 4)
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
require.False(t, objects.deleted)
}
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
t.Helper()
service, err := NewService(
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
Config{
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
},
)
require.NoError(t, err)
return service
}
func upload(data []byte, contentType string) Upload {
return Upload{
ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType,
ContentLength: int64(len(data)), Body: bytes.NewReader(data),
}
}
type repositoryStub struct {
events *[]string
markErr error
lookupReady bool
pendingDeleted bool
pending model.Attachment
}
func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) {
return model.Chat{}, nil
}
func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) {
attachment.Status = model.AttachmentPending
s.pending = attachment
s.record("pending")
return attachment, nil
}
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
if s.markErr != nil {
return model.Attachment{}, s.markErr
}
s.pending.Status = model.AttachmentReady
s.record("ready")
return s.pending, nil
}
func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error {
s.pendingDeleted = true
return nil
}
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
if s.lookupReady {
s.pending.Status = model.AttachmentReady
return s.pending, nil
}
return model.Attachment{}, model.ErrNotFound
}
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
return s.pending, nil
}
func (s *repositoryStub) record(event string) {
if s.events != nil {
*s.events = append(*s.events, event)
}
}
type objectStub struct {
events *[]string
putErr error
deleted bool
}
func (s *objectStub) Put(context.Context, string, string, []byte) error {
if s.events != nil {
*s.events = append(*s.events, "put")
}
return s.putErr
}
func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(nil)), nil
}
func (s *objectStub) Delete(context.Context, string) error {
s.deleted = true
return nil
}
func pngFixture(t *testing.T, width, height int) []byte {
t.Helper()
var data bytes.Buffer
value := image.NewRGBA(image.Rect(0, 0, width, height))
value.Set(0, 0, color.White)
require.NoError(t, png.Encode(&data, value))
return data.Bytes()
}
-260
View File
@@ -1,260 +0,0 @@
package messenger
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
}()
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
filename, valid := attachmentFilename(r.URL.Query())
if !valid {
return invalidRequest()
}
if _, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID); errors.Is(err, postgres.ErrNotFound) {
return notFound()
} else if err != nil {
result = "dependency_error"
return unavailable()
}
data, contentType, readErr := readImage(w, r, s.maxUploadBytes, s.maxImagePixels, s.decodeMemory)
if readErr != nil {
return readErr
}
attachmentID, err := newID()
if err != nil {
result = "dependency_error"
return internalError()
}
objectID, err := newID()
if err != nil {
result = "dependency_error"
return internalError()
}
objectKey := "attachments/" + objectID + "/original"
if err := s.objects.Put(r.Context(), objectKey, contentType, data); err != nil {
result = "dependency_error"
return unavailable()
}
digest := sha256.Sum256(data)
attachment, err := s.messengerStore.CreateAttachment(r.Context(), model.Attachment{
ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: objectKey,
})
if err != nil {
result = "dependency_error"
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.requestTTL)
defer cancel()
stored, lookupErr := s.messengerStore.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
if lookupErr == nil {
result = "success"
s.metrics.attachmentBytes.Add(float64(len(data)))
writeJSON(w, http.StatusCreated, stored)
return nil
}
if !errors.Is(lookupErr, postgres.ErrNotFound) {
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", lookupErr)
return unavailable()
}
if cleanupErr := s.objects.Delete(cleanupCtx, objectKey); cleanupErr != nil {
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", cleanupErr)
}
if errors.Is(err, postgres.ErrNotFound) {
return notFound()
}
return unavailable()
}
result = "success"
s.metrics.attachmentBytes.Add(float64(len(data)))
writeJSON(w, http.StatusCreated, attachment)
return nil
}
func attachmentFilename(query url.Values) (string, bool) {
if len(query) != 1 || len(query["filename"]) != 1 {
return "", false
}
filename := query["filename"][0]
length := utf8.RuneCountInString(filename)
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
return "", false
}
for _, char := range filename {
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
return "", false
}
}
return filename, true
}
func readImage(
w http.ResponseWriter,
r *http.Request,
maxBytes int64,
maxPixels int64,
decodeMemory *semaphore.Weighted,
) ([]byte, string, *apiError) {
mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
}
if r.ContentLength > maxBytes {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
}
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
}
return nil, "", invalidRequest()
}
if len(data) == 0 {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
detected := http.DetectContentType(data)
if detected != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "content type does not match image"}
}
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || imageContentType(format) != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > maxPixels {
return nil, "", &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image dimensions are too large"}
}
decodeBytes := pixels*8 + int64(len(data))
if !decodeMemory.TryAcquire(decodeBytes) {
w.Header().Set("Retry-After", "1")
return nil, "", &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
}
defer decodeMemory.Release(decodeBytes)
if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType {
return nil, "", &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "invalid image"}
}
return data, mediaType, nil
}
func imageContentType(format string) string {
switch format {
case "png":
return "image/png"
case "jpeg":
return "image/jpeg"
default:
return ""
}
}
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
attachment, apiErr := s.attachmentForUser(r, user.ID)
if apiErr != nil {
return apiErr
}
writeJSON(w, http.StatusOK, attachment)
return nil
}
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
if !originalVariant(r.URL.Query()) {
return invalidRequest()
}
attachment, apiErr := s.attachmentForUser(r, user.ID)
if apiErr != nil {
return apiErr
}
content, err := s.objects.Get(r.Context(), attachment.ObjectKey)
if errors.Is(err, objectstore.ErrNotFound) {
return unavailable()
}
if err != nil {
return unavailable()
}
defer func() { _ = content.Close() }()
w.Header().Set("Content-Type", attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", attachment.ID, "error", err)
}
return nil
}
func originalVariant(query url.Values) bool {
if len(query) == 0 {
return true
}
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
}
func (s *Server) attachmentForUser(r *http.Request, userID string) (Attachment, *apiError) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
attachmentID, valid := canonicalID(parts[3])
if !valid {
return Attachment{}, invalidRequest()
}
attachment, err := s.messengerStore.AttachmentForUser(r.Context(), attachmentID, userID)
if errors.Is(err, postgres.ErrNotFound) {
return Attachment{}, notFound()
}
if err != nil {
return Attachment{}, unavailable()
}
return attachment, nil
}
+164
View File
@@ -0,0 +1,164 @@
package httpapi
import (
"errors"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
}()
if r.Method != http.MethodPost {
return methodNotAllowed(w, http.MethodPost)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4])
if !valid {
return invalidRequest()
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return invalidRequest()
}
filename, valid := attachmentFilename(query)
if !valid {
return invalidRequest()
}
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
})
if err != nil {
if errors.Is(err, model.ErrCapacity) {
result = "capacity_rejected"
w.Header().Set("Retry-After", "1")
} else if errors.Is(err, model.ErrUnavailable) {
result = "dependency_error"
}
return attachmentError(err)
}
result = "success"
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
writeJSON(w, http.StatusCreated, uploaded)
return nil
}
func attachmentFilename(query url.Values) (string, bool) {
if len(query) != 1 || len(query["filename"]) != 1 {
return "", false
}
filename := query["filename"][0]
length := utf8.RuneCountInString(filename)
if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 {
return "", false
}
for _, char := range filename {
if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) {
return "", false
}
}
return filename, true
}
func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
attachmentID, apiErr := attachmentIDFromRequest(r)
if apiErr != nil {
return apiErr
}
value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID)
if err != nil {
return attachmentError(err)
}
writeJSON(w, http.StatusOK, value)
return nil
}
func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError {
if r.Method != http.MethodGet {
return methodNotAllowed(w, http.MethodGet)
}
user, _, authErr := s.authenticate(r)
if authErr != nil {
return authErr
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || !originalVariant(query) {
return invalidRequest()
}
attachmentID, apiErr := attachmentIDFromRequest(r)
if apiErr != nil {
return apiErr
}
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
if err != nil {
return attachmentError(err)
}
defer func() { _ = content.Body.Close() }()
w.Header().Set("Content-Type", content.Attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content.Body); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
}
return nil
}
func originalVariant(query url.Values) bool {
if len(query) == 0 {
return true
}
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
}
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
attachmentID, valid := canonicalID(parts[3])
if !valid {
return "", invalidRequest()
}
return attachmentID, nil
}
func attachmentError(err error) *apiError {
switch {
case errors.Is(err, model.ErrNotFound):
return notFound()
case errors.Is(err, model.ErrInvalidInput):
return invalidRequest()
case errors.Is(err, model.ErrUnsupportedMedia):
return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"}
case errors.Is(err, model.ErrPayloadTooLarge):
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
case errors.Is(err, model.ErrCapacity):
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
default:
return unavailable()
}
}
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"errors"
@@ -7,8 +7,6 @@ import (
"strings"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
@@ -23,8 +21,8 @@ func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError
if !ok {
return unauthorized()
}
credentials, err := s.messengerStore.UserByUsername(r.Context(), username)
if errors.Is(err, postgres.ErrNotFound) {
credentials, err := s.userRepository.UserByUsername(r.Context(), username)
if errors.Is(err, model.ErrNotFound) {
if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil {
return internalError()
}
@@ -69,7 +67,7 @@ func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError
if authErr != nil {
return authErr
}
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, session.ErrNotFound) {
if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) {
return unauthorized()
} else if err != nil {
return unavailable()
@@ -88,14 +86,14 @@ func (s *Server) authenticate(r *http.Request) (User, string, *apiError) {
return User{}, "", unauthorized()
}
userID, err := s.sessions.UserID(r.Context(), token)
if errors.Is(err, session.ErrNotFound) || errors.Is(err, session.ErrInvalidToken) {
if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) {
return User{}, "", unauthorized()
}
if err != nil {
return User{}, "", unavailable()
}
user, err := s.messengerStore.UserByID(r.Context(), userID)
if errors.Is(err, postgres.ErrNotFound) {
user, err := s.userRepository.UserByID(r.Context(), userID)
if errors.Is(err, model.ErrNotFound) {
_ = s.sessions.Delete(r.Context(), token)
return User{}, "", unauthorized()
}
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"encoding/json"
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"context"
@@ -10,7 +10,6 @@ import (
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
@@ -32,7 +31,7 @@ func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError {
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
if err := s.messengerStore.Ping(ctx); err != nil {
if err := s.database.Ping(ctx); err != nil {
return unavailable()
}
if err := s.sessions.Ping(ctx); err != nil {
@@ -64,8 +63,8 @@ func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError {
if err != nil {
return internalError()
}
user, err := s.messengerStore.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
if errors.Is(err, postgres.ErrConflict) {
user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash)
if errors.Is(err, model.ErrConflict) {
return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"}
}
if err != nil {
@@ -128,8 +127,8 @@ func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError {
if _, included := members[user.ID]; !included {
return invalidRequest()
}
chat, err := s.messengerStore.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
if errors.Is(err, postgres.ErrInvalidMembers) {
chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID)
if errors.Is(err, model.ErrInvalidMembers) {
return invalidRequest()
}
if err != nil {
@@ -144,8 +143,8 @@ func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *a
if err != nil {
return err
}
items, hasNext, listErr := s.messengerStore.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, postgres.ErrInvalidCursor) {
items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, model.ErrInvalidCursor) {
return invalidRequest()
}
if listErr != nil {
@@ -179,8 +178,8 @@ func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError {
if !valid {
return invalidRequest()
}
chat, err := s.messengerStore.ChatForUser(r.Context(), chatID, user.ID)
if errors.Is(err, postgres.ErrNotFound) {
chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID)
if errors.Is(err, model.ErrNotFound) {
return notFound()
}
if err != nil {
@@ -219,13 +218,13 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
if !validMessage(input.Text, input.AttachmentIDs) || !canonicalAttachmentIDs(input.AttachmentIDs) {
return invalidRequest()
}
message, replay, err := s.messengerStore.CreateMessage(
message, replay, err := s.messageRepository.CreateMessage(
r.Context(), chatID, user.ID, input.Text, input.AttachmentIDs, key, s.idempotencyTTL,
)
if errors.Is(err, postgres.ErrNotFound) {
if errors.Is(err, model.ErrNotFound) {
return notFound()
}
if errors.Is(err, postgres.ErrIdempotencyConflict) {
if errors.Is(err, model.ErrIdempotencyConflict) {
return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"}
}
if err != nil {
@@ -269,11 +268,11 @@ func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User,
if err != nil {
return err
}
items, hasNext, listErr := s.messengerStore.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, postgres.ErrNotFound) {
items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit)
if errors.Is(listErr, model.ErrNotFound) {
return notFound()
}
if errors.Is(listErr, postgres.ErrInvalidCursor) {
if errors.Is(listErr, model.ErrInvalidCursor) {
return invalidRequest()
}
if listErr != nil {
@@ -1,37 +1,42 @@
package messenger
package httpapi
import (
"context"
"io"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
type MessengerStore interface {
Ping(context.Context) error
type UserRepository interface {
CreateUser(context.Context, string, string, string) (model.User, error)
UserByID(context.Context, string) (model.User, error)
UserByUsername(context.Context, string) (model.UserCredentials, error)
}
type ChatRepository interface {
CreateChat(context.Context, string, []string, string) (model.Chat, error)
ChatForUser(context.Context, string, string) (model.Chat, error)
ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error)
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
CreateAttachment(context.Context, model.Attachment) (model.Attachment, error)
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
}
type ObjectStore interface {
type MessageRepository interface {
CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error)
ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error)
}
type AttachmentService interface {
Upload(context.Context, attachment.Upload) (model.Attachment, error)
Metadata(context.Context, string, string) (model.Attachment, error)
Content(context.Context, string, string) (attachment.Content, error)
}
type HealthChecker interface {
Ping(context.Context) error
Put(context.Context, string, string, []byte) error
Get(context.Context, string) (io.ReadCloser, error)
Delete(context.Context, string) error
}
type SessionStore interface {
Ping(context.Context) error
HealthChecker
Create(context.Context, string, time.Duration) (string, time.Time, error)
UserID(context.Context, string) (string, error)
Delete(context.Context, string) error
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"net/http"
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"bytes"
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"net/http"
@@ -1,4 +1,4 @@
package messenger
package httpapi
import (
"context"
@@ -6,26 +6,44 @@ import (
"net/http"
"sync/atomic"
"time"
"golang.org/x/sync/semaphore"
)
type Dependencies struct {
Users UserRepository
Chats ChatRepository
Messages MessageRepository
Attachments AttachmentService
Sessions SessionStore
Passwords PasswordService
Database HealthChecker
Objects HealthChecker
}
type Config struct {
SessionTTL time.Duration
IdempotencyTTL time.Duration
RequestTTL time.Duration
InstanceID string
SwaggerPort string
}
type Server struct {
messengerStore MessengerStore
sessions SessionStore
passwords PasswordService
objects ObjectStore
sessionTTL time.Duration
idempotencyTTL time.Duration
requestTTL time.Duration
instanceID string
logger *slog.Logger
swaggerPort string
maxUploadBytes int64
maxImagePixels int64
decodeMemory *semaphore.Weighted
metrics *metrics
ready atomic.Bool
userRepository UserRepository
chatRepository ChatRepository
messageRepository MessageRepository
attachments AttachmentService
sessions SessionStore
passwords PasswordService
database HealthChecker
objects HealthChecker
sessionTTL time.Duration
idempotencyTTL time.Duration
requestTTL time.Duration
instanceID string
logger *slog.Logger
swaggerPort string
metrics *metrics
ready atomic.Bool
}
type statusWriter struct {
@@ -40,25 +58,16 @@ func (w *statusWriter) WriteHeader(status int) {
func NewServer(
logger *slog.Logger,
messages MessengerStore,
sessions SessionStore,
passwords PasswordService,
objects ObjectStore,
sessionTTL time.Duration,
idempotencyTTL time.Duration,
requestTTL time.Duration,
instanceID string,
swaggerPort string,
maxUploadBytes int64,
maxImagePixels int64,
decodeMemoryBytes int64,
dependencies Dependencies,
config Config,
) *Server {
s := &Server{
messengerStore: messages, sessions: sessions, passwords: passwords, objects: objects,
sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, requestTTL: requestTTL,
instanceID: instanceID, logger: logger, swaggerPort: swaggerPort,
maxUploadBytes: maxUploadBytes, maxImagePixels: maxImagePixels,
decodeMemory: semaphore.NewWeighted(decodeMemoryBytes), metrics: newMetrics(),
userRepository: dependencies.Users, chatRepository: dependencies.Chats,
messageRepository: dependencies.Messages,
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
}
s.ready.Store(true)
return s
@@ -1,13 +1,9 @@
package messenger
package httpapi
import (
"bytes"
"context"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"net/http"
@@ -17,10 +13,9 @@ import (
"time"
"github.com/stretchr/testify/suite"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
)
var errDependency = errors.New("dependency unavailable")
@@ -39,19 +34,14 @@ func (s *MessengerSuite) SetupTest() {
}
type messengerStoreStub struct {
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
attachment model.Attachment
attachmentErr error
storedAttachment model.Attachment
storedAttachmentExists bool
storedAttachmentErr error
pingErr error
userByID model.User
userByIDErr error
credentials model.UserCredentials
credentialsErr error
message model.Message
messageReplay bool
messageErr error
}
func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr }
@@ -79,50 +69,26 @@ func (s messengerStoreStub) CreateMessage(context.Context, string, string, strin
func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) {
return nil, false, nil
}
func (s messengerStoreStub) CreateAttachment(context.Context, model.Attachment) (model.Attachment, error) {
return s.attachment, s.attachmentErr
}
func (s messengerStoreStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
if s.storedAttachmentErr != nil {
return model.Attachment{}, s.storedAttachmentErr
}
if !s.storedAttachmentExists {
return model.Attachment{}, postgres.ErrNotFound
}
return s.storedAttachment, nil
}
func (s messengerStoreStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
return s.attachment, s.attachmentErr
}
type objectStoreStub struct {
pingErr error
putErr error
getData []byte
getErr error
deleteErr error
putKey *string
deletedKey *string
pingErr error
}
func (s objectStoreStub) Ping(context.Context) error { return s.pingErr }
func (s objectStoreStub) Put(_ context.Context, key string, _ string, _ []byte) error {
if s.putKey != nil {
*s.putKey = key
}
return s.putErr
type attachmentServiceStub struct {
uploaded model.Attachment
err error
}
func (s objectStoreStub) Get(context.Context, string) (io.ReadCloser, error) {
if s.getErr != nil {
return nil, s.getErr
}
return io.NopCloser(bytes.NewReader(s.getData)), nil
func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) {
return s.uploaded, s.err
}
func (s objectStoreStub) Delete(_ context.Context, key string) error {
if s.deletedKey != nil {
*s.deletedKey = key
}
return s.deleteErr
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
return s.uploaded, s.err
}
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
}
type sessionStub struct {
@@ -164,10 +130,7 @@ func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := NewServer(
s.logger, tt.store, tt.sessions, passwordsStub{}, tt.objects, time.Hour, 24*time.Hour,
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{})
response := serveRequest(server, http.MethodGet, "/health/ready", "")
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
@@ -300,7 +263,7 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
t := s.T()
store := messengerStoreStub{
userByID: model.User{ID: "user-id"},
messageErr: fmt.Errorf("create message: %w", postgres.ErrIdempotencyConflict),
messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict),
}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`))
@@ -314,52 +277,6 @@ func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDiffe
}
}
func (s *MessengerSuite) TestImageValidation() {
data := testPNG(s.T(), 2, 2)
tests := []struct {
name string
contentType string
data []byte
maxBytes int64
maxPixels int64
wantStatus int
}{
{name: "valid", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 4},
{name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
{name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, wantStatus: http.StatusRequestEntityTooLarge},
{name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, wantStatus: http.StatusRequestEntityTooLarge},
{name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, wantStatus: http.StatusUnsupportedMediaType},
}
for _, tt := range tests {
s.Run(tt.name, func() {
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(tt.data))
request.Header.Set("Content-Type", tt.contentType)
response := httptest.NewRecorder()
actual, _, apiErr := readImage(
response, request, tt.maxBytes, tt.maxPixels, semaphore.NewWeighted(384*1024*1024),
)
if tt.wantStatus == 0 {
s.Require().Nil(apiErr)
s.Equal(tt.data, actual)
return
}
s.Require().NotNil(apiErr)
s.Equal(tt.wantStatus, apiErr.status)
})
}
}
func (s *MessengerSuite) TestImageValidationLimitsConcurrentDecodeMemory() {
data := testPNG(s.T(), 2, 2)
request := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(data))
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
_, _, apiErr := readImage(response, request, 1024, 4, semaphore.NewWeighted(1))
s.Require().NotNil(apiErr)
s.Equal(http.StatusTooManyRequests, apiErr.status)
s.Equal("1", response.Header().Get("Retry-After"))
}
func (s *MessengerSuite) TestAttachmentFilenameValidation() {
for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} {
_, valid := attachmentFilename(map[string][]string{"filename": {filename}})
@@ -370,79 +287,39 @@ func (s *MessengerSuite) TestAttachmentFilenameValidation() {
s.Equal("фото.png", filename)
}
func (s *MessengerSuite) TestUploadCompensatesObjectWhenMetadataWriteFails() {
var putKey, deletedKey string
store := messengerStoreStub{
userByID: model.User{ID: "00000000-0000-0000-0000-000000000001"},
attachmentErr: errDependency,
}
objects := objectStoreStub{putKey: &putKey, deletedKey: &deletedKey}
server := NewServer(
s.logger, store, sessionStub{userID: store.userByID.ID}, passwordsStub{}, objects,
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
data := testPNG(s.T(), 2, 2)
func (s *MessengerSuite) TestUploadRejectsMalformedQuery() {
store := messengerStoreStub{userByID: model.User{ID: "user-id"}}
server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{})
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
bytes.NewReader(data),
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG",
strings.NewReader("image"),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
s.Equal(http.StatusServiceUnavailable, response.Code)
s.NotEmpty(putKey)
s.Equal(putKey, deletedKey)
s.Equal(http.StatusBadRequest, response.Code)
}
func (s *MessengerSuite) TestUploadKeepsObjectWhenMetadataCommitSucceeded() {
var deletedKey string
stored := model.Attachment{
ID: "00000000-0000-0000-0000-000000000003",
ChatID: "00000000-0000-0000-0000-000000000002",
OwnerID: "00000000-0000-0000-0000-000000000001",
Filename: "photo.png", ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("0", 64),
Status: "ready", Variants: []model.Variant{}, CreatedAt: time.Now(), UpdatedAt: time.Now(),
}
store := messengerStoreStub{
userByID: model.User{ID: stored.OwnerID}, attachmentErr: errDependency,
storedAttachment: stored, storedAttachmentExists: true,
}
server := NewServer(
s.logger, store, sessionStub{userID: stored.OwnerID}, passwordsStub{}, objectStoreStub{deletedKey: &deletedKey},
time.Hour, 24*time.Hour, time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
data := testPNG(s.T(), 2, 2)
request := httptest.NewRequest(
http.MethodPost,
"/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png",
bytes.NewReader(data),
)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "image/png")
response := httptest.NewRecorder()
server.Handler().ServeHTTP(response, request)
s.Equal(http.StatusCreated, response.Code)
s.Empty(deletedKey)
func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server {
return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{})
}
func testPNG(t *testing.T, width, height int) []byte {
t.Helper()
var data bytes.Buffer
imageData := image.NewRGBA(image.Rect(0, 0, width, height))
imageData.Set(0, 0, color.White)
if err := png.Encode(&data, imageData); err != nil {
t.Fatalf("encode PNG: %v", err)
}
return data.Bytes()
}
func (s *MessengerSuite) newTestServer(messages MessengerStore, sessions SessionStore, passwords PasswordService) *Server {
return NewServer(
s.logger, messages, sessions, passwords, objectStoreStub{}, time.Hour, 24*time.Hour,
time.Second, "test-instance", "8081", 10*1024*1024, 40_000_000, 384*1024*1024,
)
func (s *MessengerSuite) server(
store messengerStoreStub,
sessions SessionStore,
passwords PasswordService,
objects objectStoreStub,
attachments AttachmentService,
) *Server {
return NewServer(s.logger, Dependencies{
Users: store, Chats: store, Messages: store, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: store, Objects: objects,
}, Config{
SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second,
InstanceID: "test-instance", SwaggerPort: "8081",
})
}
func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder {
+18
View File
@@ -0,0 +1,18 @@
package model
import "errors"
var (
ErrNotFound = errors.New("not found")
ErrConflict = errors.New("conflict")
ErrInvalidMembers = errors.New("invalid members")
ErrInvalidCursor = errors.New("invalid cursor")
ErrIdempotencyConflict = errors.New("idempotency conflict")
ErrInvalidToken = errors.New("invalid token")
ErrSessionNotFound = errors.New("session not found")
ErrInvalidInput = errors.New("invalid input")
ErrUnsupportedMedia = errors.New("unsupported media")
ErrPayloadTooLarge = errors.New("payload too large")
ErrCapacity = errors.New("capacity exhausted")
ErrUnavailable = errors.New("dependency unavailable")
)
+27 -13
View File
@@ -5,6 +5,12 @@ import (
"time"
)
const (
AttachmentPending = "pending"
AttachmentReady = "ready"
AttachmentDeleting = "deleting"
)
const timestampLayout = "2006-01-02T15:04:05.000000Z"
type User struct {
@@ -36,19 +42,27 @@ type Message struct {
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
ReadyAt *time.Time `json:"-"`
Referenced bool `json:"-"`
}
type StoredObject struct {
Key string
Size int64
LastModified time.Time
}
type Variant struct {
+8 -12
View File
@@ -12,6 +12,8 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
var ErrNotFound = errors.New("object not found")
@@ -26,12 +28,6 @@ type Config struct {
Timeout time.Duration
}
type Object struct {
Key string
Size int64
LastModified time.Time
}
type Store struct {
client *minio.Client
bucket string
@@ -115,14 +111,14 @@ func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
}
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return Object{}, objectError("stat S3 object", err)
return model.StoredObject{}, objectError("stat S3 object", err)
}
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
}
func (s *Store) Delete(ctx context.Context, key string) error {
@@ -134,19 +130,19 @@ func (s *Store) Delete(ctx context.Context, key string) error {
return nil
}
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]Object, string, error) {
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", errors.New("object list limit must be between 1 and 1000")
}
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
objects := make([]Object, 0, limit+1)
objects := make([]model.StoredObject, 0, limit+1)
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
for info := range s.client.ListObjects(ctx, s.bucket, options) {
if info.Err != nil {
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
}
objects = append(objects, Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
if len(objects) == limit+1 {
cancel()
break
+133 -57
View File
@@ -11,90 +11,115 @@ import (
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
func (s *Store) CreateAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
const query = `
const attachmentColumns = `
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, created_at, updated_at, ready_at`
type rowScanner interface {
Scan(...any) error
}
func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) {
query := `
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at
)
SELECT $1, $2, $3, $4, $5, $6, $7, $8
SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL
WHERE EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3
)
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, created_at, updated_at`
RETURNING ` + attachmentColumns
err := s.pool.QueryRow(
err := scanAttachment(s.pool.QueryRow(
ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename,
attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey,
).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("create attachment: %w", err)
return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err)
}
return readyAttachment(attachment), nil
return publicAttachment(attachment), nil
}
func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
query := `
UPDATE attachments
SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE id = $1 AND object_key = $2 AND status = 'pending'
RETURNING ` + attachmentColumns
var attachment model.Attachment
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error {
const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'`
if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil {
return fmt.Errorf("delete pending attachment: %w", err)
}
return nil
}
func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) {
const query = `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
a.sha256, a.object_key, a.created_at, a.updated_at
query := `
SELECT ` + attachmentColumnsWithAlias("a") + `
FROM attachments a
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
WHERE a.id = $1`
WHERE a.id = $1 AND a.status = 'ready'`
var attachment model.Attachment
err := s.pool.QueryRow(ctx, query, attachmentID, userID).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
}
return readyAttachment(attachment), nil
return publicAttachment(attachment), nil
}
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
const query = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, created_at, updated_at
FROM attachments
WHERE id = $1 AND object_key = $2`
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
var attachment model.Attachment
err := s.pool.QueryRow(ctx, query, attachmentID, objectKey).Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err)
}
return readyAttachment(attachment), nil
return publicAttachment(attachment), nil
}
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
var exists bool
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
return false, fmt.Errorf("check attachment by object key: %w", err)
}
return exists, nil
}
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
if limit < 1 || limit > 1000 {
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
}
const query = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, created_at, updated_at
FROM attachments
WHERE object_key > $1
ORDER BY object_key
query := `
SELECT ` + attachmentColumnsWithAlias("a") + `,
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
FROM attachments a
WHERE a.object_key > $1
ORDER BY a.object_key
LIMIT $2`
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
if err != nil {
@@ -104,12 +129,8 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
var attachment model.Attachment
err := row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.CreatedAt, &attachment.UpdatedAt,
)
return readyAttachment(attachment), err
err := scanAttachmentWithReference(row, &attachment)
return publicAttachment(attachment), err
}))
if err != nil {
return nil, "", fmt.Errorf("scan attachments: %w", err)
@@ -122,6 +143,15 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
return attachments, next, nil
}
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
&attachment.Referenced,
)
}
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`
var referenced bool
@@ -131,25 +161,71 @@ func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (
return referenced, nil
}
func (s *Store) DeleteAttachmentIfUnreferencedBefore(
ctx context.Context,
attachmentID string,
createdBefore time.Time,
) (bool, error) {
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, attachmentID string, createdBefore time.Time) (bool, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
var createdAt time.Time
err = tx.QueryRow(ctx, `SELECT created_at FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID).Scan(&createdAt)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
}
if createdAt.After(createdBefore) {
return false, nil
}
var referenced bool
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, attachmentID).Scan(&referenced); err != nil {
return false, fmt.Errorf("recheck attachment references: %w", err)
}
if referenced {
return false, nil
}
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1`, attachmentID); err != nil {
return false, fmt.Errorf("mark attachment deleting: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return false, fmt.Errorf("commit attachment deletion claim: %w", err)
}
return true, nil
}
func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) {
const query = `
DELETE FROM attachments a
WHERE a.id = $1
AND a.created_at <= $2
AND a.status = 'deleting'
AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)`
result, err := s.pool.Exec(ctx, query, attachmentID, createdBefore)
result, err := s.pool.Exec(ctx, query, attachmentID)
if err != nil {
return false, fmt.Errorf("delete old unreferenced attachment: %w", err)
return false, fmt.Errorf("delete claimed attachment: %w", err)
}
return result.RowsAffected() == 1, nil
}
func readyAttachment(attachment model.Attachment) model.Attachment {
attachment.Status = "ready"
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
)
}
func attachmentColumnsWithAlias(alias string) string {
return fmt.Sprintf(
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
alias,
)
}
func publicAttachment(attachment model.Attachment) model.Attachment {
attachment.Variants = []model.Variant{}
return attachment
}
+6 -6
View File
@@ -1,11 +1,11 @@
package postgres
import "errors"
import "gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
var (
ErrConflict = errors.New("conflict")
ErrNotFound = errors.New("not found")
ErrInvalidMembers = errors.New("invalid members")
ErrInvalidCursor = errors.New("invalid cursor")
ErrIdempotencyConflict = errors.New("idempotency conflict")
ErrConflict = model.ErrConflict
ErrNotFound = model.ErrNotFound
ErrInvalidMembers = model.ErrInvalidMembers
ErrInvalidCursor = model.ErrInvalidCursor
ErrIdempotencyConflict = model.ErrIdempotencyConflict
)
+13 -4
View File
@@ -97,11 +97,20 @@ func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, atta
return true, nil
}
const query = `
SELECT count(*)
SELECT id
FROM attachments
WHERE chat_id = $1 AND id::text = ANY($2)`
var count int
if err := tx.QueryRow(ctx, query, chatID, attachmentIDs).Scan(&count); err != nil {
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
FOR UPDATE`
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
if err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
}
defer rows.Close()
count := 0
for rows.Next() {
count++
}
if err := rows.Err(); err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
}
return count == len(attachmentIDs), nil
+7 -3
View File
@@ -126,7 +126,9 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
}
created, err := s.store.CreateAttachment(context.Background(), attachment)
pending, err := s.store.CreatePendingAttachment(context.Background(), attachment)
s.Require().NoError(err)
created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
s.Equal("ready", created.Status)
s.Empty(created.Variants)
@@ -152,7 +154,7 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
s.Require().NoError(err)
s.True(referenced)
deleted, err := s.store.DeleteAttachmentIfUnreferencedBefore(context.Background(), created.ID, time.Now().Add(time.Hour))
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created.ID, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(deleted)
}
@@ -161,12 +163,14 @@ func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
s.Require().NoError(err)
attachment, err := s.store.CreateAttachment(context.Background(), model.Attachment{
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
_, _, err = s.store.CreateMessage(
context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour,
)
+143 -40
View File
@@ -2,43 +2,50 @@ package reconcile
import (
"context"
"errors"
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
)
type MetadataStore interface {
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
AttachmentReferenced(context.Context, string) (bool, error)
DeleteAttachmentIfUnreferencedBefore(context.Context, string, time.Time) (bool, error)
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
ClaimAttachmentForDeletion(context.Context, string, time.Time) (bool, error)
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
}
type ObjectStore interface {
List(context.Context, string, string, int) ([]objectstore.Object, string, error)
List(context.Context, string, string, int) ([]model.StoredObject, string, error)
Delete(context.Context, string) error
}
type Config struct {
Prefix string
PageSize int
GraceTime time.Duration
Apply bool
Now func() time.Time
Prefix string
PageSize int
GraceTime time.Duration
UnreferencedRetention time.Duration
Apply bool
Now func() time.Time
}
type Report struct {
MetadataScanned int `json:"metadata_scanned"`
ObjectsScanned int `json:"objects_scanned"`
Consistent int `json:"consistent"`
FreshOrphanObjects int `json:"fresh_orphan_objects"`
OldOrphanObjects int `json:"old_orphan_objects"`
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
FreshMissingObjects int `json:"fresh_missing_objects"`
ReferencedMissingObjects int `json:"referenced_missing_objects"`
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
MetadataScanned int `json:"metadata_scanned"`
ObjectsScanned int `json:"objects_scanned"`
Consistent int `json:"consistent"`
FreshPendingAttachments int `json:"fresh_pending_attachments"`
OldPendingAttachments int `json:"old_pending_attachments"`
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
FreshOrphanObjects int `json:"fresh_orphan_objects"`
OldOrphanObjects int `json:"old_orphan_objects"`
DeletedOrphanObjects int `json:"deleted_orphan_objects"`
FreshMissingObjects int `json:"fresh_missing_objects"`
ReferencedMissingObjects int `json:"referenced_missing_objects"`
OldUnreferencedMetadata int `json:"old_unreferenced_metadata"`
DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"`
}
type Reconciler struct {
@@ -49,10 +56,13 @@ type Reconciler struct {
func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) {
if config.PageSize < 1 || config.PageSize > 1000 {
return nil, fmt.Errorf("page size must be between 1 and 1000")
return nil, errors.New("page size must be between 1 and 1000")
}
if config.GraceTime < 0 {
return nil, fmt.Errorf("grace time must not be negative")
return nil, errors.New("grace time must not be negative")
}
if config.UnreferencedRetention <= 0 {
return nil, errors.New("unreferenced retention must be positive")
}
if config.Now == nil {
config.Now = time.Now
@@ -70,28 +80,30 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) {
return Report{}, err
}
cutoff := r.config.Now().Add(-r.config.GraceTime)
now := r.config.Now()
anomalyCutoff := now.Add(-r.config.GraceTime)
retentionCutoff := now.Add(-r.config.UnreferencedRetention)
var report Report
for metadata.current != nil || objects.current != nil {
switch {
case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey:
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, &report); err != nil {
if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil {
return report, err
}
if err := objects.advance(ctx); err != nil {
return report, err
}
case objects.current == nil || metadata.current.ObjectKey < objects.current.Key:
if err := r.handleMissingObject(ctx, *metadata.current, cutoff, &report); err != nil {
if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil {
return report, err
}
if err := metadata.advance(ctx); err != nil {
return report, err
}
default:
report.MetadataScanned++
report.ObjectsScanned++
report.Consistent++
if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil {
return report, err
}
if err := metadata.advance(ctx); err != nil {
return report, err
}
@@ -103,7 +115,83 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) {
return report, nil
}
func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.Object, cutoff time.Time, report *Report) error {
func (r *Reconciler) handlePair(
ctx context.Context,
metadata model.Attachment,
retentionCutoff time.Time,
anomalyCutoff time.Time,
report *Report,
) error {
report.MetadataScanned++
report.ObjectsScanned++
report.Consistent++
switch metadata.Status {
case model.AttachmentPending:
if metadata.CreatedAt.After(anomalyCutoff) {
report.FreshPendingAttachments++
return nil
}
report.OldPendingAttachments++
return r.deletePair(ctx, metadata, anomalyCutoff, report)
case model.AttachmentDeleting:
return r.deleteClaimedPair(ctx, metadata, report)
case model.AttachmentReady:
if metadata.Referenced {
return nil
}
age := metadata.CreatedAt
if metadata.ReadyAt != nil {
age = *metadata.ReadyAt
}
if age.After(retentionCutoff) {
report.FreshUnreferencedAttachments++
return nil
}
report.OldUnreferencedAttachments++
return r.deletePair(ctx, metadata, retentionCutoff, report)
default:
return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status)
}
}
func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error {
if !r.config.Apply {
return nil
}
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment.ID, cutoff)
if err != nil {
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
}
if !claimed {
return nil
}
return r.deleteClaimedPair(ctx, attachment, report)
}
func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error {
if !r.config.Apply {
return nil
}
if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err)
}
report.DeletedAttachmentObjects++
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID)
if err != nil {
return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err)
}
if deleted {
report.DeletedUnreferencedRows++
}
return nil
}
func (r *Reconciler) handleOrphanObject(
ctx context.Context,
object model.StoredObject,
cutoff time.Time,
report *Report,
) error {
report.ObjectsScanned++
if object.LastModified.After(cutoff) {
report.FreshOrphanObjects++
@@ -113,6 +201,13 @@ func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.
if !r.config.Apply {
return nil
}
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
if err != nil {
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
}
if exists {
return nil
}
if err := r.objects.Delete(ctx, object.Key); err != nil {
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
}
@@ -120,17 +215,18 @@ func (r *Reconciler) handleOrphanObject(ctx context.Context, object objectstore.
return nil
}
func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Attachment, cutoff time.Time, report *Report) error {
func (r *Reconciler) handleMissingObject(
ctx context.Context,
metadata model.Attachment,
cutoff time.Time,
report *Report,
) error {
report.MetadataScanned++
if metadata.CreatedAt.After(cutoff) {
report.FreshMissingObjects++
return nil
}
referenced, err := r.metadata.AttachmentReferenced(ctx, metadata.ID)
if err != nil {
return fmt.Errorf("check attachment metadata %q: %w", metadata.ID, err)
}
if referenced {
if metadata.Referenced {
report.ReferencedMissingObjects++
return nil
}
@@ -138,9 +234,16 @@ func (r *Reconciler) handleMissingObject(ctx context.Context, metadata model.Att
if !r.config.Apply {
return nil
}
deleted, err := r.metadata.DeleteAttachmentIfUnreferencedBefore(ctx, metadata.ID, cutoff)
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata.ID, cutoff)
if err != nil {
return fmt.Errorf("delete attachment metadata %q: %w", metadata.ID, err)
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
}
if !claimed {
return nil
}
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
if err != nil {
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
}
if deleted {
report.DeletedUnreferencedRows++
@@ -186,7 +289,7 @@ func (i *metadataIterator) advance(ctx context.Context) error {
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return fmt.Errorf("metadata page is empty but has continuation")
return errors.New("metadata page is empty but has continuation")
}
}
}
@@ -196,9 +299,9 @@ type objectIterator struct {
prefix string
pageSize int
after string
page []objectstore.Object
page []model.StoredObject
index int
current *objectstore.Object
current *model.StoredObject
done bool
}
@@ -230,7 +333,7 @@ func (i *objectIterator) advance(ctx context.Context) error {
i.after = next
i.done = next == ""
if len(page) == 0 && !i.done {
return fmt.Errorf("object page is empty but has continuation")
return errors.New("object page is empty but has continuation")
}
}
}
+120 -77
View File
@@ -9,88 +9,133 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
)
func TestReconcilerDryRunAndApply(t *testing.T) {
now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC)
metadata := &metadataStub{items: []model.Attachment{
{ID: "consistent", ObjectKey: "attachments/consistent", CreatedAt: now.Add(-time.Hour)},
{ID: "fresh-missing", ObjectKey: "attachments/fresh-missing", CreatedAt: now.Add(-time.Minute)},
{ID: "old-missing", ObjectKey: "attachments/old-missing", CreatedAt: now.Add(-time.Hour)},
{ID: "referenced-missing", ObjectKey: "attachments/referenced-missing", CreatedAt: now.Add(-time.Hour)},
}, referenced: map[string]bool{"referenced-missing": true}}
objects := &objectStub{items: []objectstore.Object{
{Key: "attachments/consistent", LastModified: now.Add(-time.Hour)},
{Key: "attachments/fresh-orphan", LastModified: now.Add(-time.Minute)},
{Key: "attachments/old-orphan", LastModified: now.Add(-time.Hour)},
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
old := now.Add(-48 * time.Hour)
fresh := now.Add(-10 * time.Minute)
metadata := &metadataStub{items: map[string]model.Attachment{
"attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false),
"attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true),
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
}}
dryRun, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 2, GraceTime: 10 * time.Minute, Now: func() time.Time { return now },
objects := &objectStub{items: map[string]model.StoredObject{
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
"attachments/b/original": {Key: "attachments/b/original", LastModified: old},
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
}}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := dryRun.Run(context.Background())
require.NoError(t, err)
require.Equal(t, Report{
MetadataScanned: 4, ObjectsScanned: 3, Consistent: 1,
FreshOrphanObjects: 1, OldOrphanObjects: 1, FreshMissingObjects: 1,
ReferencedMissingObjects: 1, OldUnreferencedMetadata: 1,
}, report)
require.Empty(t, metadata.deleted)
require.Empty(t, objects.deleted)
apply, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 2, GraceTime: 10 * time.Minute, Apply: true,
Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err = apply.Run(context.Background())
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Equal(t, 1, report.OldPendingAttachments)
require.Equal(t, 1, report.OldUnreferencedAttachments)
require.Equal(t, 1, report.FreshUnreferencedAttachments)
require.Equal(t, 1, report.DeletedOrphanObjects)
require.Equal(t, 1, report.DeletedUnreferencedRows)
require.Equal(t, []string{"old-missing"}, metadata.deleted)
require.Equal(t, []string{"attachments/old-orphan"}, objects.deleted)
require.Equal(t, 2, report.DeletedAttachmentObjects)
require.Equal(t, 3, report.DeletedUnreferencedRows)
require.Contains(t, metadata.items, "attachments/b/original")
require.Contains(t, metadata.items, "attachments/d/original")
require.NotContains(t, metadata.items, "attachments/a/original")
require.NotContains(t, metadata.items, "attachments/c/original")
require.NotContains(t, metadata.items, "attachments/missing/original")
require.NotContains(t, objects.items, "attachments/orphan/original")
report, err = apply.Run(context.Background())
repeated, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Zero(t, repeated.DeletedOrphanObjects)
require.Zero(t, repeated.DeletedAttachmentObjects)
require.Zero(t, repeated.DeletedUnreferencedRows)
}
func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) {
now := time.Now().UTC()
metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}}
objects := &objectStub{items: map[string]model.StoredObject{
"attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)},
}}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute,
UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Zero(t, report.DeletedOrphanObjects)
require.Zero(t, report.DeletedUnreferencedRows)
require.Equal(t, 1, report.FreshOrphanObjects)
require.Equal(t, 1, report.FreshMissingObjects)
require.Equal(t, 1, report.ReferencedMissingObjects)
require.Contains(t, objects.items, "attachments/a/original")
}
func TestNewRejectsInvalidRetention(t *testing.T) {
_, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1})
require.Error(t, err)
}
func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment {
readyAt := createdAt
return model.Attachment{
ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt,
ReadyAt: &readyAt, Referenced: referenced,
}
}
type metadataStub struct {
items []model.Attachment
referenced map[string]bool
deleted []string
items map[string]model.Attachment
appeared map[string]bool
}
func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) {
items := append([]model.Attachment(nil), s.items...)
sort.Slice(items, func(i, j int) bool { return items[i].ObjectKey < items[j].ObjectKey })
start := 0
for start < len(items) && items[start].ObjectKey <= after {
start++
keys := make([]string, 0, len(s.items))
for key := range s.items {
if key > after {
keys = append(keys, key)
}
}
end := min(start+limit, len(items))
sort.Strings(keys)
var next string
if end < len(items) {
next = items[end-1].ObjectKey
if len(keys) > limit {
keys = keys[:limit]
next = keys[len(keys)-1]
}
return items[start:end], next, nil
items := make([]model.Attachment, 0, len(keys))
for _, key := range keys {
items = append(items, s.items[key])
}
return items, next, nil
}
func (s *metadataStub) AttachmentReferenced(_ context.Context, id string) (bool, error) {
return s.referenced[id], nil
func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) {
if s.appeared[key] {
return true, nil
}
_, exists := s.items[key]
return exists, nil
}
func (s *metadataStub) DeleteAttachmentIfUnreferencedBefore(_ context.Context, id string, _ time.Time) (bool, error) {
for index, item := range s.items {
if item.ID == id && !s.referenced[id] {
s.items = append(s.items[:index], s.items[index+1:]...)
s.deleted = append(s.deleted, id)
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, id string, cutoff time.Time) (bool, error) {
for key, item := range s.items {
if item.ID == id && !item.CreatedAt.After(cutoff) && !item.Referenced {
item.Status = model.AttachmentDeleting
s.items[key] = item
return true, nil
}
}
return false, nil
}
func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) {
for key, item := range s.items {
if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced {
delete(s.items, key)
return true, nil
}
}
@@ -98,32 +143,30 @@ func (s *metadataStub) DeleteAttachmentIfUnreferencedBefore(_ context.Context, i
}
type objectStub struct {
items []objectstore.Object
deleted []string
items map[string]model.StoredObject
}
func (s *objectStub) List(_ context.Context, _ string, after string, limit int) ([]objectstore.Object, string, error) {
items := append([]objectstore.Object(nil), s.items...)
sort.Slice(items, func(i, j int) bool { return items[i].Key < items[j].Key })
start := 0
for start < len(items) && items[start].Key <= after {
start++
func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
keys := make([]string, 0, len(s.items))
for key := range s.items {
if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix {
keys = append(keys, key)
}
}
end := min(start+limit, len(items))
sort.Strings(keys)
var next string
if end < len(items) {
next = items[end-1].Key
if len(keys) > limit {
keys = keys[:limit]
next = keys[len(keys)-1]
}
return items[start:end], next, nil
items := make([]model.StoredObject, 0, len(keys))
for _, key := range keys {
items = append(items, s.items[key])
}
return items, next, nil
}
func (s *objectStub) Delete(_ context.Context, key string) error {
for index, item := range s.items {
if item.Key == key {
s.items = append(s.items[:index], s.items[index+1:]...)
s.deleted = append(s.deleted, key)
return nil
}
}
delete(s.items, key)
return nil
}
+5 -3
View File
@@ -13,6 +13,8 @@ import (
"time"
"github.com/valkey-io/valkey-go"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/model"
)
const (
@@ -21,12 +23,12 @@ const (
)
var (
ErrNotFound = errors.New("session not found")
ErrInvalidToken = errors.New("invalid session token")
ErrNotFound = model.ErrSessionNotFound
ErrInvalidToken = model.ErrInvalidToken
ErrInvalidRecord = errors.New("invalid session record")
ErrInvalidTTL = errors.New("invalid session TTL")
ErrTokenGeneration = errors.New("session token generation failed")
ErrStore = errors.New("session store failure")
ErrStore = model.ErrUnavailable
)
type record struct {
+16
View File
@@ -0,0 +1,16 @@
ALTER TABLE attachments
ADD COLUMN status TEXT NOT NULL DEFAULT 'ready',
ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp();
UPDATE attachments
SET ready_at = created_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')),
ADD CONSTRAINT attachments_ready_at CHECK (
(status = 'pending' AND ready_at IS NULL) OR
(status IN ('ready', 'deleting') AND ready_at IS NOT NULL)
);
CREATE INDEX attachments_status_created_at_object_key_idx
ON attachments (status, created_at, object_key);