feat: add S3 attachments and reconciliation

This commit is contained in:
2026-10-02 19:55:52 +03:00
parent 7e60f099d3
commit efaeb593e8
21 changed files with 1623 additions and 85 deletions
+84 -1
View File
@@ -2,20 +2,26 @@ package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
)
@@ -34,7 +40,10 @@ func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 1 && arguments[0] == "migrate" {
return runMigrations()
}
return fmt.Errorf("usage: messenger [migrate]")
if len(arguments) > 0 && arguments[0] == "reconcile-attachments" {
return runAttachmentReconciliation(arguments[1:], os.Stdout)
}
return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]")
}
func runServer(logger *slog.Logger) error {
@@ -68,16 +77,28 @@ func runServer(logger *slog.Logger) error {
if err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
app := messenger.NewServer(
logger,
postgres.New(pool),
sessions,
passwords,
objects,
settings.SessionTTL,
settings.IdempotencyTTL,
settings.RequestTimeout,
instanceID,
settings.SwaggerPort,
settings.AttachmentMaxBytes,
settings.AttachmentMaxPixels,
settings.AttachmentDecodeMemory,
)
server := &http.Server{
Addr: settings.Address,
@@ -130,3 +151,65 @@ func runMigrations() error {
defer pool.Close()
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
}
type reconciliationOutput struct {
Mode string `json:"mode"`
GracePeriod string `json:"grace_period"`
PageSize int `json:"page_size"`
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func runAttachmentReconciliation(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
flags.SetOutput(io.Discard)
apply := flags.Bool("apply", false, "apply safe repairs")
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]")
}
settings, err := config.Load()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply,
})
if err != nil {
return err
}
started := time.Now()
report, err := reconciler.Run(ctx)
if err != nil {
return err
}
mode := "dry-run"
if *apply {
mode = "apply"
}
return json.NewEncoder(output).Encode(reconciliationOutput{
Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize,
DurationMillis: time.Since(started).Milliseconds(), Report: report,
})
}