forked from mtusi2026/mtusi2026lab4
refactor: isolate attachment lifecycle
This commit is contained in:
@@ -240,6 +240,7 @@ type consistencyResult struct {
|
||||
OrphanDeleted bool `json:"orphan_deleted"`
|
||||
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
||||
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
||||
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
||||
TotalDurationMillis int64 `json:"total_duration_ms"`
|
||||
}
|
||||
|
||||
@@ -259,7 +260,7 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
settings, err := config.Load()
|
||||
settings, err := config.LoadReconciliation()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -281,7 +282,9 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
return err
|
||||
}
|
||||
metadata := postgres.New(pool)
|
||||
clean, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0})
|
||||
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -320,7 +323,11 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID})
|
||||
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -336,7 +343,8 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
}
|
||||
|
||||
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, Apply: true,
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
||||
UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -346,9 +354,19 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
return err
|
||||
}
|
||||
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
||||
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID)
|
||||
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
||||
attachmentExists(ctx, pool, abandoned.ID)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours',
|
||||
ready_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||
}
|
||||
|
||||
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{Prefix: "attachments/", PageSize: *pageSize})
|
||||
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -357,7 +375,7 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
return err
|
||||
}
|
||||
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, Apply: true,
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -376,9 +394,10 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||
}
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved ||
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
||||
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
||||
}
|
||||
@@ -394,7 +413,8 @@ func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconcile
|
||||
func hasAnomalies(report reconcile.Report) bool {
|
||||
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||
report.OldUnreferencedMetadata != 0
|
||||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||
report.OldUnreferencedAttachments != 0
|
||||
}
|
||||
|
||||
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||
|
||||
+8
-192
@@ -1,215 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/messenger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/session"
|
||||
)
|
||||
|
||||
func main() {
|
||||
appLogger := logger.New()
|
||||
if err := run(appLogger, os.Args[1:]); err != nil {
|
||||
appLogger.Error("server stopped", "error", err)
|
||||
appLogger.Error("application stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 0 {
|
||||
return runServer(logger)
|
||||
return bootstrap.RunServer(logger)
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return runMigrations()
|
||||
return bootstrap.RunMigrations()
|
||||
}
|
||||
if len(arguments) > 0 && arguments[0] == "reconcile-attachments" {
|
||||
return runAttachmentReconciliation(arguments[1:], os.Stdout)
|
||||
if arguments[0] == "reconcile-attachments" {
|
||||
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
|
||||
}
|
||||
return fmt.Errorf("usage: messenger [migrate|reconcile-attachments]")
|
||||
}
|
||||
|
||||
func runServer(logger *slog.Logger) error {
|
||||
settings, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
sessions, err := session.New(settings.CacheURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sessions.Close() }()
|
||||
passwords, err := password.New(password.Config{
|
||||
MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations,
|
||||
Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength,
|
||||
KeyLength: settings.PasswordKeyLength,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app := messenger.NewServer(
|
||||
logger,
|
||||
postgres.New(pool),
|
||||
sessions,
|
||||
passwords,
|
||||
objects,
|
||||
settings.SessionTTL,
|
||||
settings.IdempotencyTTL,
|
||||
settings.RequestTimeout,
|
||||
instanceID,
|
||||
settings.SwaggerPort,
|
||||
settings.AttachmentMaxBytes,
|
||||
settings.AttachmentMaxPixels,
|
||||
settings.AttachmentDecodeMemory,
|
||||
)
|
||||
server := &http.Server{
|
||||
Addr: settings.Address,
|
||||
Handler: app.Handler(),
|
||||
ReadHeaderTimeout: settings.HTTPReadHeaderTimeout,
|
||||
ReadTimeout: settings.HTTPReadTimeout,
|
||||
WriteTimeout: settings.HTTPWriteTimeout,
|
||||
IdleTimeout: settings.HTTPIdleTimeout,
|
||||
}
|
||||
listener, err := net.Listen("tcp", server.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serveResult := make(chan error, 1)
|
||||
go func() {
|
||||
serveResult <- server.Serve(listener)
|
||||
}()
|
||||
logger.Info("server started", "address", server.Addr)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
select {
|
||||
case err := <-serveResult:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
logger.Info("shutdown requested")
|
||||
app.SetReady(false)
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
_ = server.Close()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runMigrations() error {
|
||||
settings, err := config.LoadMigration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
return database.Migrate(context.Background(), pool, settings.MigrationTimeout)
|
||||
}
|
||||
|
||||
type reconciliationOutput struct {
|
||||
Mode string `json:"mode"`
|
||||
GracePeriod string `json:"grace_period"`
|
||||
PageSize int `json:"page_size"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func runAttachmentReconciliation(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
apply := flags.Bool("apply", false, "apply safe repairs")
|
||||
gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age")
|
||||
pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return fmt.Errorf("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--page-size=500]")
|
||||
}
|
||||
|
||||
settings, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, Apply: *apply,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mode := "dry-run"
|
||||
if *apply {
|
||||
mode = "apply"
|
||||
}
|
||||
return json.NewEncoder(output).Encode(reconciliationOutput{
|
||||
Mode: mode, GracePeriod: gracePeriod.String(), PageSize: *pageSize,
|
||||
DurationMillis: time.Since(started).Milliseconds(), Report: report,
|
||||
})
|
||||
return errors.New("usage: messenger [migrate|reconcile-attachments]")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user