diff --git a/REPORT.md b/REPORT.md
index 77fe5799..90eb7880 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -78,7 +78,7 @@ Grace period — 1 h, минимум 2 min, выше бюджета upload. Св
| Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. |
| Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. |
-Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready.
+Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает отсутствующий blob и файл в failed. Для ready с существующим объектом того же размера действует быстрый Stat-путь. Deep-повреждение такого ready диагностируется и требует отдельного восстановления из достоверных байтов. Простое появление объекта само не меняет failed на ready.
```sh
make reconcile
@@ -136,7 +136,7 @@ Bootstrap — одноразовый `s3-init` с отдельным admin-кл
| --- | ---: | ---: | --- |
| Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart |
| Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart |
-| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart |
+| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.2 | 201, checksum совпал, без OOM/restart |
Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки.
diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go
index b7090a83..97b79811 100644
--- a/cmd/reconcile/main.go
+++ b/cmd/reconcile/main.go
@@ -43,7 +43,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m")
return 2
}
- dbConfig, err := config.LoadMigrate(os.LookupEnv)
+ databaseURL, err := config.LoadDatabaseURL(os.LookupEnv)
if err != nil {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
@@ -54,12 +54,12 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
}
- blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey)
+ blobs, err := s3store.New(s3Config)
if err != nil {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
}
- db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10)
+ db, err := postgres.NewPool(ctx, databaseURL, 10)
if err != nil {
fmt.Fprintln(errOut, "reconcile:", err)
return 1
@@ -76,7 +76,8 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
defer cancel()
encoder := json.NewEncoder(out)
var outputErr error
- stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) {
+ options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep}
+ stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) {
if outputErr == nil {
outputErr = encoder.Encode(f)
if outputErr != nil {
diff --git a/cmd/reconcile/main_test.go b/cmd/reconcile/main_test.go
new file mode 100644
index 00000000..806f2d67
--- /dev/null
+++ b/cmd/reconcile/main_test.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) {
+ for key, value := range map[string]string{
+ "DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable",
+ "S3_ACCESS_KEY_ID_FILE": "test-access",
+ "S3_SECRET_ACCESS_KEY_FILE": "test-secret",
+ } {
+ path := filepath.Join(t.TempDir(), key)
+ if err := os.WriteFile(path, []byte(value), 0600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv(key, path)
+ }
+ t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333")
+ t.Setenv("S3_REGION", "us-east-1")
+ t.Setenv("S3_BUCKET", "attachments")
+ t.Setenv("APP_LOG_LEVEL", "not-a-log-level")
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // Stop before dialing: only configuration relevant to this CLI is checked.
+ var out, errOut bytes.Buffer
+ code := run(ctx, nil, &out, &errOut)
+ if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") {
+ t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String())
+ }
+}
diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go
index db308002..164c3b01 100644
--- a/cmd/s3init/main.go
+++ b/cmd/s3init/main.go
@@ -3,10 +3,11 @@ package main
import (
"context"
"fmt"
- "messenger/internal/platform/config"
- "messenger/internal/platform/s3store"
"os"
"time"
+
+ "messenger/internal/platform/config"
+ "messenger/internal/platform/s3store"
)
func main() {
@@ -20,7 +21,7 @@ func run() error {
if err != nil {
return err
}
- s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey)
+ s, err := s3store.New(cfg)
if err != nil {
return err
}
diff --git a/cmd/server/main.go b/cmd/server/main.go
index cb4aec4a..4a3c85b8 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -1,4 +1,4 @@
-// Command server starts the messenger API over PostgreSQL and Valkey.
+// Command server starts the messenger API over PostgreSQL, Valkey and S3.
package main
import (
@@ -61,25 +61,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return err
}
defer sessions.Close()
- var blobs attachment.Blobs
- var reconciler *attachment.Reconciler
- if cfg.S3.Endpoint != "" {
- blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey)
- if err != nil {
- return err
- }
- reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv))
- if e != nil {
- return e
- }
- reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey)
- if e != nil {
- return e
- }
- reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour)
- if err != nil {
- return err
- }
+ blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv)
+ if err != nil {
+ return err
}
ln, err := net.Listen("tcp", cfg.Addr())
@@ -100,8 +84,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
InstanceID: alias,
IdempotencyTTL: cfg.IdempotencyTTL,
Blobs: blobs,
- MaxUploadBytes: cfg.S3.MaxBytes,
- MaxImagePixels: cfg.S3.MaxPixels,
+ AttachmentsCfg: attachment.Config{
+ MaxBytes: cfg.S3.MaxBytes,
+ MaxPixels: cfg.S3.MaxPixels,
+ },
})
var debugLn net.Listener
if cfg.DebugAddr != "" {
@@ -124,7 +110,7 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return
case <-ticker.C:
scan, cancel := context.WithTimeout(ctx, 2*time.Minute)
- stats, err := reconciler.Run(scan, true, false, nil)
+ stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil)
cancel()
if err != nil {
log.Warn("attachment cleanup incomplete", "error", err)
@@ -149,6 +135,29 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return err
}
+func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) {
+ if cfg.Endpoint == "" {
+ return nil, nil, nil
+ }
+ blobs, err := s3store.New(cfg)
+ if err != nil {
+ return nil, nil, err
+ }
+ reconCfg, err := config.LoadS3(config.ReconcileS3(lookup))
+ if err != nil {
+ return nil, nil, err
+ }
+ reconBlobs, err := s3store.New(reconCfg)
+ if err != nil {
+ return nil, nil, err
+ }
+ reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour)
+ if err != nil {
+ return nil, nil, err
+ }
+ return blobs, reconciler, nil
+}
+
// publishDebugVars adds to expvar what the load tests read from a replica.
func publishDebugVars(alias string, pool *pgxpool.Pool) {
expvar.NewString("instance").Set(alias)
diff --git a/compose.yaml b/compose.yaml
index bc08ab33..6290c342 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -6,6 +6,9 @@ name: lab4
services:
lb:
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
+ depends_on:
+ api:
+ condition: service_healthy
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
diff --git a/internal/app/app.go b/internal/app/app.go
index 3949add5..9ab9a811 100644
--- a/internal/app/app.go
+++ b/internal/app/app.go
@@ -39,20 +39,23 @@ type Deps struct {
Sessions *redis.Client
SessionTTL time.Duration
MaxBodyBytes int64
- // RequestTimeout bounds every API request; zero means the default.
- RequestTimeout time.Duration
- InstanceID string
+ // Zero timeouts use the defaults below.
+ RequestTimeout time.Duration
+ BinaryRequestTimeout time.Duration
+ InstanceID string
// IdempotencyTTL zero means the contract's 24 hours.
IdempotencyTTL time.Duration
Blobs attachment.Blobs
- MaxUploadBytes int64
- MaxImagePixels int64
+ AttachmentsCfg attachment.Config
}
-// defaultRequestTimeout stays below the server's 10s write timeout, so a
+// defaultRequestTimeout stays below the server's write timeout, so a
// request that gives up on a dependency can still write its 503.
const defaultRequestTimeout = 5 * time.Second
+// Binary transfers and decoding share this budget, below the 35s socket timeout.
+const defaultBinaryRequestTimeout = 30 * time.Second
+
const defaultIdempotencyTTL = 24 * time.Hour
func New(d Deps) *App {
@@ -61,7 +64,8 @@ func New(d Deps) *App {
if keyTTL == 0 {
keyTTL = defaultIdempotencyTTL
}
- chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL)
+ chatRepository := chatstore.NewChats(d.DB)
+ chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
probes := health.New(
@@ -81,9 +85,15 @@ func New(d Deps) *App {
auth.NewHandler(sessions).Register(public, protected)
user.NewHandler(users).Register(public, protected)
chat.NewHandler(chats).Register(protected)
- attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels)
- binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions))
- attachment.NewHandler(attachments).Register(protected, binary)
+ attachments := attachment.NewService(
+ attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg,
+ )
+ binaryTimeout := d.BinaryRequestTimeout
+ if binaryTimeout == 0 {
+ binaryTimeout = defaultBinaryRequestTimeout
+ }
+ binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions))
+ attachment.NewHandler(attachments).Register(protected, binaryProtected)
return &App{Engine: engine, Health: probes, Chats: chats}
}
diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go
index 2041d5e6..0b6411b7 100644
--- a/internal/app/attachment_test.go
+++ b/internal/app/attachment_test.go
@@ -30,32 +30,54 @@ func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) {
ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized")
}
-func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) {
- ts := newTester(t)
- actor := ts.user("deadline_user", "User")
- ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond})
- var held []*pgxpool.Conn
- for range db.Pool.Config().MaxConns {
- conn, err := db.Pool.Acquire(context.Background())
- if err != nil {
- t.Fatal(err)
+func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) {
+ for _, binary := range []bool{false, true} {
+ name := "metadata"
+ requestTimeout := 40 * time.Millisecond
+ var binaryTimeout time.Duration
+ if binary {
+ name = "upload"
+ requestTimeout = time.Second
+ binaryTimeout = 40 * time.Millisecond
}
- held = append(held, conn)
- }
- defer func() {
- for _, conn := range held {
- conn.Release()
- }
- }()
- ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
- defer cancel()
- req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx)
- req.Header.Set("Authorization", "Bearer "+actor.token)
- rec := httptest.NewRecorder()
- started := time.Now()
- ts.a.Engine.ServeHTTP(rec, req)
- if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
- t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
+ t.Run(name, func(t *testing.T) {
+ ts := newTester(t)
+ actor := ts.user("deadline_user", "User")
+ ts.a = app.New(app.Deps{
+ Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client,
+ SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance,
+ RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout,
+ })
+ var held []*pgxpool.Conn
+ for range db.Pool.Config().MaxConns {
+ conn, err := db.Pool.Acquire(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ held = append(held, conn)
+ }
+ defer func() {
+ for _, conn := range held {
+ conn.Release()
+ }
+ }()
+ ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
+ defer cancel()
+ path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111"
+ req := httptest.NewRequest("GET", path, nil).WithContext(ctx)
+ if binary {
+ path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png"
+ req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx)
+ req.Header.Set("Content-Type", "image/png")
+ }
+ req.Header.Set("Authorization", "Bearer "+actor.token)
+ rec := httptest.NewRecorder()
+ started := time.Now()
+ ts.a.Engine.ServeHTTP(rec, req)
+ if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
+ t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
+ }
+ })
}
}
diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go
index 9a5cd171..69177192 100644
--- a/internal/attachment/attachment.go
+++ b/internal/attachment/attachment.go
@@ -1,9 +1,17 @@
package attachment
import (
- "context"
- "github.com/google/uuid"
"time"
+
+ "github.com/google/uuid"
+)
+
+type FileStatus string
+
+const (
+ FilePending FileStatus = "pending"
+ FileReady FileStatus = "ready"
+ FileFailed FileStatus = "failed"
)
type Attachment struct {
@@ -11,12 +19,14 @@ type Attachment struct {
Filename string
FileHash []byte
SizeBytes int64
- MIMEType, Status string
+ MIMEType string
+ Status FileStatus
CreatedAt time.Time
}
-type Repository interface {
- IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error)
- Publish(context.Context, Attachment, string) error
- GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error)
+type UploadInput struct {
+ ChatID uuid.UUID
+ Filename string
+ MIMEType string
+ SizeBytes int64
}
diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go
index c211b36e..3958653b 100644
--- a/internal/attachment/blob.go
+++ b/internal/attachment/blob.go
@@ -16,10 +16,10 @@ type Object struct {
}
type Blobs interface {
- Put(context.Context, string, io.Reader, int64, string) error
- Open(context.Context, string) (io.ReadCloser, error)
- Stat(context.Context, string) (Object, error)
- Copy(context.Context, string, string) error
- Delete(context.Context, string) error
- List(context.Context, string, string, int) ([]Object, error)
+ Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error
+ Open(ctx context.Context, key string) (io.ReadCloser, error)
+ Stat(ctx context.Context, key string) (Object, error)
+ Copy(ctx context.Context, sourceKey, destinationKey string) error
+ Delete(ctx context.Context, key string) error
+ List(ctx context.Context, prefix, after string, limit int) ([]Object, error)
}
diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go
index 6420519f..5adbd0e7 100644
--- a/internal/attachment/handler.go
+++ b/internal/attachment/handler.go
@@ -8,40 +8,52 @@ import (
"net/http"
"strconv"
- "github.com/gin-gonic/gin"
- "github.com/google/uuid"
"messenger/internal/platform/apperr"
"messenger/internal/platform/httpx"
"messenger/internal/platform/ident"
+
+ "github.com/gin-gonic/gin"
+ "github.com/google/uuid"
)
type Handler struct{ s *Service }
func NewHandler(s *Service) *Handler { return &Handler{s} }
-func (h *Handler) Register(protected, binary gin.IRouter) {
- binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
+func (h *Handler) Register(protected, binaryProtected gin.IRouter) {
+ binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata))
- binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
+ binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
}
type Response struct {
- ID string `json:"id"`
- ChatID string `json:"chat_id"`
- OwnerID string `json:"owner_id"`
- Filename string `json:"filename"`
- ContentType string `json:"content_type"`
- SizeBytes int64 `json:"size_bytes"`
- SHA256 string `json:"sha256"`
- Status string `json:"status"`
- Variants []any `json:"variants"`
- ErrorCode *string `json:"error_code"`
- CreatedAt string `json:"created_at"`
- UpdatedAt string `json:"updated_at"`
+ ID string `json:"id"`
+ ChatID string `json:"chat_id"`
+ OwnerID string `json:"owner_id"`
+ Filename string `json:"filename"`
+ ContentType string `json:"content_type"`
+ SizeBytes int64 `json:"size_bytes"`
+ SHA256 string `json:"sha256"`
+ Status string `json:"status"`
+ // Lab4's shared response contract requires variants=[], even without processing.
+ Variants []any `json:"variants"`
+ ErrorCode *string `json:"error_code"`
+ CreatedAt string `json:"created_at"`
+ UpdatedAt string `json:"updated_at"`
}
func toResponse(a Attachment) Response {
return Response{
- ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt),
+ ID: a.ID.String(),
+ ChatID: a.ChatID.String(),
+ OwnerID: a.OwnerID.String(),
+ Filename: a.Filename,
+ ContentType: a.MIMEType,
+ SizeBytes: a.SizeBytes,
+ SHA256: hex.EncodeToString(a.FileHash),
+ Status: string(a.Status),
+ Variants: []any{},
+ CreatedAt: httpx.Timestamp(a.CreatedAt),
+ UpdatedAt: httpx.Timestamp(a.CreatedAt),
}
}
@@ -65,8 +77,14 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
return
}
- c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes)
- a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body)
+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes)
+ input := UploadInput{
+ ChatID: id,
+ Filename: names[0],
+ MIMEType: media,
+ SizeBytes: c.Request.ContentLength,
+ }
+ a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body)
if err != nil {
httpx.Error(c, err)
return
@@ -109,7 +127,11 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
- defer r.Close()
+ defer func() {
+ if err := r.Close(); err != nil {
+ slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err)
+ }
+ }()
c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename}))
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Type", a.MIMEType)
diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go
index 729e42f6..b8a11c05 100644
--- a/internal/attachment/reconcile.go
+++ b/internal/attachment/reconcile.go
@@ -45,6 +45,11 @@ type Reconciler struct {
grace time.Duration
}
+type ReconcileOptions struct {
+ Apply bool
+ Deep bool
+}
+
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
if db == nil || blobs == nil {
return nil, errors.New("reconcile: database and blobs are required")
@@ -61,10 +66,18 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci
// Run streams findings and keeps at most one page of metadata in memory.
// Dry runs only read storage. Each apply candidate is checked again under the
// publisher's transaction lock; no scan snapshot authorizes a later mutation.
-func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) {
+func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) {
started := time.Now()
defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }()
- run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)}
+ run := reconcileRun{
+ r: r,
+ ctx: ctx,
+ apply: options.Apply,
+ deep: options.Deep,
+ emit: emit,
+ stats: &stats,
+ cutoff: started.Add(-r.grace),
+ }
for _, prefix := range []string{"staging/", "blobs/"} {
if err = run.objects(prefix); err != nil {
return stats, err
@@ -203,7 +216,8 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
return known, nil
}
s.stats.SQLQueries++
- rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes)
+ rows, err := s.r.db.Query(s.ctx, `
+ SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes)
if err != nil {
return nil, err
}
@@ -235,7 +249,8 @@ func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
op.ctx = opCtx
var exists bool
s.stats.SQLQueries++
- if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil {
+ if err := tx.QueryRow(opCtx, `
+ SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil {
return nil, err
}
if exists {
@@ -304,7 +319,7 @@ func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Fi
type reconcileFile struct {
hash []byte
size int64
- status string
+ status FileStatus
created time.Time
referenced bool
}
@@ -316,9 +331,13 @@ func (s *reconcileRun) files() error {
return err
}
s.stats.SQLQueries++
- rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
- EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
- FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after)
+ rows, err := s.r.db.Query(s.ctx, `
+ SELECT f.hash, f.size_bytes, f.status, f.created_at,
+ EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
+ FROM files f
+ WHERE f.hash > $1
+ ORDER BY f.hash
+ LIMIT $2`, after, reconcilePageSize)
if err != nil {
return err
}
@@ -346,30 +365,9 @@ func (s *reconcileRun) files() error {
if !s.candidate(f) {
continue
}
- var finding *Finding
- if s.apply {
- finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
- op := *s
- op.ctx = opCtx
- var current reconcileFile
- s.stats.SQLQueries++
- err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
- EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
- FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash).
- Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced)
- if errors.Is(err, pgx.ErrNoRows) {
- return nil, nil
- }
- if err != nil {
- return nil, err
- }
- if !s.candidate(current) {
- return nil, nil
- }
- return op.inspect(current, tx)
- })
- } else {
- finding, err = s.inspect(f, nil)
+ finding, err := s.inspect(f)
+ if err == nil && s.apply && finding != nil && finding.Action != "report" {
+ finding, err = s.repairFile(f.hash)
}
if err != nil {
return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err)
@@ -379,17 +377,66 @@ func (s *reconcileRun) files() error {
}
}
+func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) {
+ return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
+ op := *s
+ op.ctx = opCtx
+ var current reconcileFile
+ s.stats.SQLQueries++
+ err := tx.QueryRow(opCtx, `
+ SELECT f.hash, f.size_bytes, f.status, f.created_at,
+ EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
+ FROM files f
+ WHERE f.hash = $1
+ FOR UPDATE OF f`, hash).
+ Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if !op.candidate(current) {
+ return nil, nil
+ }
+ finding, err := op.inspect(current)
+ if err != nil || finding == nil {
+ return finding, err
+ }
+ switch finding.Action {
+ case "delete_file":
+ // Delete bytes first. A failed SQL commit leaves an unreferenced
+ // missing row that the next pass can safely finish collecting.
+ if err := op.delete(finding.Key); err != nil {
+ return nil, err
+ }
+ s.stats.SQLQueries++
+ _, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash)
+ case "mark_failed":
+ s.stats.SQLQueries++
+ _, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed)
+ default:
+ return finding, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ finding.Applied = true
+ return finding, nil
+ })
+}
+
func (s *reconcileRun) candidate(f reconcileFile) bool {
if !s.old(f.created) {
return false
}
if !f.referenced {
- return f.status == "pending" || f.status == "ready"
+ return f.status == FilePending || f.status == FileReady
}
- return f.status == "ready" || f.status == "failed"
+ return f.status == FileReady || f.status == FileFailed
}
-func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
+func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) {
key := BlobKey(f.hash)
if !f.referenced {
// A publication that rolled back can leave fresh bytes behind an old
@@ -401,36 +448,14 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
if err == nil && !s.old(obj.Modified) {
return nil, nil
}
- finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"}
- if !s.apply {
- return finding, nil
- }
- // S3 must succeed first. A failed SQL commit then leaves a missing,
- // unreferenced row that the next run can safely finish collecting.
- if err := s.delete(key); err != nil {
- return nil, err
- }
- s.stats.SQLQueries++
- if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil {
- return nil, err
- }
- finding.Applied = true
- return finding, nil
+ return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil
}
obj, err := s.stat(key)
if errors.Is(err, ErrObjectMissing) {
- if f.status == "failed" {
+ if f.status == FileFailed {
return nil, nil
}
- finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}
- if s.apply {
- s.stats.SQLQueries++
- if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil {
- return nil, err
- }
- finding.Applied = true
- }
- return finding, nil
+ return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil
}
if err != nil {
return nil, err
@@ -449,7 +474,7 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
}
// A failed file stays failed even when the key reappears. Recovery requires
// an explicit publisher; a shallow HEAD cannot prove content integrity.
- if f.status == "failed" {
+ if f.status == FileFailed {
return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil
}
return nil, nil
diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go
index 39fd6dcd..d7e26884 100644
--- a/internal/attachment/reconcile_test.go
+++ b/internal/attachment/reconcile_test.go
@@ -209,7 +209,8 @@ func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment
f.t.Fatal(err)
}
var found []attachment.Finding
- stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) })
+ options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
+ stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) })
return stats, found, err
}
@@ -221,6 +222,28 @@ func TestReconciler(t *testing.T) {
t.Fatal(err)
}
defer db.Close()
+ t.Run("healthy apply does not block publication", func(t *testing.T) {
+ f := reconcileSetup(t, db)
+ hash := f.file("healthy", "ready", 2*time.Hour, true, true)
+ tx, err := db.Pool.Begin(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tx.Rollback(ctx)
+ if err := attachment.LockHash(ctx, tx, hash); err != nil {
+ t.Fatal(err)
+ }
+ r, err := attachment.NewReconciler(f.db, f.b, time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+ scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
+ if err != nil || stats.Findings != 0 || stats.Repaired != 0 {
+ t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err)
+ }
+ })
t.Run("constructor validates and defaults grace", func(t *testing.T) {
f := reconcileSetup(t, db)
for _, grace := range []time.Duration{-time.Second, time.Minute} {
@@ -361,7 +384,7 @@ func TestReconciler(t *testing.T) {
done := make(chan error, 1)
go func() {
r, _ := attachment.NewReconciler(f.db, f.b, 0)
- _, err := r.Run(ctx, true, false, nil)
+ _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
done <- err
}()
waitReconcileLock(t, ctx, f.db)
@@ -395,7 +418,7 @@ func TestReconciler(t *testing.T) {
done := make(chan error, 1)
go func() {
r, _ := attachment.NewReconciler(f.db, f.b, 0)
- _, err := r.Run(ctx, true, false, nil)
+ _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
done <- err
}()
waitReconcileLock(t, ctx, f.db)
@@ -592,7 +615,7 @@ func TestReconcilerFailureBoundaries(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- stats, err := r.Run(ctx, false, true, nil)
+ stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil)
if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" {
t.Fatalf("read only dry run: %+v %v", stats, err)
}
@@ -681,11 +704,11 @@ func TestReconcilerS3(t *testing.T) {
t.Fatalf("CLI mutated by default: %+v %v", cliStats, err)
}
var found []attachment.Finding
- stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) })
+ stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) })
if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" {
t.Fatalf("S3 dry: %+v %v %+v", stats, err, found)
}
- stats, err = r.Run(ctx, true, true, nil)
+ stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" {
t.Fatalf("S3 apply: %+v %v", stats, err)
}
@@ -695,7 +718,7 @@ func TestReconcilerS3(t *testing.T) {
if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil {
t.Fatal("fresh staging deleted", err)
}
- stats, err = r.Run(ctx, true, true, nil)
+ stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
if err != nil || stats.Repaired != 0 {
t.Fatalf("S3 repeat: %+v %v", stats, err)
}
diff --git a/internal/attachment/repository.go b/internal/attachment/repository.go
new file mode 100644
index 00000000..9c554118
--- /dev/null
+++ b/internal/attachment/repository.go
@@ -0,0 +1,23 @@
+package attachment
+
+import (
+ "context"
+
+ "github.com/google/uuid"
+)
+
+type AttachmentRepository interface {
+ // WithPublication holds the file's hash lock through the callback and commit.
+ // Publication is valid only inside the callback. Any callback error rolls back.
+ WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error
+ GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error)
+}
+
+type Publication interface {
+ FileReady() bool
+ SaveReady(ctx context.Context) error
+}
+
+type ChatMembership interface {
+ IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error)
+}
diff --git a/internal/attachment/service.go b/internal/attachment/service.go
index ee574176..8363e1e9 100644
--- a/internal/attachment/service.go
+++ b/internal/attachment/service.go
@@ -15,40 +15,54 @@ import (
)
type Service struct {
- repo Repository
- blobs Blobs
- maxBytes, maxPixels int64
- decode chan struct{}
+ attachments AttachmentRepository
+ chats ChatMembership
+ blobs Blobs
+ config Config
+ decode chan struct{}
}
-func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service {
- if maxBytes == 0 {
- maxBytes = MaxUploadBytes
- }
- if maxPixels == 0 {
- maxPixels = MaxImagePixels
- }
- return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)}
+type Config struct {
+ MaxBytes int64
+ MaxPixels int64
}
-func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) {
- if err := ValidateFilename(filename); err != nil {
+// A nil blobs dependency allows the text API to run without configured S3.
+// Binary operations report unavailability in that mode.
+func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service {
+ if cfg.MaxBytes == 0 {
+ cfg.MaxBytes = MaxUploadBytes
+ }
+ if cfg.MaxPixels == 0 {
+ cfg.MaxPixels = MaxImagePixels
+ }
+ return &Service{
+ attachments: attachments,
+ chats: chats,
+ blobs: blobs,
+ config: cfg,
+ decode: make(chan struct{}, 1),
+ }
+}
+
+func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) {
+ if err := ValidateFilename(input.Filename); err != nil {
return Attachment{}, err
}
- if mime != "image/png" && mime != "image/jpeg" {
+ if _, accepted := imageFormats[input.MIMEType]; !accepted {
return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted")
}
- ok, err := s.repo.IsMember(ctx, chatID, actor)
+ ok, err := s.chats.IsMember(ctx, input.ChatID, actor)
if err != nil {
return Attachment{}, err
}
if !ok {
return Attachment{}, apperr.NotFound("chat not found")
}
- if size > s.maxBytes {
+ if input.SizeBytes > s.config.MaxBytes {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
- if size <= 0 {
+ if input.SizeBytes <= 0 {
return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required")
}
if s.blobs == nil {
@@ -59,16 +73,17 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
+ // A failed cleanup leaves a staging object for the reconciler's grace-based collection.
if err := s.blobs.Delete(cleanup, stage); err != nil {
- slog.Warn("staging cleanup deferred", "error", err)
+ slog.Warn("staging cleanup deferred", "key", stage, "error", err)
}
}()
hash := sha256.New()
- counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)}
- err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime)
+ counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)}
+ err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType)
if err != nil {
_, tooLarge := errors.AsType[*http.MaxBytesError](err)
- if counter.n > s.maxBytes || tooLarge {
+ if counter.n > s.config.MaxBytes || tooLarge {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
@@ -78,10 +93,10 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
}
var extra [1]byte
n, readErr := counter.Read(extra[:])
- if counter.n > s.maxBytes {
+ if counter.n > s.config.MaxBytes {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
- if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) {
+ if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) {
return Attachment{}, apperr.Invalid("body length does not match Content-Length")
}
// Full decoding is bounded by dimensions and one decoder per replica. Body
@@ -91,26 +106,64 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
case <-ctx.Done():
return Attachment{}, unavailable(ctx.Err())
}
- err = s.validate(ctx, stage, mime)
+ err = s.validate(ctx, stage, input.MIMEType)
<-s.decode
if err != nil {
return Attachment{}, err
}
- a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)}
- if err := s.repo.Publish(ctx, a, stage); err != nil {
+ a := Attachment{
+ ID: id,
+ ChatID: input.ChatID,
+ OwnerID: actor,
+ Filename: input.Filename,
+ FileHash: hash.Sum(nil),
+ SizeBytes: input.SizeBytes,
+ MIMEType: input.MIMEType,
+ Status: FileReady,
+ CreatedAt: ident.TimeOf(id),
+ }
+ if err := s.attachments.WithPublication(ctx, a, func(p Publication) error {
+ if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil {
+ return err
+ }
+ return p.SaveReady(ctx)
+ }); err != nil {
return Attachment{}, err
}
return a, nil
}
-func (s *Service) validate(ctx context.Context, stage, mime string) error {
+func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error {
+ blobKey := BlobKey(a.FileHash)
+ if fileReady {
+ obj, err := s.blobs.Stat(ctx, blobKey)
+ if err == nil && obj.Size == a.SizeBytes {
+ return nil
+ }
+ if err != nil && !errors.Is(err, ErrObjectMissing) {
+ return unavailable(err)
+ }
+ }
+ // The hash is known after streaming. Copy the validated staging object to
+ // its permanent CAS key while the repository's publication lock is held.
+ if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil {
+ return unavailable(err)
+ }
+ return nil
+}
+
+func (s *Service) validate(ctx context.Context, stage, mime string) (err error) {
r, err := s.blobs.Open(ctx, stage)
if err != nil {
return unavailable(err)
}
- defer r.Close()
+ defer func() {
+ if closeErr := r.Close(); closeErr != nil && err == nil {
+ err = unavailable(closeErr)
+ }
+ }()
tracked := &storageReader{reader: r}
- err = ValidateImage(tracked, mime, s.maxPixels)
+ err = ValidateImage(tracked, mime, s.config.MaxPixels)
if tracked.err != nil {
return unavailable(tracked.err)
}
@@ -136,11 +189,11 @@ func (r *storageReader) Read(p []byte) (int, error) {
}
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
- a, err := s.repo.GetForMember(ctx, id, actor)
+ a, err := s.attachments.GetForMember(ctx, id, actor)
if err != nil {
return Attachment{}, err
}
- if a.Status != "ready" {
+ if a.Status != FileReady {
return Attachment{}, apperr.Conflict("attachment is not ready")
}
return a, nil
@@ -166,7 +219,12 @@ type countReader struct {
n int64
}
-func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e }
+func (c *countReader) Read(b []byte) (int, error) {
+ n, err := c.r.Read(b)
+ c.n += int64(n)
+ return n, err
+}
+
func unavailable(err error) error {
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
}
diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go
index c207a956..f01a3540 100644
--- a/internal/attachment/service_test.go
+++ b/internal/attachment/service_test.go
@@ -14,6 +14,7 @@ import (
"github.com/google/uuid"
"messenger/internal/attachment"
"messenger/internal/attachment/store"
+ chatstore "messenger/internal/chat/store"
"messenger/internal/platform/apperr"
"messenger/internal/platform/ident"
"messenger/internal/testenv"
@@ -73,10 +74,11 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
t.Fatal(err)
}
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
- s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0)
+ s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
uploaded := make(chan error, 1)
go func() {
- _, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data))
+ input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
+ _, e := s.Upload(ctx, actor, input, bytes.NewReader(data))
uploaded <- e
}()
select {
@@ -90,7 +92,7 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
}
collected := make(chan error, 1)
go func() {
- stats, e := r.Run(ctx, true, false, nil)
+ stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
if e == nil && stats.Repaired != 0 {
e = errors.New("collector removed a newly referenced file")
}
@@ -196,7 +198,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
}
}
b := &testBlobs{data: map[string][]byte{}}
- s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels)
+ s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Fatal(err)
@@ -210,7 +212,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
c = c2
}
var e error
- results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data))
+ input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
+ results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data))
if e != nil {
t.Error(e)
}
@@ -235,7 +238,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
b.mu.Lock()
clear(b.data)
b.mu.Unlock()
- if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil {
+ input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
+ if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil {
t.Fatal(err)
}
if b.copies != 2 {
@@ -244,7 +248,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
b.failCopy = true
other := bytes.Clone(data)
other = append(other, 0)
- if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
+ input.Filename, input.SizeBytes = "failed.png", int64(len(other))
+ if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("copy outage: %v", err)
}
if len(b.data) != 1 {
@@ -258,7 +263,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil {
+ input.Filename = "rollback.png"
+ if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil {
t.Fatal("publication survived rejected metadata")
}
if len(b.data) != 2 {
diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go
index 607932a9..32105070 100644
--- a/internal/attachment/store/postgres.go
+++ b/internal/attachment/store/postgres.go
@@ -4,32 +4,29 @@ import (
"context"
"errors"
"fmt"
+
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
+
"messenger/internal/attachment"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
)
type Attachments struct {
- pool *pgxpool.Pool
- blobs attachment.Blobs
+ pool *pgxpool.Pool
}
-func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments {
- return &Attachments{pool, blobs}
-}
+var _ attachment.AttachmentRepository = (*Attachments)(nil)
-func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) {
- var ok bool
- err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok)
- return ok, postgres.Classify(err)
+func NewAttachments(pool *pgxpool.Pool) *Attachments {
+ return &Attachments{pool: pool}
}
// Publication and garbage collection use the same hash lock. The transaction
// spans only publication, never client transfer or image decoding.
-func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error {
+func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error {
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
return err
@@ -37,41 +34,56 @@ func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stag
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
return err
}
- var status string
- err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status)
+ var status attachment.FileStatus
+ err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
- copyNeeded := status != "ready"
- if !copyNeeded {
- obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash))
- if errors.Is(e, attachment.ErrObjectMissing) {
- copyNeeded = true
- } else if e != nil {
- return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e)
- } else if obj.Size != a.SizeBytes {
- copyNeeded = true
- }
- }
- if copyNeeded {
- if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil {
- return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
- }
- }
- _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType)
- if err != nil {
- return fmt.Errorf("publish file: %w", err)
- }
- _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
- return err
+ return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady})
}))
}
+type publication struct {
+ tx pgx.Tx
+ attachment attachment.Attachment
+ ready bool
+}
+
+func (p publication) FileReady() bool { return p.ready }
+
+func (p publication) SaveReady(ctx context.Context) error {
+ a := p.attachment
+ _, err := p.tx.Exec(ctx, `
+ INSERT INTO files (hash, size_bytes, mime_type, status)
+ VALUES ($1, $2, $3, $4)
+ ON CONFLICT (hash) DO UPDATE
+ SET size_bytes = excluded.size_bytes,
+ mime_type = excluded.mime_type,
+ status = excluded.status`,
+ a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady)
+ if err != nil {
+ return fmt.Errorf("publish file: %w", err)
+ }
+ _, err = p.tx.Exec(ctx, `
+ INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6)`,
+ a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
+ return err
+}
+
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
var a attachment.Attachment
- err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at
- FROM attachments a JOIN files f ON f.hash=a.file_hash
- WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
+ err := r.pool.QueryRow(ctx, `
+ SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash,
+ f.size_bytes, f.mime_type, f.status, a.created_at
+ FROM attachments a
+ JOIN files f ON f.hash = a.file_hash
+ WHERE a.id = $1 AND EXISTS (
+ SELECT 1 FROM chat_members m
+ WHERE m.chat_id = a.chat_id AND m.user_id = $2
+ )`, id, actor).
+ Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash,
+ &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return attachment.Attachment{}, apperr.NotFound("attachment not found")
}
diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go
index a810545f..02ec9b80 100644
--- a/internal/attachment/validation.go
+++ b/internal/attachment/validation.go
@@ -16,6 +16,11 @@ import (
const MaxUploadBytes int64 = 10 * 1024 * 1024
const MaxImagePixels int64 = 40_000_000
+var imageFormats = map[string]string{
+ "image/png": "png",
+ "image/jpeg": "jpeg",
+}
+
func ValidateFilename(name string) error {
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 {
return apperr.Invalid("filename must contain 1-255 characters")
@@ -37,7 +42,7 @@ func ValidateImage(r io.Reader, mime string, maxPixels int64) error {
if err != nil {
return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err)
}
- if (format != "png" && format != "jpeg") || "image/"+format != mime {
+ if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat {
return apperr.UnsupportedMedia("image does not match Content-Type")
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) {
diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go
index 7ef3c509..0c7b4092 100644
--- a/internal/attachment/validation_test.go
+++ b/internal/attachment/validation_test.go
@@ -8,11 +8,44 @@ import (
"image/color"
"image/png"
"io"
- "messenger/internal/platform/apperr"
+ "os"
"strings"
"testing"
+
+ "github.com/google/uuid"
+
+ "messenger/internal/platform/apperr"
)
+type chatMember struct {
+ chatID, userID uuid.UUID
+}
+
+func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) {
+ return chatID == m.chatID && userID == m.userID, nil
+}
+
+func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) {
+ member := chatMember{chatID: uuid.New(), userID: uuid.New()}
+ s := NewService(nil, member, nil, Config{})
+ input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4}
+ for _, tc := range []struct {
+ name string
+ actor uuid.UUID
+ kind apperr.Kind
+ }{
+ {name: "member", actor: member.userID, kind: apperr.KindUnavailable},
+ {name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data"))
+ if apperr.KindOf(err) != tc.kind {
+ t.Fatalf("got %v, want %v", err, tc.kind)
+ }
+ })
+ }
+}
+
func TestValidateFilename(t *testing.T) {
for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} {
if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid {
@@ -41,16 +74,35 @@ type openedBlob struct {
}
func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) {
+ if reader, ok := b.reader.(io.ReadCloser); ok {
+ return reader, nil
+ }
return io.NopCloser(b.reader), nil
}
+type closeErrorBody struct{ io.Reader }
+
+func (closeErrorBody) Close() error { return errors.New("injected close failure") }
+
+func TestStagingCloseFailurePreventsPublication(t *testing.T) {
+ data, err := os.ReadFile("../../tests/fixtures/sample.png")
+ if err != nil {
+ t.Fatal(err)
+ }
+ blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}}
+ s := NewService(nil, nil, blobs, Config{})
+ if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable {
+ t.Fatalf("close failure classified as %v", err)
+ }
+}
+
func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) {
var pngData bytes.Buffer
if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
t.Fatal(err)
}
for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} {
- s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0)
+ s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{})
err := s.validate(context.Background(), "staging/test", "image/png")
if apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err)
diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go
index cd6df783..4af60ff1 100644
--- a/internal/chat/store/postgres.go
+++ b/internal/chat/store/postgres.go
@@ -18,6 +18,7 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
+ "messenger/internal/attachment"
"messenger/internal/chat"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
@@ -202,7 +203,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
if err != nil {
return fmt.Errorf("check attachments: %w", err)
}
- statuses, err := pgx.CollectRows(rows, pgx.RowTo[string])
+ statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus])
if err != nil {
return fmt.Errorf("read attachment status: %w", err)
}
@@ -210,7 +211,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
return apperr.NotFound("attachment not found")
}
for _, status := range statuses {
- if status != "ready" {
+ if status != attachment.FileReady {
return apperr.Conflict("attachment is not ready")
}
}
diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go
index 10b4b523..2a1e485b 100644
--- a/internal/platform/config/config.go
+++ b/internal/platform/config/config.go
@@ -134,7 +134,7 @@ func Load(lookup Lookup) (Config, error) {
}
cfg.DebugAddr = v
}
- if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
+ if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
return Config{}, err
}
if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil {
@@ -154,12 +154,16 @@ func LoadMigrate(lookup Lookup) (Migrate, error) {
if cfg.Logging, err = loadLogging(lookup); err != nil {
return Migrate{}, err
}
- if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
+ if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
return Migrate{}, err
}
return cfg, nil
}
+func LoadDatabaseURL(lookup Lookup) (string, error) {
+ return secret(lookup, "DATABASE_URL_FILE")
+}
+
func loadLogging(lookup Lookup) (Logging, error) {
l := Logging{Env: "lab", Level: slog.LevelInfo}
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go
index 7a6385e9..d68429ff 100644
--- a/internal/platform/s3store/bootstrap_test.go
+++ b/internal/platform/s3store/bootstrap_test.go
@@ -14,6 +14,7 @@ import (
"testing"
"time"
+ "messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -85,7 +86,7 @@ func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) {
}
}))
defer server.Close()
- s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
+ s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -126,7 +127,7 @@ func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) {
io.WriteString(w, `ServiceUnavailablevolume not registered`)
}))
defer server.Close()
- s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
+ s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go
index 9798a581..4015511b 100644
--- a/internal/platform/s3store/integration_test.go
+++ b/internal/platform/s3store/integration_test.go
@@ -19,6 +19,7 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
"messenger/internal/attachment"
+ "messenger/internal/platform/config"
"messenger/internal/platform/s3store"
"messenger/internal/testenv"
)
@@ -149,7 +150,7 @@ func TestSeaweedStore(t *testing.T) {
t.Fatal(err)
}
for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} {
- outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1])
+ outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]})
if err != nil {
t.Fatal(err)
}
@@ -219,7 +220,7 @@ func TestSeaweedStore(t *testing.T) {
t.Fatal(err)
}
}
- missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey)
+ missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go
index 2c4b9b9e..86d1bb0f 100644
--- a/internal/platform/s3store/store.go
+++ b/internal/platform/s3store/store.go
@@ -16,10 +16,13 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
"messenger/internal/attachment"
+ "messenger/internal/platform/config"
)
const MaxPageSize = 500
+const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
+
type Store struct {
client *minio.Core
region string
@@ -30,15 +33,15 @@ var _ attachment.Blobs = (*Store)(nil)
// New validates configuration without contacting S3, so text-only service startup
// and readiness do not depend on object storage being reachable.
-func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) {
- u, err := url.Parse(endpoint)
+func New(cfg config.S3) (*Store, error) {
+ u, err := url.Parse(cfg.Endpoint)
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment")
}
- if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" {
+ if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
return nil, errors.New("s3: region and credentials are required")
}
- if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+ if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil {
return nil, fmt.Errorf("s3: invalid bucket: %w", err)
}
transport := &http.Transport{
@@ -56,9 +59,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
DisableCompression: true,
}
client, err := minio.NewCore(u.Host, &minio.Options{
- Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
+ Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: u.Scheme == "https",
- Region: region,
+ Region: cfg.Region,
BucketLookup: minio.BucketLookupPath,
Transport: transport,
// A consumed upload stream cannot be replayed. Let the caller decide
@@ -68,7 +71,7 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
if err != nil {
return nil, fmt.Errorf("s3: client: %w", err)
}
- return &Store{client: client, region: region, bucket: bucket}, nil
+ return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil
}
// Bootstrap is used only by the admin init command, never by API startup.
@@ -78,7 +81,7 @@ func (s *Store) Bootstrap(ctx context.Context) error {
}
err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region})
code := minio.ToErrorResponse(err).Code
- if err != nil && code != "BucketAlreadyOwnedByYou" {
+ if err != nil && code != codeBucketAlreadyOwnedByYou {
return wrap("create bucket", err)
}
return s.WaitReady(ctx)
diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go
index cac6af56..41fb2383 100644
--- a/internal/platform/s3store/store_test.go
+++ b/internal/platform/s3store/store_test.go
@@ -11,6 +11,7 @@ import (
"testing"
"time"
+ "messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -18,7 +19,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) }))
defer server.Close()
- if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil {
+ if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil {
t.Fatal(err)
}
if calls.Load() != 0 {
@@ -36,7 +37,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
{"secret", server.URL, "us-east-1", "attachments", "access", ""},
} {
t.Run(tc.name, func(t *testing.T) {
- if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil {
+ if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil {
t.Fatal("accepted invalid configuration")
}
})
@@ -69,7 +70,7 @@ func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) {
w.Header().Set("ETag", `"probe"`)
}))
defer server.Close()
- store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
+ store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -103,7 +104,7 @@ func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) }))
defer server.Close()
- store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
+ store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -154,7 +155,7 @@ func TestListStopsAfterOneBoundedPage(t *testing.T) {
io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`)
}))
defer server.Close()
- store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
+ store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go
index 062c9f99..36e551bf 100644
--- a/internal/testenv/s3.go
+++ b/internal/testenv/s3.go
@@ -22,6 +22,7 @@ import (
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
+ "messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -131,15 +132,15 @@ func (s *S3) launch(ctx context.Context) error {
if err != nil {
return err
}
- s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey)
+ s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey})
if err != nil {
return err
}
- s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey)
+ s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey})
if err != nil {
return err
}
- s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey)
+ s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
return err
}
diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go
index b1b3ab47..2c945867 100644
--- a/tools/reconcilebench/main.go
+++ b/tools/reconcilebench/main.go
@@ -119,7 +119,8 @@ func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string,
}
}()
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
- stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
+ options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
+ stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
close(done)
m.GoHeapPeakBytes = <-peakCh
m.Stats = stats