diff --git a/REPORT.md b/REPORT.md index 77fe5799..90eb7880 100644 --- a/REPORT.md +++ b/REPORT.md @@ -78,7 +78,7 @@ Grace period — 1 h, минимум 2 min, выше бюджета upload. Св | Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. | | Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. | -Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready. +Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает отсутствующий blob и файл в failed. Для ready с существующим объектом того же размера действует быстрый Stat-путь. Deep-повреждение такого ready диагностируется и требует отдельного восстановления из достоверных байтов. Простое появление объекта само не меняет failed на ready. ```sh make reconcile @@ -136,7 +136,7 @@ Bootstrap — одноразовый `s3-init` с отдельным admin-кл | --- | ---: | ---: | --- | | Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart | | Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart | -| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart | +| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.2 | 201, checksum совпал, без OOM/restart | Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки. diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go index b7090a83..97b79811 100644 --- a/cmd/reconcile/main.go +++ b/cmd/reconcile/main.go @@ -43,7 +43,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m") return 2 } - dbConfig, err := config.LoadMigrate(os.LookupEnv) + databaseURL, err := config.LoadDatabaseURL(os.LookupEnv) if err != nil { fmt.Fprintln(errOut, "configuration error:", err) return 2 @@ -54,12 +54,12 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { fmt.Fprintln(errOut, "configuration error:", err) return 2 } - blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey) + blobs, err := s3store.New(s3Config) if err != nil { fmt.Fprintln(errOut, "configuration error:", err) return 2 } - db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10) + db, err := postgres.NewPool(ctx, databaseURL, 10) if err != nil { fmt.Fprintln(errOut, "reconcile:", err) return 1 @@ -76,7 +76,8 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { defer cancel() encoder := json.NewEncoder(out) var outputErr error - stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) { + options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep} + stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) { if outputErr == nil { outputErr = encoder.Encode(f) if outputErr != nil { diff --git a/cmd/reconcile/main_test.go b/cmd/reconcile/main_test.go new file mode 100644 index 00000000..806f2d67 --- /dev/null +++ b/cmd/reconcile/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) { + for key, value := range map[string]string{ + "DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable", + "S3_ACCESS_KEY_ID_FILE": "test-access", + "S3_SECRET_ACCESS_KEY_FILE": "test-secret", + } { + path := filepath.Join(t.TempDir(), key) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + t.Setenv(key, path) + } + t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333") + t.Setenv("S3_REGION", "us-east-1") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("APP_LOG_LEVEL", "not-a-log-level") + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Stop before dialing: only configuration relevant to this CLI is checked. + var out, errOut bytes.Buffer + code := run(ctx, nil, &out, &errOut) + if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") { + t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String()) + } +} diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go index db308002..164c3b01 100644 --- a/cmd/s3init/main.go +++ b/cmd/s3init/main.go @@ -3,10 +3,11 @@ package main import ( "context" "fmt" - "messenger/internal/platform/config" - "messenger/internal/platform/s3store" "os" "time" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" ) func main() { @@ -20,7 +21,7 @@ func run() error { if err != nil { return err } - s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey) + s, err := s3store.New(cfg) if err != nil { return err } diff --git a/cmd/server/main.go b/cmd/server/main.go index cb4aec4a..4a3c85b8 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,4 +1,4 @@ -// Command server starts the messenger API over PostgreSQL and Valkey. +// Command server starts the messenger API over PostgreSQL, Valkey and S3. package main import ( @@ -61,25 +61,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } defer sessions.Close() - var blobs attachment.Blobs - var reconciler *attachment.Reconciler - if cfg.S3.Endpoint != "" { - blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey) - if err != nil { - return err - } - reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv)) - if e != nil { - return e - } - reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey) - if e != nil { - return e - } - reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour) - if err != nil { - return err - } + blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv) + if err != nil { + return err } ln, err := net.Listen("tcp", cfg.Addr()) @@ -100,8 +84,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, Blobs: blobs, - MaxUploadBytes: cfg.S3.MaxBytes, - MaxImagePixels: cfg.S3.MaxPixels, + AttachmentsCfg: attachment.Config{ + MaxBytes: cfg.S3.MaxBytes, + MaxPixels: cfg.S3.MaxPixels, + }, }) var debugLn net.Listener if cfg.DebugAddr != "" { @@ -124,7 +110,7 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return case <-ticker.C: scan, cancel := context.WithTimeout(ctx, 2*time.Minute) - stats, err := reconciler.Run(scan, true, false, nil) + stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil) cancel() if err != nil { log.Warn("attachment cleanup incomplete", "error", err) @@ -149,6 +135,29 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } +func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) { + if cfg.Endpoint == "" { + return nil, nil, nil + } + blobs, err := s3store.New(cfg) + if err != nil { + return nil, nil, err + } + reconCfg, err := config.LoadS3(config.ReconcileS3(lookup)) + if err != nil { + return nil, nil, err + } + reconBlobs, err := s3store.New(reconCfg) + if err != nil { + return nil, nil, err + } + reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour) + if err != nil { + return nil, nil, err + } + return blobs, reconciler, nil +} + // publishDebugVars adds to expvar what the load tests read from a replica. func publishDebugVars(alias string, pool *pgxpool.Pool) { expvar.NewString("instance").Set(alias) diff --git a/compose.yaml b/compose.yaml index bc08ab33..6290c342 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,9 @@ name: lab4 services: lb: image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa + depends_on: + api: + condition: service_healthy volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro ports: diff --git a/internal/app/app.go b/internal/app/app.go index 3949add5..9ab9a811 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -39,20 +39,23 @@ type Deps struct { Sessions *redis.Client SessionTTL time.Duration MaxBodyBytes int64 - // RequestTimeout bounds every API request; zero means the default. - RequestTimeout time.Duration - InstanceID string + // Zero timeouts use the defaults below. + RequestTimeout time.Duration + BinaryRequestTimeout time.Duration + InstanceID string // IdempotencyTTL zero means the contract's 24 hours. IdempotencyTTL time.Duration Blobs attachment.Blobs - MaxUploadBytes int64 - MaxImagePixels int64 + AttachmentsCfg attachment.Config } -// defaultRequestTimeout stays below the server's 10s write timeout, so a +// defaultRequestTimeout stays below the server's write timeout, so a // request that gives up on a dependency can still write its 503. const defaultRequestTimeout = 5 * time.Second +// Binary transfers and decoding share this budget, below the 35s socket timeout. +const defaultBinaryRequestTimeout = 30 * time.Second + const defaultIdempotencyTTL = 24 * time.Hour func New(d Deps) *App { @@ -61,7 +64,8 @@ func New(d Deps) *App { if keyTTL == 0 { keyTTL = defaultIdempotencyTTL } - chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL) + chatRepository := chatstore.NewChats(d.DB) + chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL) engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) probes := health.New( @@ -81,9 +85,15 @@ func New(d Deps) *App { auth.NewHandler(sessions).Register(public, protected) user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) - attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels) - binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions)) - attachment.NewHandler(attachments).Register(protected, binary) + attachments := attachment.NewService( + attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg, + ) + binaryTimeout := d.BinaryRequestTimeout + if binaryTimeout == 0 { + binaryTimeout = defaultBinaryRequestTimeout + } + binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions)) + attachment.NewHandler(attachments).Register(protected, binaryProtected) return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go index 2041d5e6..0b6411b7 100644 --- a/internal/app/attachment_test.go +++ b/internal/app/attachment_test.go @@ -30,32 +30,54 @@ func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) { ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized") } -func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) { - ts := newTester(t) - actor := ts.user("deadline_user", "User") - ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond}) - var held []*pgxpool.Conn - for range db.Pool.Config().MaxConns { - conn, err := db.Pool.Acquire(context.Background()) - if err != nil { - t.Fatal(err) +func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) { + for _, binary := range []bool{false, true} { + name := "metadata" + requestTimeout := 40 * time.Millisecond + var binaryTimeout time.Duration + if binary { + name = "upload" + requestTimeout = time.Second + binaryTimeout = 40 * time.Millisecond } - held = append(held, conn) - } - defer func() { - for _, conn := range held { - conn.Release() - } - }() - ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) - defer cancel() - req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx) - req.Header.Set("Authorization", "Bearer "+actor.token) - rec := httptest.NewRecorder() - started := time.Now() - ts.a.Engine.ServeHTTP(rec, req) - if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { - t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + t.Run(name, func(t *testing.T) { + ts := newTester(t) + actor := ts.user("deadline_user", "User") + ts.a = app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, + SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, + RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout, + }) + var held []*pgxpool.Conn + for range db.Pool.Config().MaxConns { + conn, err := db.Pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + held = append(held, conn) + } + defer func() { + for _, conn := range held { + conn.Release() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111" + req := httptest.NewRequest("GET", path, nil).WithContext(ctx) + if binary { + path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png" + req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx) + req.Header.Set("Content-Type", "image/png") + } + req.Header.Set("Authorization", "Bearer "+actor.token) + rec := httptest.NewRecorder() + started := time.Now() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { + t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + } + }) } } diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go index 9a5cd171..69177192 100644 --- a/internal/attachment/attachment.go +++ b/internal/attachment/attachment.go @@ -1,9 +1,17 @@ package attachment import ( - "context" - "github.com/google/uuid" "time" + + "github.com/google/uuid" +) + +type FileStatus string + +const ( + FilePending FileStatus = "pending" + FileReady FileStatus = "ready" + FileFailed FileStatus = "failed" ) type Attachment struct { @@ -11,12 +19,14 @@ type Attachment struct { Filename string FileHash []byte SizeBytes int64 - MIMEType, Status string + MIMEType string + Status FileStatus CreatedAt time.Time } -type Repository interface { - IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error) - Publish(context.Context, Attachment, string) error - GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error) +type UploadInput struct { + ChatID uuid.UUID + Filename string + MIMEType string + SizeBytes int64 } diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go index c211b36e..3958653b 100644 --- a/internal/attachment/blob.go +++ b/internal/attachment/blob.go @@ -16,10 +16,10 @@ type Object struct { } type Blobs interface { - Put(context.Context, string, io.Reader, int64, string) error - Open(context.Context, string) (io.ReadCloser, error) - Stat(context.Context, string) (Object, error) - Copy(context.Context, string, string) error - Delete(context.Context, string) error - List(context.Context, string, string, int) ([]Object, error) + Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error + Open(ctx context.Context, key string) (io.ReadCloser, error) + Stat(ctx context.Context, key string) (Object, error) + Copy(ctx context.Context, sourceKey, destinationKey string) error + Delete(ctx context.Context, key string) error + List(ctx context.Context, prefix, after string, limit int) ([]Object, error) } diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go index 6420519f..5adbd0e7 100644 --- a/internal/attachment/handler.go +++ b/internal/attachment/handler.go @@ -8,40 +8,52 @@ import ( "net/http" "strconv" - "github.com/gin-gonic/gin" - "github.com/google/uuid" "messenger/internal/platform/apperr" "messenger/internal/platform/httpx" "messenger/internal/platform/ident" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" ) type Handler struct{ s *Service } func NewHandler(s *Service) *Handler { return &Handler{s} } -func (h *Handler) Register(protected, binary gin.IRouter) { - binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) +func (h *Handler) Register(protected, binaryProtected gin.IRouter) { + binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata)) - binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) + binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) } type Response struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []any `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + // Lab4's shared response contract requires variants=[], even without processing. + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } func toResponse(a Attachment) Response { return Response{ - ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt), + ID: a.ID.String(), + ChatID: a.ChatID.String(), + OwnerID: a.OwnerID.String(), + Filename: a.Filename, + ContentType: a.MIMEType, + SizeBytes: a.SizeBytes, + SHA256: hex.EncodeToString(a.FileHash), + Status: string(a.Status), + Variants: []any{}, + CreatedAt: httpx.Timestamp(a.CreatedAt), + UpdatedAt: httpx.Timestamp(a.CreatedAt), } } @@ -65,8 +77,14 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) { httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) return } - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes) - a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body) + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes) + input := UploadInput{ + ChatID: id, + Filename: names[0], + MIMEType: media, + SizeBytes: c.Request.ContentLength, + } + a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body) if err != nil { httpx.Error(c, err) return @@ -109,7 +127,11 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } - defer r.Close() + defer func() { + if err := r.Close(); err != nil { + slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err) + } + }() c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) c.Header("X-Content-Type-Options", "nosniff") c.Header("Content-Type", a.MIMEType) diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go index 729e42f6..b8a11c05 100644 --- a/internal/attachment/reconcile.go +++ b/internal/attachment/reconcile.go @@ -45,6 +45,11 @@ type Reconciler struct { grace time.Duration } +type ReconcileOptions struct { + Apply bool + Deep bool +} + func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) { if db == nil || blobs == nil { return nil, errors.New("reconcile: database and blobs are required") @@ -61,10 +66,18 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci // Run streams findings and keeps at most one page of metadata in memory. // Dry runs only read storage. Each apply candidate is checked again under the // publisher's transaction lock; no scan snapshot authorizes a later mutation. -func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) { +func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) { started := time.Now() defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }() - run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)} + run := reconcileRun{ + r: r, + ctx: ctx, + apply: options.Apply, + deep: options.Deep, + emit: emit, + stats: &stats, + cutoff: started.Add(-r.grace), + } for _, prefix := range []string{"staging/", "blobs/"} { if err = run.objects(prefix); err != nil { return stats, err @@ -203,7 +216,8 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { return known, nil } s.stats.SQLQueries++ - rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes) + rows, err := s.r.db.Query(s.ctx, ` + SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes) if err != nil { return nil, err } @@ -235,7 +249,8 @@ func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) { op.ctx = opCtx var exists bool s.stats.SQLQueries++ - if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil { + if err := tx.QueryRow(opCtx, ` + SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil { return nil, err } if exists { @@ -304,7 +319,7 @@ func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Fi type reconcileFile struct { hash []byte size int64 - status string + status FileStatus created time.Time referenced bool } @@ -316,9 +331,13 @@ func (s *reconcileRun) files() error { return err } s.stats.SQLQueries++ - rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, - EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) - FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after) + rows, err := s.r.db.Query(s.ctx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash > $1 + ORDER BY f.hash + LIMIT $2`, after, reconcilePageSize) if err != nil { return err } @@ -346,30 +365,9 @@ func (s *reconcileRun) files() error { if !s.candidate(f) { continue } - var finding *Finding - if s.apply { - finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { - op := *s - op.ctx = opCtx - var current reconcileFile - s.stats.SQLQueries++ - err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, - EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) - FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash). - Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - if !s.candidate(current) { - return nil, nil - } - return op.inspect(current, tx) - }) - } else { - finding, err = s.inspect(f, nil) + finding, err := s.inspect(f) + if err == nil && s.apply && finding != nil && finding.Action != "report" { + finding, err = s.repairFile(f.hash) } if err != nil { return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err) @@ -379,17 +377,66 @@ func (s *reconcileRun) files() error { } } +func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) { + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var current reconcileFile + s.stats.SQLQueries++ + err := tx.QueryRow(opCtx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash = $1 + FOR UPDATE OF f`, hash). + Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if !op.candidate(current) { + return nil, nil + } + finding, err := op.inspect(current) + if err != nil || finding == nil { + return finding, err + } + switch finding.Action { + case "delete_file": + // Delete bytes first. A failed SQL commit leaves an unreferenced + // missing row that the next pass can safely finish collecting. + if err := op.delete(finding.Key); err != nil { + return nil, err + } + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash) + case "mark_failed": + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed) + default: + return finding, nil + } + if err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + }) +} + func (s *reconcileRun) candidate(f reconcileFile) bool { if !s.old(f.created) { return false } if !f.referenced { - return f.status == "pending" || f.status == "ready" + return f.status == FilePending || f.status == FileReady } - return f.status == "ready" || f.status == "failed" + return f.status == FileReady || f.status == FileFailed } -func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { +func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) { key := BlobKey(f.hash) if !f.referenced { // A publication that rolled back can leave fresh bytes behind an old @@ -401,36 +448,14 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { if err == nil && !s.old(obj.Modified) { return nil, nil } - finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"} - if !s.apply { - return finding, nil - } - // S3 must succeed first. A failed SQL commit then leaves a missing, - // unreferenced row that the next run can safely finish collecting. - if err := s.delete(key); err != nil { - return nil, err - } - s.stats.SQLQueries++ - if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil { - return nil, err - } - finding.Applied = true - return finding, nil + return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil } obj, err := s.stat(key) if errors.Is(err, ErrObjectMissing) { - if f.status == "failed" { + if f.status == FileFailed { return nil, nil } - finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"} - if s.apply { - s.stats.SQLQueries++ - if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil { - return nil, err - } - finding.Applied = true - } - return finding, nil + return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil } if err != nil { return nil, err @@ -449,7 +474,7 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { } // A failed file stays failed even when the key reappears. Recovery requires // an explicit publisher; a shallow HEAD cannot prove content integrity. - if f.status == "failed" { + if f.status == FileFailed { return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil } return nil, nil diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go index 39fd6dcd..d7e26884 100644 --- a/internal/attachment/reconcile_test.go +++ b/internal/attachment/reconcile_test.go @@ -209,7 +209,8 @@ func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment f.t.Fatal(err) } var found []attachment.Finding - stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) }) + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) }) return stats, found, err } @@ -221,6 +222,28 @@ func TestReconciler(t *testing.T) { t.Fatal(err) } defer db.Close() + t.Run("healthy apply does not block publication", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("healthy", "ready", 2*time.Hour, true, true) + tx, err := db.Pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err := attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, f.b, time.Hour) + if err != nil { + t.Fatal(err) + } + scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) + if err != nil || stats.Findings != 0 || stats.Repaired != 0 { + t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err) + } + }) t.Run("constructor validates and defaults grace", func(t *testing.T) { f := reconcileSetup(t, db) for _, grace := range []time.Duration{-time.Second, time.Minute} { @@ -361,7 +384,7 @@ func TestReconciler(t *testing.T) { done := make(chan error, 1) go func() { r, _ := attachment.NewReconciler(f.db, f.b, 0) - _, err := r.Run(ctx, true, false, nil) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) done <- err }() waitReconcileLock(t, ctx, f.db) @@ -395,7 +418,7 @@ func TestReconciler(t *testing.T) { done := make(chan error, 1) go func() { r, _ := attachment.NewReconciler(f.db, f.b, 0) - _, err := r.Run(ctx, true, false, nil) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) done <- err }() waitReconcileLock(t, ctx, f.db) @@ -592,7 +615,7 @@ func TestReconcilerFailureBoundaries(t *testing.T) { if err != nil { t.Fatal(err) } - stats, err := r.Run(ctx, false, true, nil) + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil) if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" { t.Fatalf("read only dry run: %+v %v", stats, err) } @@ -681,11 +704,11 @@ func TestReconcilerS3(t *testing.T) { t.Fatalf("CLI mutated by default: %+v %v", cliStats, err) } var found []attachment.Finding - stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) }) + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) }) if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" { t.Fatalf("S3 dry: %+v %v %+v", stats, err, found) } - stats, err = r.Run(ctx, true, true, nil) + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" { t.Fatalf("S3 apply: %+v %v", stats, err) } @@ -695,7 +718,7 @@ func TestReconcilerS3(t *testing.T) { if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil { t.Fatal("fresh staging deleted", err) } - stats, err = r.Run(ctx, true, true, nil) + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) if err != nil || stats.Repaired != 0 { t.Fatalf("S3 repeat: %+v %v", stats, err) } diff --git a/internal/attachment/repository.go b/internal/attachment/repository.go new file mode 100644 index 00000000..9c554118 --- /dev/null +++ b/internal/attachment/repository.go @@ -0,0 +1,23 @@ +package attachment + +import ( + "context" + + "github.com/google/uuid" +) + +type AttachmentRepository interface { + // WithPublication holds the file's hash lock through the callback and commit. + // Publication is valid only inside the callback. Any callback error rolls back. + WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error + GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error) +} + +type Publication interface { + FileReady() bool + SaveReady(ctx context.Context) error +} + +type ChatMembership interface { + IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error) +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go index ee574176..8363e1e9 100644 --- a/internal/attachment/service.go +++ b/internal/attachment/service.go @@ -15,40 +15,54 @@ import ( ) type Service struct { - repo Repository - blobs Blobs - maxBytes, maxPixels int64 - decode chan struct{} + attachments AttachmentRepository + chats ChatMembership + blobs Blobs + config Config + decode chan struct{} } -func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service { - if maxBytes == 0 { - maxBytes = MaxUploadBytes - } - if maxPixels == 0 { - maxPixels = MaxImagePixels - } - return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)} +type Config struct { + MaxBytes int64 + MaxPixels int64 } -func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) { - if err := ValidateFilename(filename); err != nil { +// A nil blobs dependency allows the text API to run without configured S3. +// Binary operations report unavailability in that mode. +func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service { + if cfg.MaxBytes == 0 { + cfg.MaxBytes = MaxUploadBytes + } + if cfg.MaxPixels == 0 { + cfg.MaxPixels = MaxImagePixels + } + return &Service{ + attachments: attachments, + chats: chats, + blobs: blobs, + config: cfg, + decode: make(chan struct{}, 1), + } +} + +func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) { + if err := ValidateFilename(input.Filename); err != nil { return Attachment{}, err } - if mime != "image/png" && mime != "image/jpeg" { + if _, accepted := imageFormats[input.MIMEType]; !accepted { return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted") } - ok, err := s.repo.IsMember(ctx, chatID, actor) + ok, err := s.chats.IsMember(ctx, input.ChatID, actor) if err != nil { return Attachment{}, err } if !ok { return Attachment{}, apperr.NotFound("chat not found") } - if size > s.maxBytes { + if input.SizeBytes > s.config.MaxBytes { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } - if size <= 0 { + if input.SizeBytes <= 0 { return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required") } if s.blobs == nil { @@ -59,16 +73,17 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, defer func() { cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() + // A failed cleanup leaves a staging object for the reconciler's grace-based collection. if err := s.blobs.Delete(cleanup, stage); err != nil { - slog.Warn("staging cleanup deferred", "error", err) + slog.Warn("staging cleanup deferred", "key", stage, "error", err) } }() hash := sha256.New() - counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)} - err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime) + counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)} + err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType) if err != nil { _, tooLarge := errors.AsType[*http.MaxBytesError](err) - if counter.n > s.maxBytes || tooLarge { + if counter.n > s.config.MaxBytes || tooLarge { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { @@ -78,10 +93,10 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, } var extra [1]byte n, readErr := counter.Read(extra[:]) - if counter.n > s.maxBytes { + if counter.n > s.config.MaxBytes { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } - if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) { + if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) { return Attachment{}, apperr.Invalid("body length does not match Content-Length") } // Full decoding is bounded by dimensions and one decoder per replica. Body @@ -91,26 +106,64 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, case <-ctx.Done(): return Attachment{}, unavailable(ctx.Err()) } - err = s.validate(ctx, stage, mime) + err = s.validate(ctx, stage, input.MIMEType) <-s.decode if err != nil { return Attachment{}, err } - a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)} - if err := s.repo.Publish(ctx, a, stage); err != nil { + a := Attachment{ + ID: id, + ChatID: input.ChatID, + OwnerID: actor, + Filename: input.Filename, + FileHash: hash.Sum(nil), + SizeBytes: input.SizeBytes, + MIMEType: input.MIMEType, + Status: FileReady, + CreatedAt: ident.TimeOf(id), + } + if err := s.attachments.WithPublication(ctx, a, func(p Publication) error { + if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil { + return err + } + return p.SaveReady(ctx) + }); err != nil { return Attachment{}, err } return a, nil } -func (s *Service) validate(ctx context.Context, stage, mime string) error { +func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error { + blobKey := BlobKey(a.FileHash) + if fileReady { + obj, err := s.blobs.Stat(ctx, blobKey) + if err == nil && obj.Size == a.SizeBytes { + return nil + } + if err != nil && !errors.Is(err, ErrObjectMissing) { + return unavailable(err) + } + } + // The hash is known after streaming. Copy the validated staging object to + // its permanent CAS key while the repository's publication lock is held. + if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil { + return unavailable(err) + } + return nil +} + +func (s *Service) validate(ctx context.Context, stage, mime string) (err error) { r, err := s.blobs.Open(ctx, stage) if err != nil { return unavailable(err) } - defer r.Close() + defer func() { + if closeErr := r.Close(); closeErr != nil && err == nil { + err = unavailable(closeErr) + } + }() tracked := &storageReader{reader: r} - err = ValidateImage(tracked, mime, s.maxPixels) + err = ValidateImage(tracked, mime, s.config.MaxPixels) if tracked.err != nil { return unavailable(tracked.err) } @@ -136,11 +189,11 @@ func (r *storageReader) Read(p []byte) (int, error) { } func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) { - a, err := s.repo.GetForMember(ctx, id, actor) + a, err := s.attachments.GetForMember(ctx, id, actor) if err != nil { return Attachment{}, err } - if a.Status != "ready" { + if a.Status != FileReady { return Attachment{}, apperr.Conflict("attachment is not ready") } return a, nil @@ -166,7 +219,12 @@ type countReader struct { n int64 } -func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e } +func (c *countReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.n += int64(n) + return n, err +} + func unavailable(err error) error { return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) } diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go index c207a956..f01a3540 100644 --- a/internal/attachment/service_test.go +++ b/internal/attachment/service_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" "messenger/internal/attachment" "messenger/internal/attachment/store" + chatstore "messenger/internal/chat/store" "messenger/internal/platform/apperr" "messenger/internal/platform/ident" "messenger/internal/testenv" @@ -73,10 +74,11 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { t.Fatal(err) } b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})} - s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0) + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{}) uploaded := make(chan error, 1) go func() { - _, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data)) + input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + _, e := s.Upload(ctx, actor, input, bytes.NewReader(data)) uploaded <- e }() select { @@ -90,7 +92,7 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { } collected := make(chan error, 1) go func() { - stats, e := r.Run(ctx, true, false, nil) + stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) if e == nil && stats.Repaired != 0 { e = errors.New("collector removed a newly referenced file") } @@ -196,7 +198,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { } } b := &testBlobs{data: map[string][]byte{}} - s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels) + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{}) data, err := os.ReadFile("../../tests/fixtures/sample.png") if err != nil { t.Fatal(err) @@ -210,7 +212,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { c = c2 } var e error - results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data)) + input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data)) if e != nil { t.Error(e) } @@ -235,7 +238,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { b.mu.Lock() clear(b.data) b.mu.Unlock() - if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil { + input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil { t.Fatal(err) } if b.copies != 2 { @@ -244,7 +248,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { b.failCopy = true other := bytes.Clone(data) other = append(other, 0) - if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { + input.Filename, input.SizeBytes = "failed.png", int64(len(other)) + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { t.Fatalf("copy outage: %v", err) } if len(b.data) != 1 { @@ -258,7 +263,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil { + input.Filename = "rollback.png" + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil { t.Fatal("publication survived rejected metadata") } if len(b.data) != 2 { diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go index 607932a9..32105070 100644 --- a/internal/attachment/store/postgres.go +++ b/internal/attachment/store/postgres.go @@ -4,32 +4,29 @@ import ( "context" "errors" "fmt" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" ) type Attachments struct { - pool *pgxpool.Pool - blobs attachment.Blobs + pool *pgxpool.Pool } -func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments { - return &Attachments{pool, blobs} -} +var _ attachment.AttachmentRepository = (*Attachments)(nil) -func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) { - var ok bool - err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok) - return ok, postgres.Classify(err) +func NewAttachments(pool *pgxpool.Pool) *Attachments { + return &Attachments{pool: pool} } // Publication and garbage collection use the same hash lock. The transaction // spans only publication, never client transfer or image decoding. -func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error { +func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error { return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil { return err @@ -37,41 +34,56 @@ func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stag if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil { return err } - var status string - err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status) + var status attachment.FileStatus + err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status) if err != nil && !errors.Is(err, pgx.ErrNoRows) { return err } - copyNeeded := status != "ready" - if !copyNeeded { - obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash)) - if errors.Is(e, attachment.ErrObjectMissing) { - copyNeeded = true - } else if e != nil { - return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e) - } else if obj.Size != a.SizeBytes { - copyNeeded = true - } - } - if copyNeeded { - if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil { - return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) - } - } - _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType) - if err != nil { - return fmt.Errorf("publish file: %w", err) - } - _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) - return err + return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady}) })) } +type publication struct { + tx pgx.Tx + attachment attachment.Attachment + ready bool +} + +func (p publication) FileReady() bool { return p.ready } + +func (p publication) SaveReady(ctx context.Context) error { + a := p.attachment + _, err := p.tx.Exec(ctx, ` + INSERT INTO files (hash, size_bytes, mime_type, status) + VALUES ($1, $2, $3, $4) + ON CONFLICT (hash) DO UPDATE + SET size_bytes = excluded.size_bytes, + mime_type = excluded.mime_type, + status = excluded.status`, + a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady) + if err != nil { + return fmt.Errorf("publish file: %w", err) + } + _, err = p.tx.Exec(ctx, ` + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + VALUES ($1, $2, $3, $4, $5, $6)`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) + return err +} + func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) { var a attachment.Attachment - err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at - FROM attachments a JOIN files f ON f.hash=a.file_hash - WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) + err := r.pool.QueryRow(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash, + f.size_bytes, f.mime_type, f.status, a.created_at + FROM attachments a + JOIN files f ON f.hash = a.file_hash + WHERE a.id = $1 AND EXISTS ( + SELECT 1 FROM chat_members m + WHERE m.chat_id = a.chat_id AND m.user_id = $2 + )`, id, actor). + Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, + &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) if errors.Is(err, pgx.ErrNoRows) { return attachment.Attachment{}, apperr.NotFound("attachment not found") } diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go index a810545f..02ec9b80 100644 --- a/internal/attachment/validation.go +++ b/internal/attachment/validation.go @@ -16,6 +16,11 @@ import ( const MaxUploadBytes int64 = 10 * 1024 * 1024 const MaxImagePixels int64 = 40_000_000 +var imageFormats = map[string]string{ + "image/png": "png", + "image/jpeg": "jpeg", +} + func ValidateFilename(name string) error { if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { return apperr.Invalid("filename must contain 1-255 characters") @@ -37,7 +42,7 @@ func ValidateImage(r io.Reader, mime string, maxPixels int64) error { if err != nil { return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) } - if (format != "png" && format != "jpeg") || "image/"+format != mime { + if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat { return apperr.UnsupportedMedia("image does not match Content-Type") } if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) { diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go index 7ef3c509..0c7b4092 100644 --- a/internal/attachment/validation_test.go +++ b/internal/attachment/validation_test.go @@ -8,11 +8,44 @@ import ( "image/color" "image/png" "io" - "messenger/internal/platform/apperr" + "os" "strings" "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" ) +type chatMember struct { + chatID, userID uuid.UUID +} + +func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) { + return chatID == m.chatID && userID == m.userID, nil +} + +func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) { + member := chatMember{chatID: uuid.New(), userID: uuid.New()} + s := NewService(nil, member, nil, Config{}) + input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4} + for _, tc := range []struct { + name string + actor uuid.UUID + kind apperr.Kind + }{ + {name: "member", actor: member.userID, kind: apperr.KindUnavailable}, + {name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data")) + if apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want %v", err, tc.kind) + } + }) + } +} + func TestValidateFilename(t *testing.T) { for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} { if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid { @@ -41,16 +74,35 @@ type openedBlob struct { } func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) { + if reader, ok := b.reader.(io.ReadCloser); ok { + return reader, nil + } return io.NopCloser(b.reader), nil } +type closeErrorBody struct{ io.Reader } + +func (closeErrorBody) Close() error { return errors.New("injected close failure") } + +func TestStagingCloseFailurePreventsPublication(t *testing.T) { + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}} + s := NewService(nil, nil, blobs, Config{}) + if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("close failure classified as %v", err) + } +} + func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) { var pngData bytes.Buffer if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil { t.Fatal(err) } for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} { - s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0) + s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{}) err := s.validate(context.Background(), "staging/test", "image/png") if apperr.KindOf(err) != apperr.KindUnavailable { t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err) diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index cd6df783..4af60ff1 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -18,6 +18,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" "messenger/internal/chat" "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" @@ -202,7 +203,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [ if err != nil { return fmt.Errorf("check attachments: %w", err) } - statuses, err := pgx.CollectRows(rows, pgx.RowTo[string]) + statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus]) if err != nil { return fmt.Errorf("read attachment status: %w", err) } @@ -210,7 +211,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [ return apperr.NotFound("attachment not found") } for _, status := range statuses { - if status != "ready" { + if status != attachment.FileReady { return apperr.Conflict("attachment is not ready") } } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 10b4b523..2a1e485b 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -134,7 +134,7 @@ func Load(lookup Lookup) (Config, error) { } cfg.DebugAddr = v } - if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { return Config{}, err } if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { @@ -154,12 +154,16 @@ func LoadMigrate(lookup Lookup) (Migrate, error) { if cfg.Logging, err = loadLogging(lookup); err != nil { return Migrate{}, err } - if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { return Migrate{}, err } return cfg, nil } +func LoadDatabaseURL(lookup Lookup) (string, error) { + return secret(lookup, "DATABASE_URL_FILE") +} + func loadLogging(lookup Lookup) (Logging, error) { l := Logging{Env: "lab", Level: slog.LevelInfo} if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go index 7a6385e9..d68429ff 100644 --- a/internal/platform/s3store/bootstrap_test.go +++ b/internal/platform/s3store/bootstrap_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -85,7 +86,7 @@ func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) { } })) defer server.Close() - s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -126,7 +127,7 @@ func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) { io.WriteString(w, `ServiceUnavailablevolume not registered`) })) defer server.Close() - s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go index 9798a581..4015511b 100644 --- a/internal/platform/s3store/integration_test.go +++ b/internal/platform/s3store/integration_test.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "messenger/internal/attachment" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" "messenger/internal/testenv" ) @@ -149,7 +150,7 @@ func TestSeaweedStore(t *testing.T) { t.Fatal(err) } for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} { - outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1]) + outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]}) if err != nil { t.Fatal(err) } @@ -219,7 +220,7 @@ func TestSeaweedStore(t *testing.T) { t.Fatal(err) } } - missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey) + missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) if err != nil { t.Fatal(err) } diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go index 2c4b9b9e..86d1bb0f 100644 --- a/internal/platform/s3store/store.go +++ b/internal/platform/s3store/store.go @@ -16,10 +16,13 @@ import ( "github.com/minio/minio-go/v7/pkg/s3utils" "messenger/internal/attachment" + "messenger/internal/platform/config" ) const MaxPageSize = 500 +const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" + type Store struct { client *minio.Core region string @@ -30,15 +33,15 @@ var _ attachment.Blobs = (*Store)(nil) // New validates configuration without contacting S3, so text-only service startup // and readiness do not depend on object storage being reachable. -func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) { - u, err := url.Parse(endpoint) +func New(cfg config.S3) (*Store, error) { + u, err := url.Parse(cfg.Endpoint) if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment") } - if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" { + if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { return nil, errors.New("s3: region and credentials are required") } - if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil { + if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil { return nil, fmt.Errorf("s3: invalid bucket: %w", err) } transport := &http.Transport{ @@ -56,9 +59,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) DisableCompression: true, } client, err := minio.NewCore(u.Host, &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), Secure: u.Scheme == "https", - Region: region, + Region: cfg.Region, BucketLookup: minio.BucketLookupPath, Transport: transport, // A consumed upload stream cannot be replayed. Let the caller decide @@ -68,7 +71,7 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) if err != nil { return nil, fmt.Errorf("s3: client: %w", err) } - return &Store{client: client, region: region, bucket: bucket}, nil + return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil } // Bootstrap is used only by the admin init command, never by API startup. @@ -78,7 +81,7 @@ func (s *Store) Bootstrap(ctx context.Context) error { } err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region}) code := minio.ToErrorResponse(err).Code - if err != nil && code != "BucketAlreadyOwnedByYou" { + if err != nil && code != codeBucketAlreadyOwnedByYou { return wrap("create bucket", err) } return s.WaitReady(ctx) diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go index cac6af56..41fb2383 100644 --- a/internal/platform/s3store/store_test.go +++ b/internal/platform/s3store/store_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -18,7 +19,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) })) defer server.Close() - if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil { + if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil { t.Fatal(err) } if calls.Load() != 0 { @@ -36,7 +37,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { {"secret", server.URL, "us-east-1", "attachments", "access", ""}, } { t.Run(tc.name, func(t *testing.T) { - if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil { + if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil { t.Fatal("accepted invalid configuration") } }) @@ -69,7 +70,7 @@ func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) { w.Header().Set("ETag", `"probe"`) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -103,7 +104,7 @@ func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -154,7 +155,7 @@ func TestListStopsAfterOneBoundedPage(t *testing.T) { io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go index 062c9f99..36e551bf 100644 --- a/internal/testenv/s3.go +++ b/internal/testenv/s3.go @@ -22,6 +22,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -131,15 +132,15 @@ func (s *S3) launch(ctx context.Context) error { if err != nil { return err } - s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey) + s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey}) if err != nil { return err } - s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey) + s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey}) if err != nil { return err } - s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey) + s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) return err } diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go index b1b3ab47..2c945867 100644 --- a/tools/reconcilebench/main.go +++ b/tools/reconcilebench/main.go @@ -119,7 +119,8 @@ func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, } }() m := measurement{Phase: phase, Mode: mode, TargetFiles: n} - stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) close(done) m.GoHeapPeakBytes = <-peakCh m.Stats = stats