Доработайте публикацию и сверку вложений по итогам ревью
This commit is contained in:
@@ -78,7 +78,7 @@ Grace period — 1 h, минимум 2 min, выше бюджета upload. Св
|
||||
| Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. |
|
||||
| Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. |
|
||||
|
||||
Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready.
|
||||
Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает отсутствующий blob и файл в failed. Для ready с существующим объектом того же размера действует быстрый Stat-путь. Deep-повреждение такого ready диагностируется и требует отдельного восстановления из достоверных байтов. Простое появление объекта само не меняет failed на ready.
|
||||
|
||||
```sh
|
||||
make reconcile
|
||||
@@ -136,7 +136,7 @@ Bootstrap — одноразовый `s3-init` с отдельным admin-кл
|
||||
| --- | ---: | ---: | --- |
|
||||
| Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart |
|
||||
| Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart |
|
||||
| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart |
|
||||
| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.2 | 201, checksum совпал, без OOM/restart |
|
||||
|
||||
Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки.
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
|
||||
fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m")
|
||||
return 2
|
||||
}
|
||||
dbConfig, err := config.LoadMigrate(os.LookupEnv)
|
||||
databaseURL, err := config.LoadDatabaseURL(os.LookupEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
@@ -54,12 +54,12 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
}
|
||||
blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey)
|
||||
blobs, err := s3store.New(s3Config)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "configuration error:", err)
|
||||
return 2
|
||||
}
|
||||
db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10)
|
||||
db, err := postgres.NewPool(ctx, databaseURL, 10)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "reconcile:", err)
|
||||
return 1
|
||||
@@ -76,7 +76,8 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
|
||||
defer cancel()
|
||||
encoder := json.NewEncoder(out)
|
||||
var outputErr error
|
||||
stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) {
|
||||
options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep}
|
||||
stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) {
|
||||
if outputErr == nil {
|
||||
outputErr = encoder.Encode(f)
|
||||
if outputErr != nil {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) {
|
||||
for key, value := range map[string]string{
|
||||
"DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable",
|
||||
"S3_ACCESS_KEY_ID_FILE": "test-access",
|
||||
"S3_SECRET_ACCESS_KEY_FILE": "test-secret",
|
||||
} {
|
||||
path := filepath.Join(t.TempDir(), key)
|
||||
if err := os.WriteFile(path, []byte(value), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(key, path)
|
||||
}
|
||||
t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333")
|
||||
t.Setenv("S3_REGION", "us-east-1")
|
||||
t.Setenv("S3_BUCKET", "attachments")
|
||||
t.Setenv("APP_LOG_LEVEL", "not-a-log-level")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // Stop before dialing: only configuration relevant to this CLI is checked.
|
||||
var out, errOut bytes.Buffer
|
||||
code := run(ctx, nil, &out, &errOut)
|
||||
if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") {
|
||||
t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String())
|
||||
}
|
||||
}
|
||||
+4
-3
@@ -3,10 +3,11 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -20,7 +21,7 @@ func run() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey)
|
||||
s, err := s3store.New(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+32
-23
@@ -1,4 +1,4 @@
|
||||
// Command server starts the messenger API over PostgreSQL and Valkey.
|
||||
// Command server starts the messenger API over PostgreSQL, Valkey and S3.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -61,25 +61,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
return err
|
||||
}
|
||||
defer sessions.Close()
|
||||
var blobs attachment.Blobs
|
||||
var reconciler *attachment.Reconciler
|
||||
if cfg.S3.Endpoint != "" {
|
||||
blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ln, err := net.Listen("tcp", cfg.Addr())
|
||||
@@ -100,8 +84,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
InstanceID: alias,
|
||||
IdempotencyTTL: cfg.IdempotencyTTL,
|
||||
Blobs: blobs,
|
||||
MaxUploadBytes: cfg.S3.MaxBytes,
|
||||
MaxImagePixels: cfg.S3.MaxPixels,
|
||||
AttachmentsCfg: attachment.Config{
|
||||
MaxBytes: cfg.S3.MaxBytes,
|
||||
MaxPixels: cfg.S3.MaxPixels,
|
||||
},
|
||||
})
|
||||
var debugLn net.Listener
|
||||
if cfg.DebugAddr != "" {
|
||||
@@ -124,7 +110,7 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
return
|
||||
case <-ticker.C:
|
||||
scan, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
stats, err := reconciler.Run(scan, true, false, nil)
|
||||
stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("attachment cleanup incomplete", "error", err)
|
||||
@@ -149,6 +135,29 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
return err
|
||||
}
|
||||
|
||||
func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) {
|
||||
if cfg.Endpoint == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
blobs, err := s3store.New(cfg)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
reconCfg, err := config.LoadS3(config.ReconcileS3(lookup))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
reconBlobs, err := s3store.New(reconCfg)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return blobs, reconciler, nil
|
||||
}
|
||||
|
||||
// publishDebugVars adds to expvar what the load tests read from a replica.
|
||||
func publishDebugVars(alias string, pool *pgxpool.Pool) {
|
||||
expvar.NewString("instance").Set(alias)
|
||||
|
||||
@@ -6,6 +6,9 @@ name: lab4
|
||||
services:
|
||||
lb:
|
||||
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
|
||||
+20
-10
@@ -39,20 +39,23 @@ type Deps struct {
|
||||
Sessions *redis.Client
|
||||
SessionTTL time.Duration
|
||||
MaxBodyBytes int64
|
||||
// RequestTimeout bounds every API request; zero means the default.
|
||||
RequestTimeout time.Duration
|
||||
InstanceID string
|
||||
// Zero timeouts use the defaults below.
|
||||
RequestTimeout time.Duration
|
||||
BinaryRequestTimeout time.Duration
|
||||
InstanceID string
|
||||
// IdempotencyTTL zero means the contract's 24 hours.
|
||||
IdempotencyTTL time.Duration
|
||||
Blobs attachment.Blobs
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
AttachmentsCfg attachment.Config
|
||||
}
|
||||
|
||||
// defaultRequestTimeout stays below the server's 10s write timeout, so a
|
||||
// defaultRequestTimeout stays below the server's write timeout, so a
|
||||
// request that gives up on a dependency can still write its 503.
|
||||
const defaultRequestTimeout = 5 * time.Second
|
||||
|
||||
// Binary transfers and decoding share this budget, below the 35s socket timeout.
|
||||
const defaultBinaryRequestTimeout = 30 * time.Second
|
||||
|
||||
const defaultIdempotencyTTL = 24 * time.Hour
|
||||
|
||||
func New(d Deps) *App {
|
||||
@@ -61,7 +64,8 @@ func New(d Deps) *App {
|
||||
if keyTTL == 0 {
|
||||
keyTTL = defaultIdempotencyTTL
|
||||
}
|
||||
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL)
|
||||
chatRepository := chatstore.NewChats(d.DB)
|
||||
chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL)
|
||||
|
||||
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
|
||||
probes := health.New(
|
||||
@@ -81,9 +85,15 @@ func New(d Deps) *App {
|
||||
auth.NewHandler(sessions).Register(public, protected)
|
||||
user.NewHandler(users).Register(public, protected)
|
||||
chat.NewHandler(chats).Register(protected)
|
||||
attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels)
|
||||
binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions))
|
||||
attachment.NewHandler(attachments).Register(protected, binary)
|
||||
attachments := attachment.NewService(
|
||||
attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg,
|
||||
)
|
||||
binaryTimeout := d.BinaryRequestTimeout
|
||||
if binaryTimeout == 0 {
|
||||
binaryTimeout = defaultBinaryRequestTimeout
|
||||
}
|
||||
binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions))
|
||||
attachment.NewHandler(attachments).Register(protected, binaryProtected)
|
||||
|
||||
return &App{Engine: engine, Health: probes, Chats: chats}
|
||||
}
|
||||
|
||||
@@ -30,32 +30,54 @@ func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) {
|
||||
ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
|
||||
func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
actor := ts.user("deadline_user", "User")
|
||||
ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond})
|
||||
var held []*pgxpool.Conn
|
||||
for range db.Pool.Config().MaxConns {
|
||||
conn, err := db.Pool.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) {
|
||||
for _, binary := range []bool{false, true} {
|
||||
name := "metadata"
|
||||
requestTimeout := 40 * time.Millisecond
|
||||
var binaryTimeout time.Duration
|
||||
if binary {
|
||||
name = "upload"
|
||||
requestTimeout = time.Second
|
||||
binaryTimeout = 40 * time.Millisecond
|
||||
}
|
||||
held = append(held, conn)
|
||||
}
|
||||
defer func() {
|
||||
for _, conn := range held {
|
||||
conn.Release()
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
|
||||
defer cancel()
|
||||
req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx)
|
||||
req.Header.Set("Authorization", "Bearer "+actor.token)
|
||||
rec := httptest.NewRecorder()
|
||||
started := time.Now()
|
||||
ts.a.Engine.ServeHTTP(rec, req)
|
||||
if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
|
||||
t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
actor := ts.user("deadline_user", "User")
|
||||
ts.a = app.New(app.Deps{
|
||||
Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client,
|
||||
SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance,
|
||||
RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout,
|
||||
})
|
||||
var held []*pgxpool.Conn
|
||||
for range db.Pool.Config().MaxConns {
|
||||
conn, err := db.Pool.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held = append(held, conn)
|
||||
}
|
||||
defer func() {
|
||||
for _, conn := range held {
|
||||
conn.Release()
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
|
||||
defer cancel()
|
||||
path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111"
|
||||
req := httptest.NewRequest("GET", path, nil).WithContext(ctx)
|
||||
if binary {
|
||||
path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png"
|
||||
req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx)
|
||||
req.Header.Set("Content-Type", "image/png")
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+actor.token)
|
||||
rec := httptest.NewRecorder()
|
||||
started := time.Now()
|
||||
ts.a.Engine.ServeHTTP(rec, req)
|
||||
if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
|
||||
t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/uuid"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type FileStatus string
|
||||
|
||||
const (
|
||||
FilePending FileStatus = "pending"
|
||||
FileReady FileStatus = "ready"
|
||||
FileFailed FileStatus = "failed"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
@@ -11,12 +19,14 @@ type Attachment struct {
|
||||
Filename string
|
||||
FileHash []byte
|
||||
SizeBytes int64
|
||||
MIMEType, Status string
|
||||
MIMEType string
|
||||
Status FileStatus
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Repository interface {
|
||||
IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error)
|
||||
Publish(context.Context, Attachment, string) error
|
||||
GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error)
|
||||
type UploadInput struct {
|
||||
ChatID uuid.UUID
|
||||
Filename string
|
||||
MIMEType string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
@@ -16,10 +16,10 @@ type Object struct {
|
||||
}
|
||||
|
||||
type Blobs interface {
|
||||
Put(context.Context, string, io.Reader, int64, string) error
|
||||
Open(context.Context, string) (io.ReadCloser, error)
|
||||
Stat(context.Context, string) (Object, error)
|
||||
Copy(context.Context, string, string) error
|
||||
Delete(context.Context, string) error
|
||||
List(context.Context, string, string, int) ([]Object, error)
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error
|
||||
Open(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
Stat(ctx context.Context, key string) (Object, error)
|
||||
Copy(ctx context.Context, sourceKey, destinationKey string) error
|
||||
Delete(ctx context.Context, key string) error
|
||||
List(ctx context.Context, prefix, after string, limit int) ([]Object, error)
|
||||
}
|
||||
|
||||
@@ -8,40 +8,52 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/httpx"
|
||||
"messenger/internal/platform/ident"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Handler struct{ s *Service }
|
||||
|
||||
func NewHandler(s *Service) *Handler { return &Handler{s} }
|
||||
func (h *Handler) Register(protected, binary gin.IRouter) {
|
||||
binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
|
||||
func (h *Handler) Register(protected, binaryProtected gin.IRouter) {
|
||||
binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
|
||||
protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata))
|
||||
binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
|
||||
binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
// Lab4's shared response contract requires variants=[], even without processing.
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func toResponse(a Attachment) Response {
|
||||
return Response{
|
||||
ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
ID: a.ID.String(),
|
||||
ChatID: a.ChatID.String(),
|
||||
OwnerID: a.OwnerID.String(),
|
||||
Filename: a.Filename,
|
||||
ContentType: a.MIMEType,
|
||||
SizeBytes: a.SizeBytes,
|
||||
SHA256: hex.EncodeToString(a.FileHash),
|
||||
Status: string(a.Status),
|
||||
Variants: []any{},
|
||||
CreatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
UpdatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +77,14 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
|
||||
return
|
||||
}
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes)
|
||||
a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body)
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes)
|
||||
input := UploadInput{
|
||||
ChatID: id,
|
||||
Filename: names[0],
|
||||
MIMEType: media,
|
||||
SizeBytes: c.Request.ContentLength,
|
||||
}
|
||||
a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
@@ -109,7 +127,11 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
defer func() {
|
||||
if err := r.Close(); err != nil {
|
||||
slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err)
|
||||
}
|
||||
}()
|
||||
c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename}))
|
||||
c.Header("X-Content-Type-Options", "nosniff")
|
||||
c.Header("Content-Type", a.MIMEType)
|
||||
|
||||
@@ -45,6 +45,11 @@ type Reconciler struct {
|
||||
grace time.Duration
|
||||
}
|
||||
|
||||
type ReconcileOptions struct {
|
||||
Apply bool
|
||||
Deep bool
|
||||
}
|
||||
|
||||
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
|
||||
if db == nil || blobs == nil {
|
||||
return nil, errors.New("reconcile: database and blobs are required")
|
||||
@@ -61,10 +66,18 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci
|
||||
// Run streams findings and keeps at most one page of metadata in memory.
|
||||
// Dry runs only read storage. Each apply candidate is checked again under the
|
||||
// publisher's transaction lock; no scan snapshot authorizes a later mutation.
|
||||
func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) {
|
||||
func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) {
|
||||
started := time.Now()
|
||||
defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }()
|
||||
run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)}
|
||||
run := reconcileRun{
|
||||
r: r,
|
||||
ctx: ctx,
|
||||
apply: options.Apply,
|
||||
deep: options.Deep,
|
||||
emit: emit,
|
||||
stats: &stats,
|
||||
cutoff: started.Add(-r.grace),
|
||||
}
|
||||
for _, prefix := range []string{"staging/", "blobs/"} {
|
||||
if err = run.objects(prefix); err != nil {
|
||||
return stats, err
|
||||
@@ -203,7 +216,8 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
|
||||
return known, nil
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes)
|
||||
rows, err := s.r.db.Query(s.ctx, `
|
||||
SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -235,7 +249,8 @@ func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
|
||||
op.ctx = opCtx
|
||||
var exists bool
|
||||
s.stats.SQLQueries++
|
||||
if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil {
|
||||
if err := tx.QueryRow(opCtx, `
|
||||
SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
@@ -304,7 +319,7 @@ func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Fi
|
||||
type reconcileFile struct {
|
||||
hash []byte
|
||||
size int64
|
||||
status string
|
||||
status FileStatus
|
||||
created time.Time
|
||||
referenced bool
|
||||
}
|
||||
@@ -316,9 +331,13 @@ func (s *reconcileRun) files() error {
|
||||
return err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
|
||||
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
|
||||
FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after)
|
||||
rows, err := s.r.db.Query(s.ctx, `
|
||||
SELECT f.hash, f.size_bytes, f.status, f.created_at,
|
||||
EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
|
||||
FROM files f
|
||||
WHERE f.hash > $1
|
||||
ORDER BY f.hash
|
||||
LIMIT $2`, after, reconcilePageSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -346,30 +365,9 @@ func (s *reconcileRun) files() error {
|
||||
if !s.candidate(f) {
|
||||
continue
|
||||
}
|
||||
var finding *Finding
|
||||
if s.apply {
|
||||
finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
op := *s
|
||||
op.ctx = opCtx
|
||||
var current reconcileFile
|
||||
s.stats.SQLQueries++
|
||||
err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
|
||||
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
|
||||
FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash).
|
||||
Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.candidate(current) {
|
||||
return nil, nil
|
||||
}
|
||||
return op.inspect(current, tx)
|
||||
})
|
||||
} else {
|
||||
finding, err = s.inspect(f, nil)
|
||||
finding, err := s.inspect(f)
|
||||
if err == nil && s.apply && finding != nil && finding.Action != "report" {
|
||||
finding, err = s.repairFile(f.hash)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err)
|
||||
@@ -379,17 +377,66 @@ func (s *reconcileRun) files() error {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) {
|
||||
return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
op := *s
|
||||
op.ctx = opCtx
|
||||
var current reconcileFile
|
||||
s.stats.SQLQueries++
|
||||
err := tx.QueryRow(opCtx, `
|
||||
SELECT f.hash, f.size_bytes, f.status, f.created_at,
|
||||
EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
|
||||
FROM files f
|
||||
WHERE f.hash = $1
|
||||
FOR UPDATE OF f`, hash).
|
||||
Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !op.candidate(current) {
|
||||
return nil, nil
|
||||
}
|
||||
finding, err := op.inspect(current)
|
||||
if err != nil || finding == nil {
|
||||
return finding, err
|
||||
}
|
||||
switch finding.Action {
|
||||
case "delete_file":
|
||||
// Delete bytes first. A failed SQL commit leaves an unreferenced
|
||||
// missing row that the next pass can safely finish collecting.
|
||||
if err := op.delete(finding.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
_, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash)
|
||||
case "mark_failed":
|
||||
s.stats.SQLQueries++
|
||||
_, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed)
|
||||
default:
|
||||
return finding, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
return finding, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *reconcileRun) candidate(f reconcileFile) bool {
|
||||
if !s.old(f.created) {
|
||||
return false
|
||||
}
|
||||
if !f.referenced {
|
||||
return f.status == "pending" || f.status == "ready"
|
||||
return f.status == FilePending || f.status == FileReady
|
||||
}
|
||||
return f.status == "ready" || f.status == "failed"
|
||||
return f.status == FileReady || f.status == FileFailed
|
||||
}
|
||||
|
||||
func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
|
||||
func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) {
|
||||
key := BlobKey(f.hash)
|
||||
if !f.referenced {
|
||||
// A publication that rolled back can leave fresh bytes behind an old
|
||||
@@ -401,36 +448,14 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
|
||||
if err == nil && !s.old(obj.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"}
|
||||
if !s.apply {
|
||||
return finding, nil
|
||||
}
|
||||
// S3 must succeed first. A failed SQL commit then leaves a missing,
|
||||
// unreferenced row that the next run can safely finish collecting.
|
||||
if err := s.delete(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
return finding, nil
|
||||
return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil
|
||||
}
|
||||
obj, err := s.stat(key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
if f.status == "failed" {
|
||||
if f.status == FileFailed {
|
||||
return nil, nil
|
||||
}
|
||||
finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}
|
||||
if s.apply {
|
||||
s.stats.SQLQueries++
|
||||
if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
}
|
||||
return finding, nil
|
||||
return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -449,7 +474,7 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
|
||||
}
|
||||
// A failed file stays failed even when the key reappears. Recovery requires
|
||||
// an explicit publisher; a shallow HEAD cannot prove content integrity.
|
||||
if f.status == "failed" {
|
||||
if f.status == FileFailed {
|
||||
return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil
|
||||
}
|
||||
return nil, nil
|
||||
|
||||
@@ -209,7 +209,8 @@ func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
var found []attachment.Finding
|
||||
stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) })
|
||||
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
|
||||
stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) })
|
||||
return stats, found, err
|
||||
}
|
||||
|
||||
@@ -221,6 +222,28 @@ func TestReconciler(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
t.Run("healthy apply does not block publication", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("healthy", "ready", 2*time.Hour, true, true)
|
||||
tx, err := db.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if err := attachment.LockHash(ctx, tx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := attachment.NewReconciler(f.db, f.b, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
|
||||
if err != nil || stats.Findings != 0 || stats.Repaired != 0 {
|
||||
t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("constructor validates and defaults grace", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
for _, grace := range []time.Duration{-time.Second, time.Minute} {
|
||||
@@ -361,7 +384,7 @@ func TestReconciler(t *testing.T) {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
r, _ := attachment.NewReconciler(f.db, f.b, 0)
|
||||
_, err := r.Run(ctx, true, false, nil)
|
||||
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
done <- err
|
||||
}()
|
||||
waitReconcileLock(t, ctx, f.db)
|
||||
@@ -395,7 +418,7 @@ func TestReconciler(t *testing.T) {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
r, _ := attachment.NewReconciler(f.db, f.b, 0)
|
||||
_, err := r.Run(ctx, true, false, nil)
|
||||
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
done <- err
|
||||
}()
|
||||
waitReconcileLock(t, ctx, f.db)
|
||||
@@ -592,7 +615,7 @@ func TestReconcilerFailureBoundaries(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats, err := r.Run(ctx, false, true, nil)
|
||||
stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil)
|
||||
if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" {
|
||||
t.Fatalf("read only dry run: %+v %v", stats, err)
|
||||
}
|
||||
@@ -681,11 +704,11 @@ func TestReconcilerS3(t *testing.T) {
|
||||
t.Fatalf("CLI mutated by default: %+v %v", cliStats, err)
|
||||
}
|
||||
var found []attachment.Finding
|
||||
stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) })
|
||||
stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) })
|
||||
if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" {
|
||||
t.Fatalf("S3 dry: %+v %v %+v", stats, err, found)
|
||||
}
|
||||
stats, err = r.Run(ctx, true, true, nil)
|
||||
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
|
||||
if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" {
|
||||
t.Fatalf("S3 apply: %+v %v", stats, err)
|
||||
}
|
||||
@@ -695,7 +718,7 @@ func TestReconcilerS3(t *testing.T) {
|
||||
if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil {
|
||||
t.Fatal("fresh staging deleted", err)
|
||||
}
|
||||
stats, err = r.Run(ctx, true, true, nil)
|
||||
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
|
||||
if err != nil || stats.Repaired != 0 {
|
||||
t.Fatalf("S3 repeat: %+v %v", stats, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type AttachmentRepository interface {
|
||||
// WithPublication holds the file's hash lock through the callback and commit.
|
||||
// Publication is valid only inside the callback. Any callback error rolls back.
|
||||
WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error
|
||||
GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error)
|
||||
}
|
||||
|
||||
type Publication interface {
|
||||
FileReady() bool
|
||||
SaveReady(ctx context.Context) error
|
||||
}
|
||||
|
||||
type ChatMembership interface {
|
||||
IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error)
|
||||
}
|
||||
@@ -15,40 +15,54 @@ import (
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
repo Repository
|
||||
blobs Blobs
|
||||
maxBytes, maxPixels int64
|
||||
decode chan struct{}
|
||||
attachments AttachmentRepository
|
||||
chats ChatMembership
|
||||
blobs Blobs
|
||||
config Config
|
||||
decode chan struct{}
|
||||
}
|
||||
|
||||
func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service {
|
||||
if maxBytes == 0 {
|
||||
maxBytes = MaxUploadBytes
|
||||
}
|
||||
if maxPixels == 0 {
|
||||
maxPixels = MaxImagePixels
|
||||
}
|
||||
return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)}
|
||||
type Config struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
}
|
||||
|
||||
func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) {
|
||||
if err := ValidateFilename(filename); err != nil {
|
||||
// A nil blobs dependency allows the text API to run without configured S3.
|
||||
// Binary operations report unavailability in that mode.
|
||||
func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service {
|
||||
if cfg.MaxBytes == 0 {
|
||||
cfg.MaxBytes = MaxUploadBytes
|
||||
}
|
||||
if cfg.MaxPixels == 0 {
|
||||
cfg.MaxPixels = MaxImagePixels
|
||||
}
|
||||
return &Service{
|
||||
attachments: attachments,
|
||||
chats: chats,
|
||||
blobs: blobs,
|
||||
config: cfg,
|
||||
decode: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) {
|
||||
if err := ValidateFilename(input.Filename); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if mime != "image/png" && mime != "image/jpeg" {
|
||||
if _, accepted := imageFormats[input.MIMEType]; !accepted {
|
||||
return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted")
|
||||
}
|
||||
ok, err := s.repo.IsMember(ctx, chatID, actor)
|
||||
ok, err := s.chats.IsMember(ctx, input.ChatID, actor)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if !ok {
|
||||
return Attachment{}, apperr.NotFound("chat not found")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
if input.SizeBytes > s.config.MaxBytes {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if size <= 0 {
|
||||
if input.SizeBytes <= 0 {
|
||||
return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required")
|
||||
}
|
||||
if s.blobs == nil {
|
||||
@@ -59,16 +73,17 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
// A failed cleanup leaves a staging object for the reconciler's grace-based collection.
|
||||
if err := s.blobs.Delete(cleanup, stage); err != nil {
|
||||
slog.Warn("staging cleanup deferred", "error", err)
|
||||
slog.Warn("staging cleanup deferred", "key", stage, "error", err)
|
||||
}
|
||||
}()
|
||||
hash := sha256.New()
|
||||
counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)}
|
||||
err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime)
|
||||
counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)}
|
||||
err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType)
|
||||
if err != nil {
|
||||
_, tooLarge := errors.AsType[*http.MaxBytesError](err)
|
||||
if counter.n > s.maxBytes || tooLarge {
|
||||
if counter.n > s.config.MaxBytes || tooLarge {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
@@ -78,10 +93,10 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
|
||||
}
|
||||
var extra [1]byte
|
||||
n, readErr := counter.Read(extra[:])
|
||||
if counter.n > s.maxBytes {
|
||||
if counter.n > s.config.MaxBytes {
|
||||
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
|
||||
}
|
||||
if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) {
|
||||
if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) {
|
||||
return Attachment{}, apperr.Invalid("body length does not match Content-Length")
|
||||
}
|
||||
// Full decoding is bounded by dimensions and one decoder per replica. Body
|
||||
@@ -91,26 +106,64 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
|
||||
case <-ctx.Done():
|
||||
return Attachment{}, unavailable(ctx.Err())
|
||||
}
|
||||
err = s.validate(ctx, stage, mime)
|
||||
err = s.validate(ctx, stage, input.MIMEType)
|
||||
<-s.decode
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)}
|
||||
if err := s.repo.Publish(ctx, a, stage); err != nil {
|
||||
a := Attachment{
|
||||
ID: id,
|
||||
ChatID: input.ChatID,
|
||||
OwnerID: actor,
|
||||
Filename: input.Filename,
|
||||
FileHash: hash.Sum(nil),
|
||||
SizeBytes: input.SizeBytes,
|
||||
MIMEType: input.MIMEType,
|
||||
Status: FileReady,
|
||||
CreatedAt: ident.TimeOf(id),
|
||||
}
|
||||
if err := s.attachments.WithPublication(ctx, a, func(p Publication) error {
|
||||
if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.SaveReady(ctx)
|
||||
}); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *Service) validate(ctx context.Context, stage, mime string) error {
|
||||
func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error {
|
||||
blobKey := BlobKey(a.FileHash)
|
||||
if fileReady {
|
||||
obj, err := s.blobs.Stat(ctx, blobKey)
|
||||
if err == nil && obj.Size == a.SizeBytes {
|
||||
return nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrObjectMissing) {
|
||||
return unavailable(err)
|
||||
}
|
||||
}
|
||||
// The hash is known after streaming. Copy the validated staging object to
|
||||
// its permanent CAS key while the repository's publication lock is held.
|
||||
if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil {
|
||||
return unavailable(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) validate(ctx context.Context, stage, mime string) (err error) {
|
||||
r, err := s.blobs.Open(ctx, stage)
|
||||
if err != nil {
|
||||
return unavailable(err)
|
||||
}
|
||||
defer r.Close()
|
||||
defer func() {
|
||||
if closeErr := r.Close(); closeErr != nil && err == nil {
|
||||
err = unavailable(closeErr)
|
||||
}
|
||||
}()
|
||||
tracked := &storageReader{reader: r}
|
||||
err = ValidateImage(tracked, mime, s.maxPixels)
|
||||
err = ValidateImage(tracked, mime, s.config.MaxPixels)
|
||||
if tracked.err != nil {
|
||||
return unavailable(tracked.err)
|
||||
}
|
||||
@@ -136,11 +189,11 @@ func (r *storageReader) Read(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
|
||||
a, err := s.repo.GetForMember(ctx, id, actor)
|
||||
a, err := s.attachments.GetForMember(ctx, id, actor)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if a.Status != "ready" {
|
||||
if a.Status != FileReady {
|
||||
return Attachment{}, apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
return a, nil
|
||||
@@ -166,7 +219,12 @@ type countReader struct {
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e }
|
||||
func (c *countReader) Read(b []byte) (int, error) {
|
||||
n, err := c.r.Read(b)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func unavailable(err error) error {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/attachment/store"
|
||||
chatstore "messenger/internal/chat/store"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/testenv"
|
||||
@@ -73,10 +74,11 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0)
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
|
||||
uploaded := make(chan error, 1)
|
||||
go func() {
|
||||
_, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data))
|
||||
input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
_, e := s.Upload(ctx, actor, input, bytes.NewReader(data))
|
||||
uploaded <- e
|
||||
}()
|
||||
select {
|
||||
@@ -90,7 +92,7 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
|
||||
}
|
||||
collected := make(chan error, 1)
|
||||
go func() {
|
||||
stats, e := r.Run(ctx, true, false, nil)
|
||||
stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
if e == nil && stats.Repaired != 0 {
|
||||
e = errors.New("collector removed a newly referenced file")
|
||||
}
|
||||
@@ -196,7 +198,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
}
|
||||
}
|
||||
b := &testBlobs{data: map[string][]byte{}}
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels)
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -210,7 +212,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
c = c2
|
||||
}
|
||||
var e error
|
||||
results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data))
|
||||
input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data))
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
@@ -235,7 +238,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
b.mu.Lock()
|
||||
clear(b.data)
|
||||
b.mu.Unlock()
|
||||
if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil {
|
||||
input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.copies != 2 {
|
||||
@@ -244,7 +248,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
b.failCopy = true
|
||||
other := bytes.Clone(data)
|
||||
other = append(other, 0)
|
||||
if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
input.Filename, input.SizeBytes = "failed.png", int64(len(other))
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("copy outage: %v", err)
|
||||
}
|
||||
if len(b.data) != 1 {
|
||||
@@ -258,7 +263,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil {
|
||||
input.Filename = "rollback.png"
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil {
|
||||
t.Fatal("publication survived rejected metadata")
|
||||
}
|
||||
if len(b.data) != 2 {
|
||||
|
||||
@@ -4,32 +4,29 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
)
|
||||
|
||||
type Attachments struct {
|
||||
pool *pgxpool.Pool
|
||||
blobs attachment.Blobs
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments {
|
||||
return &Attachments{pool, blobs}
|
||||
}
|
||||
var _ attachment.AttachmentRepository = (*Attachments)(nil)
|
||||
|
||||
func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) {
|
||||
var ok bool
|
||||
err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok)
|
||||
return ok, postgres.Classify(err)
|
||||
func NewAttachments(pool *pgxpool.Pool) *Attachments {
|
||||
return &Attachments{pool: pool}
|
||||
}
|
||||
|
||||
// Publication and garbage collection use the same hash lock. The transaction
|
||||
// spans only publication, never client transfer or image decoding.
|
||||
func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error {
|
||||
func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error {
|
||||
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
|
||||
return err
|
||||
@@ -37,41 +34,56 @@ func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stag
|
||||
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
|
||||
return err
|
||||
}
|
||||
var status string
|
||||
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status)
|
||||
var status attachment.FileStatus
|
||||
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
copyNeeded := status != "ready"
|
||||
if !copyNeeded {
|
||||
obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash))
|
||||
if errors.Is(e, attachment.ErrObjectMissing) {
|
||||
copyNeeded = true
|
||||
} else if e != nil {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e)
|
||||
} else if obj.Size != a.SizeBytes {
|
||||
copyNeeded = true
|
||||
}
|
||||
}
|
||||
if copyNeeded {
|
||||
if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil {
|
||||
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
|
||||
}
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish file: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
|
||||
return err
|
||||
return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady})
|
||||
}))
|
||||
}
|
||||
|
||||
type publication struct {
|
||||
tx pgx.Tx
|
||||
attachment attachment.Attachment
|
||||
ready bool
|
||||
}
|
||||
|
||||
func (p publication) FileReady() bool { return p.ready }
|
||||
|
||||
func (p publication) SaveReady(ctx context.Context) error {
|
||||
a := p.attachment
|
||||
_, err := p.tx.Exec(ctx, `
|
||||
INSERT INTO files (hash, size_bytes, mime_type, status)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (hash) DO UPDATE
|
||||
SET size_bytes = excluded.size_bytes,
|
||||
mime_type = excluded.mime_type,
|
||||
status = excluded.status`,
|
||||
a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish file: %w", err)
|
||||
}
|
||||
_, err = p.tx.Exec(ctx, `
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
|
||||
var a attachment.Attachment
|
||||
err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at
|
||||
FROM attachments a JOIN files f ON f.hash=a.file_hash
|
||||
WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash,
|
||||
f.size_bytes, f.mime_type, f.status, a.created_at
|
||||
FROM attachments a
|
||||
JOIN files f ON f.hash = a.file_hash
|
||||
WHERE a.id = $1 AND EXISTS (
|
||||
SELECT 1 FROM chat_members m
|
||||
WHERE m.chat_id = a.chat_id AND m.user_id = $2
|
||||
)`, id, actor).
|
||||
Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash,
|
||||
&a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return attachment.Attachment{}, apperr.NotFound("attachment not found")
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ import (
|
||||
const MaxUploadBytes int64 = 10 * 1024 * 1024
|
||||
const MaxImagePixels int64 = 40_000_000
|
||||
|
||||
var imageFormats = map[string]string{
|
||||
"image/png": "png",
|
||||
"image/jpeg": "jpeg",
|
||||
}
|
||||
|
||||
func ValidateFilename(name string) error {
|
||||
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 {
|
||||
return apperr.Invalid("filename must contain 1-255 characters")
|
||||
@@ -37,7 +42,7 @@ func ValidateImage(r io.Reader, mime string, maxPixels int64) error {
|
||||
if err != nil {
|
||||
return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err)
|
||||
}
|
||||
if (format != "png" && format != "jpeg") || "image/"+format != mime {
|
||||
if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat {
|
||||
return apperr.UnsupportedMedia("image does not match Content-Type")
|
||||
}
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) {
|
||||
|
||||
@@ -8,11 +8,44 @@ import (
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"messenger/internal/platform/apperr"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/platform/apperr"
|
||||
)
|
||||
|
||||
type chatMember struct {
|
||||
chatID, userID uuid.UUID
|
||||
}
|
||||
|
||||
func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) {
|
||||
return chatID == m.chatID && userID == m.userID, nil
|
||||
}
|
||||
|
||||
func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) {
|
||||
member := chatMember{chatID: uuid.New(), userID: uuid.New()}
|
||||
s := NewService(nil, member, nil, Config{})
|
||||
input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
actor uuid.UUID
|
||||
kind apperr.Kind
|
||||
}{
|
||||
{name: "member", actor: member.userID, kind: apperr.KindUnavailable},
|
||||
{name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data"))
|
||||
if apperr.KindOf(err) != tc.kind {
|
||||
t.Fatalf("got %v, want %v", err, tc.kind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFilename(t *testing.T) {
|
||||
for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} {
|
||||
if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid {
|
||||
@@ -41,16 +74,35 @@ type openedBlob struct {
|
||||
}
|
||||
|
||||
func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) {
|
||||
if reader, ok := b.reader.(io.ReadCloser); ok {
|
||||
return reader, nil
|
||||
}
|
||||
return io.NopCloser(b.reader), nil
|
||||
}
|
||||
|
||||
type closeErrorBody struct{ io.Reader }
|
||||
|
||||
func (closeErrorBody) Close() error { return errors.New("injected close failure") }
|
||||
|
||||
func TestStagingCloseFailurePreventsPublication(t *testing.T) {
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}}
|
||||
s := NewService(nil, nil, blobs, Config{})
|
||||
if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("close failure classified as %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) {
|
||||
var pngData bytes.Buffer
|
||||
if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} {
|
||||
s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0)
|
||||
s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{})
|
||||
err := s.validate(context.Background(), "staging/test", "image/png")
|
||||
if apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err)
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
@@ -202,7 +203,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
|
||||
if err != nil {
|
||||
return fmt.Errorf("check attachments: %w", err)
|
||||
}
|
||||
statuses, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus])
|
||||
if err != nil {
|
||||
return fmt.Errorf("read attachment status: %w", err)
|
||||
}
|
||||
@@ -210,7 +211,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
|
||||
return apperr.NotFound("attachment not found")
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status != "ready" {
|
||||
if status != attachment.FileReady {
|
||||
return apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ func Load(lookup Lookup) (Config, error) {
|
||||
}
|
||||
cfg.DebugAddr = v
|
||||
}
|
||||
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
|
||||
if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil {
|
||||
@@ -154,12 +154,16 @@ func LoadMigrate(lookup Lookup) (Migrate, error) {
|
||||
if cfg.Logging, err = loadLogging(lookup); err != nil {
|
||||
return Migrate{}, err
|
||||
}
|
||||
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
|
||||
if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
|
||||
return Migrate{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func LoadDatabaseURL(lookup Lookup) (string, error) {
|
||||
return secret(lookup, "DATABASE_URL_FILE")
|
||||
}
|
||||
|
||||
func loadLogging(lookup Lookup) (Logging, error) {
|
||||
l := Logging{Env: "lab", Level: slog.LevelInfo}
|
||||
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
@@ -85,7 +86,7 @@ func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) {
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -126,7 +127,7 @@ func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) {
|
||||
io.WriteString(w, `<Error><Code>ServiceUnavailable</Code><Message>volume not registered</Message></Error>`)
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/emptypb"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
@@ -149,7 +150,7 @@ func TestSeaweedStore(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} {
|
||||
outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1])
|
||||
outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -219,7 +220,7 @@ func TestSeaweedStore(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey)
|
||||
missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -16,10 +16,13 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/config"
|
||||
)
|
||||
|
||||
const MaxPageSize = 500
|
||||
|
||||
const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
|
||||
|
||||
type Store struct {
|
||||
client *minio.Core
|
||||
region string
|
||||
@@ -30,15 +33,15 @@ var _ attachment.Blobs = (*Store)(nil)
|
||||
|
||||
// New validates configuration without contacting S3, so text-only service startup
|
||||
// and readiness do not depend on object storage being reachable.
|
||||
func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) {
|
||||
u, err := url.Parse(endpoint)
|
||||
func New(cfg config.S3) (*Store, error) {
|
||||
u, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
|
||||
return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment")
|
||||
}
|
||||
if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" {
|
||||
if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
|
||||
return nil, errors.New("s3: region and credentials are required")
|
||||
}
|
||||
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
|
||||
if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil {
|
||||
return nil, fmt.Errorf("s3: invalid bucket: %w", err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
@@ -56,9 +59,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
|
||||
DisableCompression: true,
|
||||
}
|
||||
client, err := minio.NewCore(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
Region: region,
|
||||
Region: cfg.Region,
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
Transport: transport,
|
||||
// A consumed upload stream cannot be replayed. Let the caller decide
|
||||
@@ -68,7 +71,7 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: client: %w", err)
|
||||
}
|
||||
return &Store{client: client, region: region, bucket: bucket}, nil
|
||||
return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
|
||||
// Bootstrap is used only by the admin init command, never by API startup.
|
||||
@@ -78,7 +81,7 @@ func (s *Store) Bootstrap(ctx context.Context) error {
|
||||
}
|
||||
err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region})
|
||||
code := minio.ToErrorResponse(err).Code
|
||||
if err != nil && code != "BucketAlreadyOwnedByYou" {
|
||||
if err != nil && code != codeBucketAlreadyOwnedByYou {
|
||||
return wrap("create bucket", err)
|
||||
}
|
||||
return s.WaitReady(ctx)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
@@ -18,7 +19,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) }))
|
||||
defer server.Close()
|
||||
if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil {
|
||||
if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 0 {
|
||||
@@ -36,7 +37,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
|
||||
{"secret", server.URL, "us-east-1", "attachments", "access", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil {
|
||||
if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil {
|
||||
t.Fatal("accepted invalid configuration")
|
||||
}
|
||||
})
|
||||
@@ -69,7 +70,7 @@ func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) {
|
||||
w.Header().Set("ETag", `"probe"`)
|
||||
}))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -103,7 +104,7 @@ func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) }))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -154,7 +155,7 @@ func TestListStopsAfterOneBoundedPage(t *testing.T) {
|
||||
io.WriteString(w, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>attachments</Name><IsTruncated>true</IsTruncated><NextContinuationToken>next-page</NextContinuationToken><Contents><Key>blobs/b</Key><Size>1</Size><LastModified>2026-01-01T00:00:00Z</LastModified></Contents><Contents><Key>blobs/c</Key><Size>2</Size><LastModified>2026-01-01T00:00:01Z</LastModified></Contents></ListBucketResult>`)
|
||||
}))
|
||||
defer server.Close()
|
||||
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
|
||||
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/s3store"
|
||||
)
|
||||
|
||||
@@ -131,15 +132,15 @@ func (s *S3) launch(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey)
|
||||
s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey)
|
||||
s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey)
|
||||
s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -119,7 +119,8 @@ func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string,
|
||||
}
|
||||
}()
|
||||
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
|
||||
stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
|
||||
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
|
||||
stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
|
||||
close(done)
|
||||
m.GoHeapPeakBytes = <-peakCh
|
||||
m.Stats = stats
|
||||
|
||||
Reference in New Issue
Block a user