Доработайте публикацию и сверку вложений по итогам ревью

This commit is contained in:
2026-11-18 11:43:27 +03:00
parent 1f06d9f4ab
commit 80c4a56200
27 changed files with 605 additions and 275 deletions
+2 -2
View File
@@ -78,7 +78,7 @@ Grace period — 1 h, минимум 2 min, выше бюджета upload. Св
| Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. |
| Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. |
Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready.
Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает отсутствующий blob и файл в failed. Для ready с существующим объектом того же размера действует быстрый Stat-путь. Deep-повреждение такого ready диагностируется и требует отдельного восстановления из достоверных байтов. Простое появление объекта само не меняет failed на ready.
```sh
make reconcile
@@ -136,7 +136,7 @@ Bootstrap — одноразовый `s3-init` с отдельным admin-кл
| --- | ---: | ---: | --- |
| Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart |
| Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart |
| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart |
| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.2 | 201, checksum совпал, без OOM/restart |
Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки.
+5 -4
View File
@@ -43,7 +43,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m")
return 2
}
dbConfig, err := config.LoadMigrate(os.LookupEnv)
databaseURL, err := config.LoadDatabaseURL(os.LookupEnv)
if err != nil {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
@@ -54,12 +54,12 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
}
blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey)
blobs, err := s3store.New(s3Config)
if err != nil {
fmt.Fprintln(errOut, "configuration error:", err)
return 2
}
db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10)
db, err := postgres.NewPool(ctx, databaseURL, 10)
if err != nil {
fmt.Fprintln(errOut, "reconcile:", err)
return 1
@@ -76,7 +76,8 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
defer cancel()
encoder := json.NewEncoder(out)
var outputErr error
stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) {
options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep}
stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) {
if outputErr == nil {
outputErr = encoder.Encode(f)
if outputErr != nil {
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) {
for key, value := range map[string]string{
"DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable",
"S3_ACCESS_KEY_ID_FILE": "test-access",
"S3_SECRET_ACCESS_KEY_FILE": "test-secret",
} {
path := filepath.Join(t.TempDir(), key)
if err := os.WriteFile(path, []byte(value), 0600); err != nil {
t.Fatal(err)
}
t.Setenv(key, path)
}
t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333")
t.Setenv("S3_REGION", "us-east-1")
t.Setenv("S3_BUCKET", "attachments")
t.Setenv("APP_LOG_LEVEL", "not-a-log-level")
ctx, cancel := context.WithCancel(context.Background())
cancel() // Stop before dialing: only configuration relevant to this CLI is checked.
var out, errOut bytes.Buffer
code := run(ctx, nil, &out, &errOut)
if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") {
t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String())
}
}
+4 -3
View File
@@ -3,10 +3,11 @@ package main
import (
"context"
"fmt"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
"os"
"time"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
func main() {
@@ -20,7 +21,7 @@ func run() error {
if err != nil {
return err
}
s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey)
s, err := s3store.New(cfg)
if err != nil {
return err
}
+32 -23
View File
@@ -1,4 +1,4 @@
// Command server starts the messenger API over PostgreSQL and Valkey.
// Command server starts the messenger API over PostgreSQL, Valkey and S3.
package main
import (
@@ -61,25 +61,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return err
}
defer sessions.Close()
var blobs attachment.Blobs
var reconciler *attachment.Reconciler
if cfg.S3.Endpoint != "" {
blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey)
if err != nil {
return err
}
reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv))
if e != nil {
return e
}
reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey)
if e != nil {
return e
}
reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour)
if err != nil {
return err
}
blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv)
if err != nil {
return err
}
ln, err := net.Listen("tcp", cfg.Addr())
@@ -100,8 +84,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
InstanceID: alias,
IdempotencyTTL: cfg.IdempotencyTTL,
Blobs: blobs,
MaxUploadBytes: cfg.S3.MaxBytes,
MaxImagePixels: cfg.S3.MaxPixels,
AttachmentsCfg: attachment.Config{
MaxBytes: cfg.S3.MaxBytes,
MaxPixels: cfg.S3.MaxPixels,
},
})
var debugLn net.Listener
if cfg.DebugAddr != "" {
@@ -124,7 +110,7 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return
case <-ticker.C:
scan, cancel := context.WithTimeout(ctx, 2*time.Minute)
stats, err := reconciler.Run(scan, true, false, nil)
stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil)
cancel()
if err != nil {
log.Warn("attachment cleanup incomplete", "error", err)
@@ -149,6 +135,29 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
return err
}
func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) {
if cfg.Endpoint == "" {
return nil, nil, nil
}
blobs, err := s3store.New(cfg)
if err != nil {
return nil, nil, err
}
reconCfg, err := config.LoadS3(config.ReconcileS3(lookup))
if err != nil {
return nil, nil, err
}
reconBlobs, err := s3store.New(reconCfg)
if err != nil {
return nil, nil, err
}
reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour)
if err != nil {
return nil, nil, err
}
return blobs, reconciler, nil
}
// publishDebugVars adds to expvar what the load tests read from a replica.
func publishDebugVars(alias string, pool *pgxpool.Pool) {
expvar.NewString("instance").Set(alias)
+3
View File
@@ -6,6 +6,9 @@ name: lab4
services:
lb:
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
depends_on:
api:
condition: service_healthy
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
+20 -10
View File
@@ -39,20 +39,23 @@ type Deps struct {
Sessions *redis.Client
SessionTTL time.Duration
MaxBodyBytes int64
// RequestTimeout bounds every API request; zero means the default.
RequestTimeout time.Duration
InstanceID string
// Zero timeouts use the defaults below.
RequestTimeout time.Duration
BinaryRequestTimeout time.Duration
InstanceID string
// IdempotencyTTL zero means the contract's 24 hours.
IdempotencyTTL time.Duration
Blobs attachment.Blobs
MaxUploadBytes int64
MaxImagePixels int64
AttachmentsCfg attachment.Config
}
// defaultRequestTimeout stays below the server's 10s write timeout, so a
// defaultRequestTimeout stays below the server's write timeout, so a
// request that gives up on a dependency can still write its 503.
const defaultRequestTimeout = 5 * time.Second
// Binary transfers and decoding share this budget, below the 35s socket timeout.
const defaultBinaryRequestTimeout = 30 * time.Second
const defaultIdempotencyTTL = 24 * time.Hour
func New(d Deps) *App {
@@ -61,7 +64,8 @@ func New(d Deps) *App {
if keyTTL == 0 {
keyTTL = defaultIdempotencyTTL
}
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL)
chatRepository := chatstore.NewChats(d.DB)
chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
probes := health.New(
@@ -81,9 +85,15 @@ func New(d Deps) *App {
auth.NewHandler(sessions).Register(public, protected)
user.NewHandler(users).Register(public, protected)
chat.NewHandler(chats).Register(protected)
attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels)
binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions))
attachment.NewHandler(attachments).Register(protected, binary)
attachments := attachment.NewService(
attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg,
)
binaryTimeout := d.BinaryRequestTimeout
if binaryTimeout == 0 {
binaryTimeout = defaultBinaryRequestTimeout
}
binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions))
attachment.NewHandler(attachments).Register(protected, binaryProtected)
return &App{Engine: engine, Health: probes, Chats: chats}
}
+47 -25
View File
@@ -30,32 +30,54 @@ func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) {
ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized")
}
func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) {
ts := newTester(t)
actor := ts.user("deadline_user", "User")
ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond})
var held []*pgxpool.Conn
for range db.Pool.Config().MaxConns {
conn, err := db.Pool.Acquire(context.Background())
if err != nil {
t.Fatal(err)
func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) {
for _, binary := range []bool{false, true} {
name := "metadata"
requestTimeout := 40 * time.Millisecond
var binaryTimeout time.Duration
if binary {
name = "upload"
requestTimeout = time.Second
binaryTimeout = 40 * time.Millisecond
}
held = append(held, conn)
}
defer func() {
for _, conn := range held {
conn.Release()
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
defer cancel()
req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx)
req.Header.Set("Authorization", "Bearer "+actor.token)
rec := httptest.NewRecorder()
started := time.Now()
ts.a.Engine.ServeHTTP(rec, req)
if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
t.Run(name, func(t *testing.T) {
ts := newTester(t)
actor := ts.user("deadline_user", "User")
ts.a = app.New(app.Deps{
Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client,
SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance,
RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout,
})
var held []*pgxpool.Conn
for range db.Pool.Config().MaxConns {
conn, err := db.Pool.Acquire(context.Background())
if err != nil {
t.Fatal(err)
}
held = append(held, conn)
}
defer func() {
for _, conn := range held {
conn.Release()
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
defer cancel()
path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111"
req := httptest.NewRequest("GET", path, nil).WithContext(ctx)
if binary {
path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png"
req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx)
req.Header.Set("Content-Type", "image/png")
}
req.Header.Set("Authorization", "Bearer "+actor.token)
rec := httptest.NewRecorder()
started := time.Now()
ts.a.Engine.ServeHTTP(rec, req)
if rec.Code != 503 || time.Since(started) > 200*time.Millisecond {
t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String())
}
})
}
}
+17 -7
View File
@@ -1,9 +1,17 @@
package attachment
import (
"context"
"github.com/google/uuid"
"time"
"github.com/google/uuid"
)
type FileStatus string
const (
FilePending FileStatus = "pending"
FileReady FileStatus = "ready"
FileFailed FileStatus = "failed"
)
type Attachment struct {
@@ -11,12 +19,14 @@ type Attachment struct {
Filename string
FileHash []byte
SizeBytes int64
MIMEType, Status string
MIMEType string
Status FileStatus
CreatedAt time.Time
}
type Repository interface {
IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error)
Publish(context.Context, Attachment, string) error
GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error)
type UploadInput struct {
ChatID uuid.UUID
Filename string
MIMEType string
SizeBytes int64
}
+6 -6
View File
@@ -16,10 +16,10 @@ type Object struct {
}
type Blobs interface {
Put(context.Context, string, io.Reader, int64, string) error
Open(context.Context, string) (io.ReadCloser, error)
Stat(context.Context, string) (Object, error)
Copy(context.Context, string, string) error
Delete(context.Context, string) error
List(context.Context, string, string, int) ([]Object, error)
Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error
Open(ctx context.Context, key string) (io.ReadCloser, error)
Stat(ctx context.Context, key string) (Object, error)
Copy(ctx context.Context, sourceKey, destinationKey string) error
Delete(ctx context.Context, key string) error
List(ctx context.Context, prefix, after string, limit int) ([]Object, error)
}
+43 -21
View File
@@ -8,40 +8,52 @@ import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"messenger/internal/platform/apperr"
"messenger/internal/platform/httpx"
"messenger/internal/platform/ident"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
type Handler struct{ s *Service }
func NewHandler(s *Service) *Handler { return &Handler{s} }
func (h *Handler) Register(protected, binary gin.IRouter) {
binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
func (h *Handler) Register(protected, binaryProtected gin.IRouter) {
binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload))
protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata))
binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content))
}
type Response struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
// Lab4's shared response contract requires variants=[], even without processing.
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
func toResponse(a Attachment) Response {
return Response{
ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt),
ID: a.ID.String(),
ChatID: a.ChatID.String(),
OwnerID: a.OwnerID.String(),
Filename: a.Filename,
ContentType: a.MIMEType,
SizeBytes: a.SizeBytes,
SHA256: hex.EncodeToString(a.FileHash),
Status: string(a.Status),
Variants: []any{},
CreatedAt: httpx.Timestamp(a.CreatedAt),
UpdatedAt: httpx.Timestamp(a.CreatedAt),
}
}
@@ -65,8 +77,14 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
return
}
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes)
a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body)
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes)
input := UploadInput{
ChatID: id,
Filename: names[0],
MIMEType: media,
SizeBytes: c.Request.ContentLength,
}
a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body)
if err != nil {
httpx.Error(c, err)
return
@@ -109,7 +127,11 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
defer r.Close()
defer func() {
if err := r.Close(); err != nil {
slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err)
}
}()
c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename}))
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Type", a.MIMEType)
+86 -61
View File
@@ -45,6 +45,11 @@ type Reconciler struct {
grace time.Duration
}
type ReconcileOptions struct {
Apply bool
Deep bool
}
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
if db == nil || blobs == nil {
return nil, errors.New("reconcile: database and blobs are required")
@@ -61,10 +66,18 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci
// Run streams findings and keeps at most one page of metadata in memory.
// Dry runs only read storage. Each apply candidate is checked again under the
// publisher's transaction lock; no scan snapshot authorizes a later mutation.
func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) {
func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) {
started := time.Now()
defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }()
run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)}
run := reconcileRun{
r: r,
ctx: ctx,
apply: options.Apply,
deep: options.Deep,
emit: emit,
stats: &stats,
cutoff: started.Add(-r.grace),
}
for _, prefix := range []string{"staging/", "blobs/"} {
if err = run.objects(prefix); err != nil {
return stats, err
@@ -203,7 +216,8 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
return known, nil
}
s.stats.SQLQueries++
rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes)
rows, err := s.r.db.Query(s.ctx, `
SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes)
if err != nil {
return nil, err
}
@@ -235,7 +249,8 @@ func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
op.ctx = opCtx
var exists bool
s.stats.SQLQueries++
if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil {
if err := tx.QueryRow(opCtx, `
SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil {
return nil, err
}
if exists {
@@ -304,7 +319,7 @@ func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Fi
type reconcileFile struct {
hash []byte
size int64
status string
status FileStatus
created time.Time
referenced bool
}
@@ -316,9 +331,13 @@ func (s *reconcileRun) files() error {
return err
}
s.stats.SQLQueries++
rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after)
rows, err := s.r.db.Query(s.ctx, `
SELECT f.hash, f.size_bytes, f.status, f.created_at,
EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
FROM files f
WHERE f.hash > $1
ORDER BY f.hash
LIMIT $2`, after, reconcilePageSize)
if err != nil {
return err
}
@@ -346,30 +365,9 @@ func (s *reconcileRun) files() error {
if !s.candidate(f) {
continue
}
var finding *Finding
if s.apply {
finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
op := *s
op.ctx = opCtx
var current reconcileFile
s.stats.SQLQueries++
err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at,
EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash)
FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash).
Scan(&current.hash, &current.size, &current.status, &current.created, &current.referenced)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
if !s.candidate(current) {
return nil, nil
}
return op.inspect(current, tx)
})
} else {
finding, err = s.inspect(f, nil)
finding, err := s.inspect(f)
if err == nil && s.apply && finding != nil && finding.Action != "report" {
finding, err = s.repairFile(f.hash)
}
if err != nil {
return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err)
@@ -379,17 +377,66 @@ func (s *reconcileRun) files() error {
}
}
func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) {
return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
op := *s
op.ctx = opCtx
var current reconcileFile
s.stats.SQLQueries++
err := tx.QueryRow(opCtx, `
SELECT f.hash, f.size_bytes, f.status, f.created_at,
EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash)
FROM files f
WHERE f.hash = $1
FOR UPDATE OF f`, hash).
Scan(&current.hash, &current.size, &current.status, &current.created, &current.referenced)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
if !op.candidate(current) {
return nil, nil
}
finding, err := op.inspect(current)
if err != nil || finding == nil {
return finding, err
}
switch finding.Action {
case "delete_file":
// Delete bytes first. A failed SQL commit leaves an unreferenced
// missing row that the next pass can safely finish collecting.
if err := op.delete(finding.Key); err != nil {
return nil, err
}
s.stats.SQLQueries++
_, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash)
case "mark_failed":
s.stats.SQLQueries++
_, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed)
default:
return finding, nil
}
if err != nil {
return nil, err
}
finding.Applied = true
return finding, nil
})
}
func (s *reconcileRun) candidate(f reconcileFile) bool {
if !s.old(f.created) {
return false
}
if !f.referenced {
return f.status == "pending" || f.status == "ready"
return f.status == FilePending || f.status == FileReady
}
return f.status == "ready" || f.status == "failed"
return f.status == FileReady || f.status == FileFailed
}
func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) {
key := BlobKey(f.hash)
if !f.referenced {
// A publication that rolled back can leave fresh bytes behind an old
@@ -401,36 +448,14 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
if err == nil && !s.old(obj.Modified) {
return nil, nil
}
finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"}
if !s.apply {
return finding, nil
}
// S3 must succeed first. A failed SQL commit then leaves a missing,
// unreferenced row that the next run can safely finish collecting.
if err := s.delete(key); err != nil {
return nil, err
}
s.stats.SQLQueries++
if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil {
return nil, err
}
finding.Applied = true
return finding, nil
return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil
}
obj, err := s.stat(key)
if errors.Is(err, ErrObjectMissing) {
if f.status == "failed" {
if f.status == FileFailed {
return nil, nil
}
finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}
if s.apply {
s.stats.SQLQueries++
if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil {
return nil, err
}
finding.Applied = true
}
return finding, nil
return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil
}
if err != nil {
return nil, err
@@ -449,7 +474,7 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) {
}
// A failed file stays failed even when the key reappears. Recovery requires
// an explicit publisher; a shallow HEAD cannot prove content integrity.
if f.status == "failed" {
if f.status == FileFailed {
return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil
}
return nil, nil
+30 -7
View File
@@ -209,7 +209,8 @@ func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment
f.t.Fatal(err)
}
var found []attachment.Finding
stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) })
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) })
return stats, found, err
}
@@ -221,6 +222,28 @@ func TestReconciler(t *testing.T) {
t.Fatal(err)
}
defer db.Close()
t.Run("healthy apply does not block publication", func(t *testing.T) {
f := reconcileSetup(t, db)
hash := f.file("healthy", "ready", 2*time.Hour, true, true)
tx, err := db.Pool.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(ctx)
if err := attachment.LockHash(ctx, tx, hash); err != nil {
t.Fatal(err)
}
r, err := attachment.NewReconciler(f.db, f.b, time.Hour)
if err != nil {
t.Fatal(err)
}
scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
if err != nil || stats.Findings != 0 || stats.Repaired != 0 {
t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err)
}
})
t.Run("constructor validates and defaults grace", func(t *testing.T) {
f := reconcileSetup(t, db)
for _, grace := range []time.Duration{-time.Second, time.Minute} {
@@ -361,7 +384,7 @@ func TestReconciler(t *testing.T) {
done := make(chan error, 1)
go func() {
r, _ := attachment.NewReconciler(f.db, f.b, 0)
_, err := r.Run(ctx, true, false, nil)
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
done <- err
}()
waitReconcileLock(t, ctx, f.db)
@@ -395,7 +418,7 @@ func TestReconciler(t *testing.T) {
done := make(chan error, 1)
go func() {
r, _ := attachment.NewReconciler(f.db, f.b, 0)
_, err := r.Run(ctx, true, false, nil)
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
done <- err
}()
waitReconcileLock(t, ctx, f.db)
@@ -592,7 +615,7 @@ func TestReconcilerFailureBoundaries(t *testing.T) {
if err != nil {
t.Fatal(err)
}
stats, err := r.Run(ctx, false, true, nil)
stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil)
if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" {
t.Fatalf("read only dry run: %+v %v", stats, err)
}
@@ -681,11 +704,11 @@ func TestReconcilerS3(t *testing.T) {
t.Fatalf("CLI mutated by default: %+v %v", cliStats, err)
}
var found []attachment.Finding
stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) })
stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) })
if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" {
t.Fatalf("S3 dry: %+v %v %+v", stats, err, found)
}
stats, err = r.Run(ctx, true, true, nil)
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" {
t.Fatalf("S3 apply: %+v %v", stats, err)
}
@@ -695,7 +718,7 @@ func TestReconcilerS3(t *testing.T) {
if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil {
t.Fatal("fresh staging deleted", err)
}
stats, err = r.Run(ctx, true, true, nil)
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil)
if err != nil || stats.Repaired != 0 {
t.Fatalf("S3 repeat: %+v %v", stats, err)
}
+23
View File
@@ -0,0 +1,23 @@
package attachment
import (
"context"
"github.com/google/uuid"
)
type AttachmentRepository interface {
// WithPublication holds the file's hash lock through the callback and commit.
// Publication is valid only inside the callback. Any callback error rolls back.
WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error
GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error)
}
type Publication interface {
FileReady() bool
SaveReady(ctx context.Context) error
}
type ChatMembership interface {
IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error)
}
+91 -33
View File
@@ -15,40 +15,54 @@ import (
)
type Service struct {
repo Repository
blobs Blobs
maxBytes, maxPixels int64
decode chan struct{}
attachments AttachmentRepository
chats ChatMembership
blobs Blobs
config Config
decode chan struct{}
}
func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service {
if maxBytes == 0 {
maxBytes = MaxUploadBytes
}
if maxPixels == 0 {
maxPixels = MaxImagePixels
}
return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)}
type Config struct {
MaxBytes int64
MaxPixels int64
}
func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) {
if err := ValidateFilename(filename); err != nil {
// A nil blobs dependency allows the text API to run without configured S3.
// Binary operations report unavailability in that mode.
func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service {
if cfg.MaxBytes == 0 {
cfg.MaxBytes = MaxUploadBytes
}
if cfg.MaxPixels == 0 {
cfg.MaxPixels = MaxImagePixels
}
return &Service{
attachments: attachments,
chats: chats,
blobs: blobs,
config: cfg,
decode: make(chan struct{}, 1),
}
}
func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) {
if err := ValidateFilename(input.Filename); err != nil {
return Attachment{}, err
}
if mime != "image/png" && mime != "image/jpeg" {
if _, accepted := imageFormats[input.MIMEType]; !accepted {
return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted")
}
ok, err := s.repo.IsMember(ctx, chatID, actor)
ok, err := s.chats.IsMember(ctx, input.ChatID, actor)
if err != nil {
return Attachment{}, err
}
if !ok {
return Attachment{}, apperr.NotFound("chat not found")
}
if size > s.maxBytes {
if input.SizeBytes > s.config.MaxBytes {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
if size <= 0 {
if input.SizeBytes <= 0 {
return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required")
}
if s.blobs == nil {
@@ -59,16 +73,17 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
// A failed cleanup leaves a staging object for the reconciler's grace-based collection.
if err := s.blobs.Delete(cleanup, stage); err != nil {
slog.Warn("staging cleanup deferred", "error", err)
slog.Warn("staging cleanup deferred", "key", stage, "error", err)
}
}()
hash := sha256.New()
counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)}
err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime)
counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)}
err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType)
if err != nil {
_, tooLarge := errors.AsType[*http.MaxBytesError](err)
if counter.n > s.maxBytes || tooLarge {
if counter.n > s.config.MaxBytes || tooLarge {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
@@ -78,10 +93,10 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
}
var extra [1]byte
n, readErr := counter.Read(extra[:])
if counter.n > s.maxBytes {
if counter.n > s.config.MaxBytes {
return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit")
}
if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) {
if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) {
return Attachment{}, apperr.Invalid("body length does not match Content-Length")
}
// Full decoding is bounded by dimensions and one decoder per replica. Body
@@ -91,26 +106,64 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename,
case <-ctx.Done():
return Attachment{}, unavailable(ctx.Err())
}
err = s.validate(ctx, stage, mime)
err = s.validate(ctx, stage, input.MIMEType)
<-s.decode
if err != nil {
return Attachment{}, err
}
a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)}
if err := s.repo.Publish(ctx, a, stage); err != nil {
a := Attachment{
ID: id,
ChatID: input.ChatID,
OwnerID: actor,
Filename: input.Filename,
FileHash: hash.Sum(nil),
SizeBytes: input.SizeBytes,
MIMEType: input.MIMEType,
Status: FileReady,
CreatedAt: ident.TimeOf(id),
}
if err := s.attachments.WithPublication(ctx, a, func(p Publication) error {
if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil {
return err
}
return p.SaveReady(ctx)
}); err != nil {
return Attachment{}, err
}
return a, nil
}
func (s *Service) validate(ctx context.Context, stage, mime string) error {
func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error {
blobKey := BlobKey(a.FileHash)
if fileReady {
obj, err := s.blobs.Stat(ctx, blobKey)
if err == nil && obj.Size == a.SizeBytes {
return nil
}
if err != nil && !errors.Is(err, ErrObjectMissing) {
return unavailable(err)
}
}
// The hash is known after streaming. Copy the validated staging object to
// its permanent CAS key while the repository's publication lock is held.
if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil {
return unavailable(err)
}
return nil
}
func (s *Service) validate(ctx context.Context, stage, mime string) (err error) {
r, err := s.blobs.Open(ctx, stage)
if err != nil {
return unavailable(err)
}
defer r.Close()
defer func() {
if closeErr := r.Close(); closeErr != nil && err == nil {
err = unavailable(closeErr)
}
}()
tracked := &storageReader{reader: r}
err = ValidateImage(tracked, mime, s.maxPixels)
err = ValidateImage(tracked, mime, s.config.MaxPixels)
if tracked.err != nil {
return unavailable(tracked.err)
}
@@ -136,11 +189,11 @@ func (r *storageReader) Read(p []byte) (int, error) {
}
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
a, err := s.repo.GetForMember(ctx, id, actor)
a, err := s.attachments.GetForMember(ctx, id, actor)
if err != nil {
return Attachment{}, err
}
if a.Status != "ready" {
if a.Status != FileReady {
return Attachment{}, apperr.Conflict("attachment is not ready")
}
return a, nil
@@ -166,7 +219,12 @@ type countReader struct {
n int64
}
func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e }
func (c *countReader) Read(b []byte) (int, error) {
n, err := c.r.Read(b)
c.n += int64(n)
return n, err
}
func unavailable(err error) error {
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
}
+14 -8
View File
@@ -14,6 +14,7 @@ import (
"github.com/google/uuid"
"messenger/internal/attachment"
"messenger/internal/attachment/store"
chatstore "messenger/internal/chat/store"
"messenger/internal/platform/apperr"
"messenger/internal/platform/ident"
"messenger/internal/testenv"
@@ -73,10 +74,11 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
t.Fatal(err)
}
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0)
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
uploaded := make(chan error, 1)
go func() {
_, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data))
input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
_, e := s.Upload(ctx, actor, input, bytes.NewReader(data))
uploaded <- e
}()
select {
@@ -90,7 +92,7 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
}
collected := make(chan error, 1)
go func() {
stats, e := r.Run(ctx, true, false, nil)
stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
if e == nil && stats.Repaired != 0 {
e = errors.New("collector removed a newly referenced file")
}
@@ -196,7 +198,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
}
}
b := &testBlobs{data: map[string][]byte{}}
s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels)
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Fatal(err)
@@ -210,7 +212,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
c = c2
}
var e error
results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data))
input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data))
if e != nil {
t.Error(e)
}
@@ -235,7 +238,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
b.mu.Lock()
clear(b.data)
b.mu.Unlock()
if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil {
input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil {
t.Fatal(err)
}
if b.copies != 2 {
@@ -244,7 +248,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
b.failCopy = true
other := bytes.Clone(data)
other = append(other, 0)
if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
input.Filename, input.SizeBytes = "failed.png", int64(len(other))
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("copy outage: %v", err)
}
if len(b.data) != 1 {
@@ -258,7 +263,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil {
input.Filename = "rollback.png"
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil {
t.Fatal("publication survived rejected metadata")
}
if len(b.data) != 2 {
+49 -37
View File
@@ -4,32 +4,29 @@ import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
)
type Attachments struct {
pool *pgxpool.Pool
blobs attachment.Blobs
pool *pgxpool.Pool
}
func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments {
return &Attachments{pool, blobs}
}
var _ attachment.AttachmentRepository = (*Attachments)(nil)
func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) {
var ok bool
err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok)
return ok, postgres.Classify(err)
func NewAttachments(pool *pgxpool.Pool) *Attachments {
return &Attachments{pool: pool}
}
// Publication and garbage collection use the same hash lock. The transaction
// spans only publication, never client transfer or image decoding.
func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error {
func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error {
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
return err
@@ -37,41 +34,56 @@ func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stag
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
return err
}
var status string
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status)
var status attachment.FileStatus
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
copyNeeded := status != "ready"
if !copyNeeded {
obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash))
if errors.Is(e, attachment.ErrObjectMissing) {
copyNeeded = true
} else if e != nil {
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e)
} else if obj.Size != a.SizeBytes {
copyNeeded = true
}
}
if copyNeeded {
if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil {
return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err)
}
}
_, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType)
if err != nil {
return fmt.Errorf("publish file: %w", err)
}
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
return err
return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady})
}))
}
type publication struct {
tx pgx.Tx
attachment attachment.Attachment
ready bool
}
func (p publication) FileReady() bool { return p.ready }
func (p publication) SaveReady(ctx context.Context) error {
a := p.attachment
_, err := p.tx.Exec(ctx, `
INSERT INTO files (hash, size_bytes, mime_type, status)
VALUES ($1, $2, $3, $4)
ON CONFLICT (hash) DO UPDATE
SET size_bytes = excluded.size_bytes,
mime_type = excluded.mime_type,
status = excluded.status`,
a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady)
if err != nil {
return fmt.Errorf("publish file: %w", err)
}
_, err = p.tx.Exec(ctx, `
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
VALUES ($1, $2, $3, $4, $5, $6)`,
a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
return err
}
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
var a attachment.Attachment
err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at
FROM attachments a JOIN files f ON f.hash=a.file_hash
WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
err := r.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash,
f.size_bytes, f.mime_type, f.status, a.created_at
FROM attachments a
JOIN files f ON f.hash = a.file_hash
WHERE a.id = $1 AND EXISTS (
SELECT 1 FROM chat_members m
WHERE m.chat_id = a.chat_id AND m.user_id = $2
)`, id, actor).
Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash,
&a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return attachment.Attachment{}, apperr.NotFound("attachment not found")
}
+6 -1
View File
@@ -16,6 +16,11 @@ import (
const MaxUploadBytes int64 = 10 * 1024 * 1024
const MaxImagePixels int64 = 40_000_000
var imageFormats = map[string]string{
"image/png": "png",
"image/jpeg": "jpeg",
}
func ValidateFilename(name string) error {
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 {
return apperr.Invalid("filename must contain 1-255 characters")
@@ -37,7 +42,7 @@ func ValidateImage(r io.Reader, mime string, maxPixels int64) error {
if err != nil {
return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err)
}
if (format != "png" && format != "jpeg") || "image/"+format != mime {
if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat {
return apperr.UnsupportedMedia("image does not match Content-Type")
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) {
+54 -2
View File
@@ -8,11 +8,44 @@ import (
"image/color"
"image/png"
"io"
"messenger/internal/platform/apperr"
"os"
"strings"
"testing"
"github.com/google/uuid"
"messenger/internal/platform/apperr"
)
type chatMember struct {
chatID, userID uuid.UUID
}
func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) {
return chatID == m.chatID && userID == m.userID, nil
}
func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) {
member := chatMember{chatID: uuid.New(), userID: uuid.New()}
s := NewService(nil, member, nil, Config{})
input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4}
for _, tc := range []struct {
name string
actor uuid.UUID
kind apperr.Kind
}{
{name: "member", actor: member.userID, kind: apperr.KindUnavailable},
{name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound},
} {
t.Run(tc.name, func(t *testing.T) {
_, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data"))
if apperr.KindOf(err) != tc.kind {
t.Fatalf("got %v, want %v", err, tc.kind)
}
})
}
}
func TestValidateFilename(t *testing.T) {
for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} {
if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid {
@@ -41,16 +74,35 @@ type openedBlob struct {
}
func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) {
if reader, ok := b.reader.(io.ReadCloser); ok {
return reader, nil
}
return io.NopCloser(b.reader), nil
}
type closeErrorBody struct{ io.Reader }
func (closeErrorBody) Close() error { return errors.New("injected close failure") }
func TestStagingCloseFailurePreventsPublication(t *testing.T) {
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Fatal(err)
}
blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}}
s := NewService(nil, nil, blobs, Config{})
if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("close failure classified as %v", err)
}
}
func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) {
var pngData bytes.Buffer
if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
t.Fatal(err)
}
for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} {
s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0)
s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{})
err := s.validate(context.Background(), "staging/test", "image/png")
if apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err)
+3 -2
View File
@@ -18,6 +18,7 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/chat"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
@@ -202,7 +203,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
if err != nil {
return fmt.Errorf("check attachments: %w", err)
}
statuses, err := pgx.CollectRows(rows, pgx.RowTo[string])
statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus])
if err != nil {
return fmt.Errorf("read attachment status: %w", err)
}
@@ -210,7 +211,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
return apperr.NotFound("attachment not found")
}
for _, status := range statuses {
if status != "ready" {
if status != attachment.FileReady {
return apperr.Conflict("attachment is not ready")
}
}
+6 -2
View File
@@ -134,7 +134,7 @@ func Load(lookup Lookup) (Config, error) {
}
cfg.DebugAddr = v
}
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
return Config{}, err
}
if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil {
@@ -154,12 +154,16 @@ func LoadMigrate(lookup Lookup) (Migrate, error) {
if cfg.Logging, err = loadLogging(lookup); err != nil {
return Migrate{}, err
}
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil {
return Migrate{}, err
}
return cfg, nil
}
func LoadDatabaseURL(lookup Lookup) (string, error) {
return secret(lookup, "DATABASE_URL_FILE")
}
func loadLogging(lookup Lookup) (Logging, error) {
l := Logging{Env: "lab", Level: slog.LevelInfo}
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
+3 -2
View File
@@ -14,6 +14,7 @@ import (
"testing"
"time"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -85,7 +86,7 @@ func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) {
}
}))
defer server.Close()
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -126,7 +127,7 @@ func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) {
io.WriteString(w, `<Error><Code>ServiceUnavailable</Code><Message>volume not registered</Message></Error>`)
}))
defer server.Close()
s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -19,6 +19,7 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
"messenger/internal/attachment"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
"messenger/internal/testenv"
)
@@ -149,7 +150,7 @@ func TestSeaweedStore(t *testing.T) {
t.Fatal(err)
}
for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} {
outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1])
outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]})
if err != nil {
t.Fatal(err)
}
@@ -219,7 +220,7 @@ func TestSeaweedStore(t *testing.T) {
t.Fatal(err)
}
}
missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey)
missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
if err != nil {
t.Fatal(err)
}
+11 -8
View File
@@ -16,10 +16,13 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
"messenger/internal/attachment"
"messenger/internal/platform/config"
)
const MaxPageSize = 500
const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
type Store struct {
client *minio.Core
region string
@@ -30,15 +33,15 @@ var _ attachment.Blobs = (*Store)(nil)
// New validates configuration without contacting S3, so text-only service startup
// and readiness do not depend on object storage being reachable.
func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) {
u, err := url.Parse(endpoint)
func New(cfg config.S3) (*Store, error) {
u, err := url.Parse(cfg.Endpoint)
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment")
}
if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" {
if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
return nil, errors.New("s3: region and credentials are required")
}
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil {
return nil, fmt.Errorf("s3: invalid bucket: %w", err)
}
transport := &http.Transport{
@@ -56,9 +59,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
DisableCompression: true,
}
client, err := minio.NewCore(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: u.Scheme == "https",
Region: region,
Region: cfg.Region,
BucketLookup: minio.BucketLookupPath,
Transport: transport,
// A consumed upload stream cannot be replayed. Let the caller decide
@@ -68,7 +71,7 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error)
if err != nil {
return nil, fmt.Errorf("s3: client: %w", err)
}
return &Store{client: client, region: region, bucket: bucket}, nil
return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil
}
// Bootstrap is used only by the admin init command, never by API startup.
@@ -78,7 +81,7 @@ func (s *Store) Bootstrap(ctx context.Context) error {
}
err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region})
code := minio.ToErrorResponse(err).Code
if err != nil && code != "BucketAlreadyOwnedByYou" {
if err != nil && code != codeBucketAlreadyOwnedByYou {
return wrap("create bucket", err)
}
return s.WaitReady(ctx)
+6 -5
View File
@@ -11,6 +11,7 @@ import (
"testing"
"time"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -18,7 +19,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) }))
defer server.Close()
if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil {
if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil {
t.Fatal(err)
}
if calls.Load() != 0 {
@@ -36,7 +37,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) {
{"secret", server.URL, "us-east-1", "attachments", "access", ""},
} {
t.Run(tc.name, func(t *testing.T) {
if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil {
if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil {
t.Fatal("accepted invalid configuration")
}
})
@@ -69,7 +70,7 @@ func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) {
w.Header().Set("ETag", `"probe"`)
}))
defer server.Close()
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -103,7 +104,7 @@ func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) }))
defer server.Close()
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
@@ -154,7 +155,7 @@ func TestListStopsAfterOneBoundedPage(t *testing.T) {
io.WriteString(w, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>attachments</Name><IsTruncated>true</IsTruncated><NextContinuationToken>next-page</NextContinuationToken><Contents><Key>blobs/b</Key><Size>1</Size><LastModified>2026-01-01T00:00:00Z</LastModified></Contents><Contents><Key>blobs/c</Key><Size>2</Size><LastModified>2026-01-01T00:00:01Z</LastModified></Contents></ListBucketResult>`)
}))
defer server.Close()
store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret")
store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"})
if err != nil {
t.Fatal(err)
}
+4 -3
View File
@@ -22,6 +22,7 @@ import (
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
"messenger/internal/platform/config"
"messenger/internal/platform/s3store"
)
@@ -131,15 +132,15 @@ func (s *S3) launch(ctx context.Context) error {
if err != nil {
return err
}
s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey)
s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey})
if err != nil {
return err
}
s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey)
s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey})
if err != nil {
return err
}
s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey)
s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey})
return err
}
+2 -1
View File
@@ -119,7 +119,8 @@ func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string,
}
}()
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
close(done)
m.GoHeapPeakBytes = <-peakCh
m.Stats = stats