Переведите приём и чтение вложений на асинхронный контракт

This commit is contained in:
2026-11-18 15:33:32 +03:00
parent efc56b1382
commit bf970227d6
19 changed files with 667 additions and 367 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
return 1
}
defer db.Close()
r, err := attachment.NewReconciler(db, blobs, *grace)
r, err := attachment.NewReconcilerWithRetention(db, blobs, *grace, s3Config.StagingRetention)
if err != nil {
fmt.Fprintln(errOut, "reconcile:", err)
return 2
+78 -8
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"image"
"image/jpeg"
@@ -22,6 +23,7 @@ import (
"messenger/internal/app"
"messenger/internal/attachment"
"messenger/internal/chat"
"messenger/internal/pipeline"
"messenger/internal/testenv"
)
@@ -117,10 +119,10 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
t.Fatal(err)
}
res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data)))
ts.expect(res, 201, "")
ts.expect(res, 202, "")
a := decode[attachment.Response](t, res)
digest := sha256.Sum256(data)
if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "ready" || a.ErrorCode != nil || len(a.Variants) != 0 {
if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "queued" || a.ErrorCode != nil || len(a.Variants) != 0 {
t.Fatalf("attachment: %+v", a)
}
base := "/api/v1/attachments/" + a.ID
@@ -129,11 +131,9 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
}
ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "")
got := ts.binary("GET", base+"/content", bob.token, "", nil, 0)
ts.expect(got, 200, "")
if !bytes.Equal(got.raw, data) {
t.Fatal("download bytes changed")
}
ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 400, "invalid_request")
ts.expect(got, 409, "conflict")
ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 409, "conflict")
ts.expect(ts.do("GET", base+"/content?variant=unknown", bob.token, nil, nil), 400, "invalid_request")
msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey())
ts.expect(msg, 201, "")
if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID {
@@ -168,7 +168,7 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil {
t.Fatal(err)
}
ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 201, "")
ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 202, "")
ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found")
// An interrupted read is never a 500 or a successfully published identity.
before := 0
@@ -183,6 +183,76 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after {
t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err)
}
// Complete one claimed job with known bytes. Image encoding has its own
// tests; this checks the published version, exact download and ACL on S3.
repo := pipeline.NewStore(db.Pool)
var job pipeline.Job
found, err := repo.PublishOne(ctx, func(_ context.Context, j pipeline.Job) error { job = j; return nil })
if err != nil || !found || job.AttachmentID.String() != a.ID {
t.Fatalf("outbox: %+v %v", job, err)
}
claim, err := repo.Claim(ctx, job, time.Minute)
if err != nil || claim == nil {
t.Fatal("claim", err)
}
meta := decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil))
if meta.Status != "processing" || len(meta.Variants) != 0 {
t.Fatal("processing snapshot", meta)
}
if err := s3.Store.Copy(ctx, claim.SourceKey, attachment.BlobKey(claim.Hash)); err != nil {
t.Fatal(err)
}
bodies := map[string][]byte{"original": data}
var outputs []pipeline.Variant
for _, name := range []string{"thumbnail", "crop", "optimized"} {
body := []byte("published-" + name)
mime := "image/jpeg"
if name == "optimized" {
mime = "image/webp"
}
digest := sha256.Sum256(body)
key := pipeline.VariantKey(claim.AttachmentID, claim.PipelineVersion, name, digest[:])
if err := s3.Store.Put(ctx, key, bytes.NewReader(body), int64(len(body)), mime); err != nil {
t.Fatal(err)
}
outputs = append(outputs, pipeline.Variant{Name: name, Key: key, MIMEType: mime, Width: 1, Height: 1, SizeBytes: int64(len(body)), SHA256: digest[:]})
bodies[name] = body
}
if ok, err := repo.Complete(ctx, claim, outputs); err != nil || !ok {
t.Fatal("publish", err)
}
if err := repo.Reprocess(ctx, claim.AttachmentID, 2, func(ctx context.Context, key string) (bool, error) {
_, err := s3.Store.Stat(ctx, key)
if errors.Is(err, attachment.ErrObjectMissing) {
return false, nil
}
return err == nil, err
}); err != nil {
t.Fatal(err)
}
meta = decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil))
if meta.Status != "ready" || len(meta.Variants) != 3 || meta.ErrorCode != nil || meta.SHA256 != a.SHA256 {
t.Fatalf("ready snapshot during reprocess: %+v", meta)
}
for _, v := range meta.Variants {
digest := sha256.Sum256(bodies[v.Name])
if v.SHA256 != hex.EncodeToString(digest[:]) || v.SizeBytes != int64(len(bodies[v.Name])) {
t.Fatal("variant checksum", v)
}
}
for name, body := range bodies {
target := base + "/content?variant=" + name
ts.expect(ts.do("GET", target, eve.token, nil, nil), 404, "not_found")
got := ts.binary("GET", target, bob.token, "", nil, 0)
ts.expect(got, 200, "")
if !bytes.Equal(got.raw, body) {
t.Fatalf("wrong published bytes for %s", name)
}
}
if err := s3.Store.Delete(ctx, outputs[0].Key); err != nil {
t.Fatal(err)
}
ts.expect(ts.do("GET", base+"/content?variant="+outputs[0].Name, bob.token, nil, nil), 404, "not_found")
if err := s3.Stop(ctx); err != nil {
t.Fatal(err)
}
+23 -10
View File
@@ -9,19 +9,23 @@ import (
type FileStatus string
const (
FilePending FileStatus = "pending"
FileReady FileStatus = "ready"
FileFailed FileStatus = "failed"
FileQueued FileStatus = "queued"
FileProcessing FileStatus = "processing"
FilePending FileStatus = "pending"
FileReady FileStatus = "ready"
FileFailed FileStatus = "failed"
)
type Attachment struct {
ID, ChatID, OwnerID uuid.UUID
Filename string
FileHash []byte
SizeBytes int64
MIMEType string
Status FileStatus
CreatedAt time.Time
ID, ChatID, OwnerID uuid.UUID
Filename string
FileHash []byte
SizeBytes int64
MIMEType string
Status FileStatus
CreatedAt, UpdatedAt time.Time
ErrorCode *string
Variants []Variant
}
type UploadInput struct {
@@ -30,3 +34,12 @@ type UploadInput struct {
MIMEType string
SizeBytes int64
}
type Variant struct {
Name string
ObjectKey string
MIMEType string
SizeBytes int64
Width, Height int
Hash []byte
}
+43 -18
View File
@@ -26,22 +26,37 @@ func (h *Handler) Register(protected, binaryProtected gin.IRouter) {
}
type Response struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []VariantResponse `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type VariantResponse struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
// Lab4's shared response contract requires variants=[], even without processing.
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
func toResponse(a Attachment) Response {
variants := make([]VariantResponse, 0, len(a.Variants))
for _, v := range a.Variants {
variants = append(variants, VariantResponse{
Name: v.Name, ContentType: v.MIMEType, SizeBytes: v.SizeBytes,
Width: v.Width, Height: v.Height, SHA256: hex.EncodeToString(v.Hash),
})
}
return Response{
ID: a.ID.String(),
ChatID: a.ChatID.String(),
@@ -51,9 +66,10 @@ func toResponse(a Attachment) Response {
SizeBytes: a.SizeBytes,
SHA256: hex.EncodeToString(a.FileHash),
Status: string(a.Status),
Variants: []any{},
Variants: variants,
ErrorCode: a.ErrorCode,
CreatedAt: httpx.Timestamp(a.CreatedAt),
UpdatedAt: httpx.Timestamp(a.CreatedAt),
UpdatedAt: httpx.Timestamp(a.UpdatedAt),
}
}
@@ -89,7 +105,7 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
httpx.JSON(c, http.StatusCreated, toResponse(a))
httpx.JSON(c, http.StatusAccepted, toResponse(a))
}
func attachmentID(c *gin.Context) (uuid.UUID, error) {
id, err := ident.Parse(c.Param("attachment_id"))
@@ -117,12 +133,21 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
variant, present := c.GetQuery("variant")
if present && variant != "original" {
httpx.Error(c, apperr.Invalid("variant must be original"))
return
variant := "original"
if values, ok := c.Request.URL.Query()["variant"]; ok {
if len(values) != 1 {
httpx.Error(c, apperr.Invalid("one variant is required"))
return
}
variant = values[0]
switch variant {
case "original", "thumbnail", "crop", "optimized":
default:
httpx.Error(c, apperr.Invalid("unknown variant"))
return
}
}
a, r, err := h.s.Content(c.Request.Context(), actor, id)
a, r, err := h.s.Content(c.Request.Context(), actor, id, variant)
if err != nil {
httpx.Error(c, err)
return
+87 -26
View File
@@ -9,8 +9,11 @@ import (
"fmt"
"io"
"math"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -40,9 +43,10 @@ type Stats struct {
}
type Reconciler struct {
db *pgxpool.Pool
blobs Blobs
grace time.Duration
db *pgxpool.Pool
blobs Blobs
grace time.Duration
retention time.Duration
}
type ReconcileOptions struct {
@@ -51,6 +55,13 @@ type ReconcileOptions struct {
}
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
return NewReconcilerWithRetention(db, blobs, grace, 24*time.Hour)
}
func NewReconcilerWithRetention(db *pgxpool.Pool, blobs Blobs, grace, retention time.Duration) (*Reconciler, error) {
if retention < time.Second {
return nil, errors.New("reconcile: staging retention must be at least one second")
}
if db == nil || blobs == nil {
return nil, errors.New("reconcile: database and blobs are required")
}
@@ -60,7 +71,7 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci
if grace < 2*time.Minute {
return nil, errors.New("reconcile: grace must be at least two minutes")
}
return &Reconciler{db: db, blobs: blobs, grace: grace}, nil
return &Reconciler{db: db, blobs: blobs, grace: grace, retention: retention}, nil
}
// Run streams findings and keeps at most one page of metadata in memory.
@@ -78,7 +89,7 @@ func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit fun
stats: &stats,
cutoff: started.Add(-r.grace),
}
for _, prefix := range []string{"staging/", "blobs/"} {
for _, prefix := range []string{"staging/", "variants/", "blobs/"} {
if err = run.objects(prefix); err != nil {
return stats, err
}
@@ -147,10 +158,10 @@ func (s *reconcileRun) objects(prefix string) error {
continue
}
var finding *Finding
if prefix == "staging/" {
finding, err = s.staging(obj)
} else {
if prefix == "blobs/" {
finding, err = s.orphan(obj, known[obj.Key])
} else {
finding, err = s.jobObject(prefix, obj)
}
if err != nil {
return fmt.Errorf("reconcile %s: %w", obj.Key, err)
@@ -178,26 +189,76 @@ func (s *reconcileRun) delete(key string) error {
return err
}
func (s *reconcileRun) staging(obj Object) (*Finding, error) {
f := &Finding{Kind: "stale_staging", Key: obj.Key, Action: "delete_object"}
// Failed sources remain available for operator replay until the retention period after failure elapses.
// Staging shared by several pipeline versions is retained while any active job needs it.
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
parts := strings.Split(strings.TrimPrefix(obj.Key, prefix), "/")
id, idErr := uuid.Parse(parts[0])
kind := "stale_staging"
query := `SELECT EXISTS(SELECT 1 FROM attachment_jobs
WHERE attachment_id=$1 AND source_key=$2 AND
(state IN ('queued','processing') OR (state='failed' AND updated_at >= now()-$3::bigint*interval '1 second')))`
args := []any{id, obj.Key, int64(s.r.retention / time.Second)}
if prefix == "variants/" {
if len(parts) != 4 || idErr != nil {
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
}
version, err := strconv.Atoi(parts[1])
if err != nil || version < 1 || version > math.MaxInt32 {
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
}
kind = "orphan_variant"
query = `SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$3 AND object_key=$2)
OR EXISTS(SELECT 1 FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=$3 AND state IN ('queued','processing'))`
args[2] = version
}
finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"}
var keep bool
s.stats.SQLQueries++
if err := s.r.db.QueryRow(s.ctx, query, args...).Scan(&keep); err != nil {
return nil, err
}
if keep {
return nil, nil
}
if !s.apply {
return f, nil
return finding, nil
}
current, err := s.stat(obj.Key)
if errors.Is(err, ErrObjectMissing) {
return nil, nil
}
if err != nil {
return nil, err
}
if !s.old(current.Modified) {
return nil, nil
}
if err = s.delete(obj.Key); err != nil {
return nil, err
}
f.Applied = true
return f, nil
keyHash := sha256.Sum256([]byte(obj.Key))
return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
// Intake locks the staging key before inserting an attachment. Replay,
// reprocess and completion lock the existing attachment before changing job state.
if idErr == nil {
s.stats.SQLQueries++
if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil {
return nil, err
}
}
s.stats.SQLQueries++
if err := tx.QueryRow(opCtx, query, args...).Scan(&keep); err != nil {
return nil, err
}
if keep {
return nil, nil
}
op := *s
op.ctx = opCtx
current, err := op.stat(obj.Key)
if errors.Is(err, ErrObjectMissing) {
return nil, nil
}
if err != nil {
return nil, err
}
if !s.old(current.Modified) {
return nil, nil
}
if err := op.delete(obj.Key); err != nil {
return nil, err
}
finding.Applied = true
return finding, nil
})
}
func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
+44
View File
@@ -729,3 +729,47 @@ func TestReconcileLockRejectsMalformedHash(t *testing.T) {
t.Fatal("malformed hash accepted")
}
}
func TestGCRespectsJobSourcesAndVersionedOutputs(t *testing.T) {
ctx := context.Background()
db, err := testenv.StartPostgres(ctx)
if err != nil {
t.Fatal(err)
}
defer db.Close()
f := reconcileSetup(t, db)
hash := f.file("job input", "pending", 48*time.Hour, true, false)
var id uuid.UUID
if err := db.Pool.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil {
t.Fatal(err)
}
stage := "staging/" + id.String()
output := "variants/" + id.String() + "/1/thumbnail/" + strings.Repeat("0", 64)
old := time.Now().Add(-48 * time.Hour)
f.b.add(stage, []byte("job input"), old)
f.b.add(output, []byte("partial"), old)
f.exec(`INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, stage)
for _, apply := range []bool{false, true} {
stats, _, err := f.run(apply, false)
if err != nil || stats.Findings != 0 || !f.b.has(stage) || !f.b.has(output) {
t.Fatalf("active job lost: %+v %v", stats, err)
}
}
f.exec(`UPDATE attachment_jobs SET state='failed', error_code='retry_exhausted',updated_at=now()`)
stats, _, err := f.run(true, false)
if err != nil || stats.Repaired != 1 || !f.b.has(stage) || f.b.has(output) {
t.Fatalf("failed retention: %+v %v", stats, err)
}
f.exec(`UPDATE attachment_jobs SET updated_at=now()-interval '3 hours'`)
stats, _, err = f.run(true, false)
if err != nil || stats.Repaired != 0 || !f.b.has(stage) {
t.Fatalf("default failed retention: %+v %v", stats, err)
}
r, err := attachment.NewReconcilerWithRetention(db.Pool, f.b, time.Hour, 2*time.Hour)
if err != nil {
t.Fatal(err)
}
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
if err != nil || stats.Repaired != 1 || f.b.has(stage) {
t.Fatalf("expired source: %+v %v", stats, err)
}
}
+2 -8
View File
@@ -7,17 +7,11 @@ import (
)
type AttachmentRepository interface {
// WithPublication holds the file's hash lock through the callback and commit.
// Publication is valid only inside the callback. Any callback error rolls back.
WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error
// Enqueue checks the staging object under the collector's lock before committing its job.
Enqueue(ctx context.Context, attachment Attachment, stage string, checkStage func() error) error
GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error)
}
type Publication interface {
FileReady() bool
SaveReady(ctx context.Context) error
}
type ChatMembership interface {
IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error)
}
+51 -37
View File
@@ -3,14 +3,17 @@ package attachment
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log/slog"
"net/http"
"path/filepath"
"time"
"github.com/google/uuid"
"messenger/internal/platform/apperr"
"messenger/internal/platform/faultbarrier"
"messenger/internal/platform/ident"
)
@@ -23,8 +26,9 @@ type Service struct {
}
type Config struct {
MaxBytes int64
MaxPixels int64
MaxBytes int64
MaxPixels int64
BarrierDir string
}
// A nil blobs dependency allows the text API to run without configured S3.
@@ -70,7 +74,11 @@ func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput
}
id := ident.New()
stage := "staging/" + id.String()
discard := true
defer func() {
if !discard {
return
}
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
// A failed cleanup leaves a staging object for the reconciler's grace-based collection.
@@ -119,37 +127,30 @@ func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput
FileHash: hash.Sum(nil),
SizeBytes: input.SizeBytes,
MIMEType: input.MIMEType,
Status: FileReady,
Status: FileQueued,
UpdatedAt: ident.TimeOf(id),
CreatedAt: ident.TimeOf(id),
}
if err := s.attachments.WithPublication(ctx, a, func(p Publication) error {
if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil {
return err
if s.config.BarrierDir != "" {
base := filepath.Join(s.config.BarrierDir, hex.EncodeToString(a.FileHash)+"-staged")
if err := faultbarrier.Wait(ctx, base, a.ID.String()); err != nil {
return Attachment{}, unavailable(err)
}
return p.SaveReady(ctx)
}); err != nil {
return Attachment{}, err
}
return a, nil
}
func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error {
blobKey := BlobKey(a.FileHash)
if fileReady {
obj, err := s.blobs.Stat(ctx, blobKey)
if err == nil && obj.Size == a.SizeBytes {
return nil
}
if err != nil && !errors.Is(err, ErrObjectMissing) {
// COMMIT can succeed even if its reply is lost. From this point only the
// collector, after consulting durable job state, may delete this source.
discard = false
err = s.attachments.Enqueue(ctx, a, stage, func() error {
obj, err := s.blobs.Stat(ctx, stage)
if err != nil {
return unavailable(err)
}
}
// The hash is known after streaming. Copy the validated staging object to
// its permanent CAS key while the repository's publication lock is held.
if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil {
return unavailable(err)
}
return nil
if obj.Size != a.SizeBytes {
return unavailable(errors.New("staging size changed"))
}
return nil
})
return a, err
}
func (s *Service) validate(ctx context.Context, stage, mime string) (err error) {
@@ -189,25 +190,38 @@ func (r *storageReader) Read(p []byte) (int, error) {
}
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
a, err := s.attachments.GetForMember(ctx, id, actor)
if err != nil {
return Attachment{}, err
}
if a.Status != FileReady {
return Attachment{}, apperr.Conflict("attachment is not ready")
}
return a, nil
return s.attachments.GetForMember(ctx, id, actor)
}
func (s *Service) Content(ctx context.Context, actor, id uuid.UUID) (Attachment, io.ReadCloser, error) {
func (s *Service) Content(ctx context.Context, actor, id uuid.UUID, name string) (Attachment, io.ReadCloser, error) {
a, err := s.Get(ctx, actor, id)
if err != nil {
return Attachment{}, nil, err
}
if a.Status != FileReady {
return Attachment{}, nil, apperr.Conflict("attachment is not ready")
}
key := BlobKey(a.FileHash)
if name != "original" {
found := false
for _, v := range a.Variants {
if name == v.Name {
key, a.MIMEType, a.SizeBytes = v.ObjectKey, v.MIMEType, v.SizeBytes
found = true
break
}
}
if !found {
return Attachment{}, nil, apperr.NotFound("variant not found")
}
}
if s.blobs == nil {
return Attachment{}, nil, apperr.Unavailable("object storage is unavailable")
}
r, err := s.blobs.Open(ctx, BlobKey(a.FileHash))
r, err := s.blobs.Open(ctx, key)
if name != "original" && errors.Is(err, ErrObjectMissing) {
return Attachment{}, nil, apperr.NotFound("variant not found")
}
if err != nil {
return Attachment{}, nil, unavailable(err)
}
+64 -159
View File
@@ -3,7 +3,6 @@ package attachment_test
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"io"
"os"
@@ -20,159 +19,79 @@ import (
"messenger/internal/testenv"
)
type testBlobs struct {
mu sync.Mutex
data map[string][]byte
copies int
failCopy bool
}
type blockingPublication struct {
*testBlobs
entered, resume chan struct{}
}
func (b *blockingPublication) Copy(ctx context.Context, from, to string) error {
close(b.entered)
select {
case <-b.resume:
case <-ctx.Done():
return ctx.Err()
}
return b.testBlobs.Copy(ctx, from, to)
}
func (b *blockingPublication) Stat(ctx context.Context, key string) (attachment.Object, error) {
o, e := b.testBlobs.Stat(ctx, key)
o.Modified = time.Now().Add(-2 * time.Hour)
return o, e
}
func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
func TestCollectorRechecksStageAfterConcurrentEnqueue(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
db, err := testenv.StartPostgres(ctx)
if err != nil {
t.Fatal(err)
}
defer db.Close()
actor, room := ident.New(), ident.New()
if _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'race_user','User','hash',now())`, actor); err != nil {
t.Fatal(err)
}
if _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'race',now())`, room); err != nil {
t.Fatal(err)
}
if _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, room, actor); err != nil {
t.Fatal(err)
}
f := reconcileSetup(t, db)
f.exec(`INSERT INTO chat_members VALUES ($1,$2)`, f.chat, f.user)
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Fatal(err)
}
hash := sha256.Sum256(data)
if _, err = db.Pool.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png','pending',now()-interval '2 hours')`, hash[:], len(data)); err != nil {
t.Fatal(err)
entered := make(chan string, 1)
resume := make(chan struct{})
var once sync.Once
f.b.stating = func(key string) {
once.Do(func() {
f.b.add(key, data, time.Now().Add(-2*time.Hour))
entered <- key
select {
case <-resume:
case <-ctx.Done():
}
})
}
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), f.b, attachment.Config{})
uploaded := make(chan error, 1)
go func() {
input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
_, e := s.Upload(ctx, actor, input, bytes.NewReader(data))
uploaded <- e
_, err := s.Upload(ctx, f.user, attachment.UploadInput{ChatID: f.chat, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}, bytes.NewReader(data))
uploaded <- err
}()
var key string
select {
case <-b.entered:
case key = <-entered:
case <-ctx.Done():
t.Fatal("publisher never reached copy")
t.Fatal("upload never verified staging under lock")
}
r, err := attachment.NewReconciler(db.Pool, b, time.Hour)
r, err := attachment.NewReconciler(db.Pool, f.b, time.Hour)
if err != nil {
t.Fatal(err)
}
collected := make(chan error, 1)
collect := make(chan error, 1)
go func() {
stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
if e == nil && stats.Repaired != 0 {
e = errors.New("collector removed a newly referenced file")
stats, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
if err == nil && stats.Repaired != 0 {
err = errors.New("removed concurrent upload")
}
collected <- e
collect <- err
}()
// Observe a real waiting advisory lock, rather than guessing which goroutine won.
deadline := time.Now().Add(3 * time.Second)
waiting := false
for time.Now().Before(deadline) {
var n int
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted`).Scan(&n); err != nil {
t.Fatal(err)
}
if n > 0 {
waiting = true
break
}
time.Sleep(5 * time.Millisecond)
}
close(b.resume)
if !waiting {
t.Fatal("collector did not serialize with the publisher")
}
waitReconcileLock(t, ctx, db.Pool)
close(resume)
if err := <-uploaded; err != nil {
t.Fatal(err)
}
if err := <-collected; err != nil {
if err := <-collect; err != nil {
t.Fatal(err)
}
if _, err := b.Stat(ctx, attachment.BlobKey(hash[:])); err != nil {
t.Fatal("collector deleted published data", err)
if !f.b.has(key) {
t.Fatal("accepted source lost")
}
}
func (b *testBlobs) Put(_ context.Context, k string, r io.Reader, n int64, m string) error {
data, err := io.ReadAll(r)
if err != nil {
type lostCommitReply struct {
attachment.AttachmentRepository
}
func (r lostCommitReply) Enqueue(ctx context.Context, a attachment.Attachment, stage string, check func() error) error {
if err := r.AttachmentRepository.Enqueue(ctx, a, stage, check); err != nil {
return err
}
b.mu.Lock()
defer b.mu.Unlock()
b.data[k] = data
return nil
}
func (b *testBlobs) Open(_ context.Context, k string) (io.ReadCloser, error) {
b.mu.Lock()
defer b.mu.Unlock()
data, ok := b.data[k]
if !ok {
return nil, attachment.ErrObjectMissing
}
return io.NopCloser(bytes.NewReader(data)), nil
}
func (b *testBlobs) Stat(_ context.Context, k string) (attachment.Object, error) {
b.mu.Lock()
defer b.mu.Unlock()
data, ok := b.data[k]
if !ok {
return attachment.Object{}, attachment.ErrObjectMissing
}
return attachment.Object{Key: k, Size: int64(len(data)), Modified: time.Now()}, nil
}
func (b *testBlobs) Copy(_ context.Context, from, to string) error {
b.mu.Lock()
defer b.mu.Unlock()
if b.failCopy {
return errors.New("copy outage")
}
b.copies++
b.data[to] = bytes.Clone(b.data[from])
return nil
}
func (b *testBlobs) Delete(_ context.Context, k string) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.data, k)
return nil
}
func (b *testBlobs) List(context.Context, string, string, int) ([]attachment.Object, error) {
return nil, nil
return io.ErrUnexpectedEOF
}
func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
@@ -197,7 +116,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
t.Fatal(err)
}
}
b := &testBlobs{data: map[string][]byte{}}
b := newReconcileBlobs()
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
@@ -220,13 +139,16 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
})
}
wg.Wait()
var files, atts int
err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments)`).Scan(&files, &atts)
if err != nil {
t.Fatal(err)
var files, atts, jobs int
err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments),
(SELECT count(*) FROM attachment_jobs WHERE state='queued' AND attempt=1 AND published_at IS NULL)`).Scan(&files, &atts, &jobs)
if err != nil || files != 1 || atts != 10 || jobs != 10 || len(b.objects) != 10 {
t.Fatalf("handoff: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err)
}
if files != 1 || atts != 10 || b.copies != 1 || len(b.data) != 1 {
t.Fatalf("files=%d attachments=%d copies=%d objects=%d", files, atts, b.copies, len(b.data))
for _, a := range results {
if a.Status != attachment.FileQueued || !bytes.Equal(b.bodies["staging/"+a.ID.String()], data) {
t.Fatal("missing queued source", a.ID)
}
}
if results[0].ID == results[1].ID {
t.Fatal("deduplication merged chat ACL identity")
@@ -234,43 +156,26 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound {
t.Fatalf("outsider got %v", err)
}
// A removed blob is republished from the new upload, preserving old references.
b.mu.Lock()
clear(b.data)
b.mu.Unlock()
input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil {
t.Fatal(err)
input := attachment.UploadInput{ChatID: c1, Filename: "lost.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
lost := attachment.NewService(lostCommitReply{store.NewAttachments(db.Pool)}, chatstore.NewChats(db.Pool), b, attachment.Config{})
if _, err := lost.Upload(ctx, actor, input, bytes.NewReader(data)); !errors.Is(err, io.ErrUnexpectedEOF) {
t.Fatal("lost commit reply:", err)
}
if b.copies != 2 {
t.Fatalf("missing ready blob was not restored: copies=%d", b.copies)
var source string
if err := db.Pool.QueryRow(ctx, `SELECT j.source_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.filename='lost.png'`).Scan(&source); err != nil || !bytes.Equal(b.bodies[source], data) {
t.Fatal("committed job lost source", err)
}
b.failCopy = true
other := bytes.Clone(data)
other = append(other, 0)
input.Filename, input.SizeBytes = "failed.png", int64(len(other))
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
t.Fatalf("copy outage: %v", err)
}
if len(b.data) != 1 {
t.Fatalf("failed publication left staging: %d objects", len(b.data))
}
b.failCopy = false
// S3 publication succeeds, then PostgreSQL refuses the logical attachment.
// Compensation may remove staging but must leave shared CAS data intact.
_, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$;
CREATE TRIGGER reject_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_attachment()`)
_, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_job() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$;
CREATE TRIGGER reject_job BEFORE INSERT ON attachment_jobs FOR EACH ROW EXECUTE FUNCTION reject_job()`)
if err != nil {
t.Fatal(err)
}
input.Filename = "rollback.png"
other := append(bytes.Clone(data), 0)
input.Filename, input.SizeBytes = "rollback.png", int64(len(other))
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil {
t.Fatal("publication survived rejected metadata")
t.Fatal("accepted rejected job")
}
if len(b.data) != 2 {
t.Fatalf("expected original and recoverable orphan, got %d objects", len(b.data))
}
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&files); err != nil || files != 1 {
t.Fatalf("failed DB insert committed file metadata: %d %v", files, err)
if err := db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments),(SELECT count(*) FROM attachment_jobs)`).Scan(&files, &atts, &jobs); err != nil || files != 1 || atts != 11 || jobs != 11 || len(b.objects) != 12 {
t.Fatalf("rollback: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err)
}
}
+55 -57
View File
@@ -2,8 +2,7 @@ package store
import (
"context"
"errors"
"fmt"
"crypto/sha256"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -14,78 +13,77 @@ import (
"messenger/internal/platform/postgres"
)
type Attachments struct {
pool *pgxpool.Pool
}
type Attachments struct{ pool *pgxpool.Pool }
var _ attachment.AttachmentRepository = (*Attachments)(nil)
func NewAttachments(pool *pgxpool.Pool) *Attachments {
return &Attachments{pool: pool}
}
func NewAttachments(pool *pgxpool.Pool) *Attachments { return &Attachments{pool: pool} }
// Publication and garbage collection use the same hash lock. The transaction
// spans only publication, never client transfer or image decoding.
func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error {
func (r *Attachments) Enqueue(ctx context.Context, a attachment.Attachment, stage string, checkStage func() error) error {
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
// A request can pause after validation for longer than the GC grace.
// Verify the source after serializing with deletion of this exact key.
stageHash := sha256.Sum256([]byte(stage))
if err := attachment.LockHash(ctx, tx, stageHash[:]); err != nil {
return err
}
if err := checkStage(); err != nil {
return err
}
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
return err
}
var status attachment.FileStatus
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
if _, err := tx.Exec(ctx, `
INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ($1,$2,$3,'pending')
ON CONFLICT (hash) DO NOTHING`, a.FileHash, a.SizeBytes, a.MIMEType); err != nil {
return err
}
return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady})
if _, err := tx.Exec(ctx, `
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
VALUES ($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt); err != nil {
return err
}
_, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, a.ID, stage)
return err
}))
}
type publication struct {
tx pgx.Tx
attachment attachment.Attachment
ready bool
}
func (p publication) FileReady() bool { return p.ready }
func (p publication) SaveReady(ctx context.Context) error {
a := p.attachment
_, err := p.tx.Exec(ctx, `
INSERT INTO files (hash, size_bytes, mime_type, status)
VALUES ($1, $2, $3, $4)
ON CONFLICT (hash) DO UPDATE
SET size_bytes = excluded.size_bytes,
mime_type = excluded.mime_type,
status = excluded.status`,
a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady)
if err != nil {
return fmt.Errorf("publish file: %w", err)
}
_, err = p.tx.Exec(ctx, `
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
VALUES ($1, $2, $3, $4, $5, $6)`,
a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
return err
}
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
var a attachment.Attachment
err := r.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash,
f.size_bytes, f.mime_type, f.status, a.created_at
// A single statement returns one snapshot of the published version, even
// when a worker completes a newer version during a download.
rows, err := r.pool.Query(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash, f.size_bytes, f.mime_type,
s.status, s.error_code, a.created_at, s.updated_at,
coalesce(v.variant,''),coalesce(v.object_key,''),coalesce(v.mime_type,''),
coalesce(v.size_bytes,0),coalesce(v.width,0),coalesce(v.height,0),v.sha256
FROM attachments a
JOIN files f ON f.hash = a.file_hash
WHERE a.id = $1 AND EXISTS (
SELECT 1 FROM chat_members m
WHERE m.chat_id = a.chat_id AND m.user_id = $2
)`, id, actor).
Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash,
&a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return attachment.Attachment{}, apperr.NotFound("attachment not found")
JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2
JOIN files f ON f.hash=a.file_hash
JOIN attachment_states s ON s.id=a.id
LEFT JOIN attachment_variants v ON v.attachment_id=a.id AND v.pipeline_version=a.published_version AND s.status='ready'
WHERE a.id=$1 ORDER BY v.variant`, id, actor)
if err != nil {
return a, postgres.Classify(err)
}
return a, postgres.Classify(err)
defer rows.Close()
for rows.Next() {
var v attachment.Variant
err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType,
&a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
&v.Name, &v.ObjectKey, &v.MIMEType, &v.SizeBytes, &v.Width, &v.Height, &v.Hash)
if err != nil {
return attachment.Attachment{}, postgres.Classify(err)
}
if v.Name != "" {
a.Variants = append(a.Variants, v)
}
}
if err := rows.Err(); err != nil {
return attachment.Attachment{}, postgres.Classify(err)
}
if a.ID == uuid.Nil {
return a, apperr.NotFound("attachment not found")
}
return a, nil
}
+45 -7
View File
@@ -25,7 +25,13 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
INSERT INTO files (hash, size_bytes, mime_type, status, created_at)
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], status, id, chatID, owner)
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], "pending", id, chatID, owner)
if err != nil {
t.Fatal(err)
}
_, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code,lease_token,lease_until)
VALUES($1,$2,$3,CASE WHEN $3='failed' THEN 'processing_failed' END,CASE WHEN $3='processing' THEN gen_random_uuid() END,
CASE WHEN $3='processing' THEN now()+interval '1 minute' END)`, id, "staging/"+id.String(), status)
if err != nil {
t.Fatal(err)
}
@@ -35,7 +41,7 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
a, b := f.attachment(t, c, f.bob, "ready"), f.attachment(t, c, f.alice, "ready")
a, b := f.attachment(t, c, f.bob, "queued"), f.attachment(t, c, f.alice, "queued")
r := messageRouter(f, f.alice)
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a)
first := sendJSON(r, c, "attachments", body)
@@ -80,14 +86,46 @@ func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) {
}
}
func TestHTTPReplayAfterAttachmentFailsAndMembershipChanges(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
a := f.attachment(t, c, f.alice, "queued")
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, a)
router := messageRouter(f, f.alice)
first := sendJSON(router, c, "key", body)
if first.Code != http.StatusCreated {
t.Fatal(first.Code, first.Body)
}
if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state='failed',error_code='source_missing' WHERE attachment_id=$1`, a); err != nil {
t.Fatal(err)
}
replay := sendJSON(router, c, "key", body)
if replay.Code != http.StatusCreated || replay.Body.String() != first.Body.String() {
t.Fatalf("replay=%d %s, want original %s", replay.Code, replay.Body, first.Body)
}
changed := fmt.Sprintf(`{"text":"changed","attachment_ids":[%q]}`, ident.New())
if r := sendJSON(router, c, "key", changed); r.Code != http.StatusConflict {
t.Fatalf("changed body: %d %s", r.Code, r.Body)
}
if r := sendJSON(router, c, "new", body); r.Code != http.StatusConflict {
t.Fatalf("new message: %d %s", r.Code, r.Body)
}
if _, err := db.Pool.Exec(f.ctx, `DELETE FROM chat_members WHERE chat_id=$1 AND user_id=$2`, c, f.alice); err != nil {
t.Fatal(err)
}
if r := sendJSON(router, c, "key", body); r.Code != http.StatusNotFound {
t.Fatalf("former member: %d %s", r.Code, r.Body)
}
}
func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
other := f.newChat(t, f.alice)
ready := f.attachment(t, c, f.bob, "ready")
foreign := f.attachment(t, other, f.alice, "ready")
ready := f.attachment(t, c, f.bob, "queued")
foreign := f.attachment(t, other, f.alice, "queued")
failed := f.attachment(t, c, f.alice, "failed")
pending := f.attachment(t, c, f.alice, "pending")
pending := f.attachment(t, c, f.alice, "processing")
for _, tc := range []struct {
name string
actor, attachment uuid.UUID
@@ -96,13 +134,13 @@ func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) {
{"unknown", f.alice, ident.New(), http.StatusNotFound},
{"foreign", f.alice, foreign, http.StatusNotFound},
{"failed", f.alice, failed, http.StatusConflict},
{"pending", f.alice, pending, http.StatusConflict},
{"processing", f.alice, pending, http.StatusCreated},
{"outsider", f.eve, ready, http.StatusNotFound},
{"other-owner", f.alice, ready, http.StatusCreated},
} {
t.Run(tc.name, func(t *testing.T) {
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment)
rec := sendJSON(messageRouter(f, tc.actor), c, "same", body)
rec := sendJSON(messageRouter(f, tc.actor), c, tc.name, body)
if rec.Code != tc.want {
t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want)
}
+23 -14
View File
@@ -30,7 +30,12 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`,
hash[:], status, id, chatID, owner)
hash[:], "pending", id, chatID, owner)
if err != nil {
t.Fatal(err)
}
_, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code)
VALUES ($1,$2,$3,CASE WHEN $3='failed' THEN 'source_missing' END)`, id, "staging/"+id.String(), status)
if err != nil {
t.Fatal(err)
}
@@ -75,18 +80,18 @@ func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) {
}
func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) {
for _, status := range []string{"foreign", "failed", "pending", "mixed"} {
for _, status := range []string{"foreign", "failed", "mixed"} {
t.Run(status, func(t *testing.T) {
f := newFixture(t)
alice, bob := f.mustUser(t), f.mustUser(t)
c := f.mustChat(t, alice, bob)
other := f.mustChat(t, alice)
ready := f.attachment(t, c.ID, bob, "ready")
ready := f.attachment(t, c.ID, bob, "queued")
var invalid uuid.UUID
want := apperr.KindConflict
switch status {
case "foreign":
invalid = f.attachment(t, other.ID, alice, "ready")
invalid = f.attachment(t, other.ID, alice, "queued")
want = apperr.KindNotFound
case "mixed":
ready = f.attachment(t, c.ID, bob, "failed")
@@ -100,7 +105,7 @@ func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) {
t.Fatalf("validation: %v; want %v", err, want)
}
assertCounts(t, 0, 0, 0)
good := f.attachment(t, c.ID, bob, "ready")
good := f.attachment(t, c.ID, bob, "queued")
f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad")
assertCounts(t, 1, 1, 1)
})
@@ -111,7 +116,7 @@ func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) {
f := newFixture(t)
alice, bob := f.mustUser(t), f.mustUser(t)
c := f.mustChat(t, alice, bob)
a, b := f.attachment(t, c.ID, bob, "ready"), f.attachment(t, c.ID, bob, "ready")
a, b := f.attachment(t, c.ID, bob, "queued"), f.attachment(t, c.ID, bob, "queued")
first := attachedMessage(t, c.ID, alice, b, a)
got := f.appendOnce(t, first, "k")
if !reflect.DeepEqual(got, first) {
@@ -138,19 +143,23 @@ func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) {
}
}
func TestAppendOnceRevalidatesAttachmentOnReplay(t *testing.T) {
func TestAppendOnceReplaysAcceptedMessageAfterAttachmentFails(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
a := f.attachment(t, c.ID, alice, "ready")
a := f.attachment(t, c.ID, alice, "queued")
first := attachedMessage(t, c.ID, alice, a)
f.appendOnce(t, first, "k")
if _, err := db.Pool.Exec(f.ctx, `UPDATE files SET status = 'failed'`); err != nil {
if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state = 'failed',error_code='source_missing'`); err != nil {
t.Fatal(err)
}
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL)
got := f.appendOnce(t, attachedMessage(t, c.ID, alice, a), "k")
if !reflect.DeepEqual(got, first) {
t.Fatalf("replay = %+v, want original %+v", got, first)
}
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "new", keyTTL)
if err == nil || apperr.KindOf(err) != apperr.KindConflict {
t.Fatalf("failed replay attachment = %v, want conflict", err)
t.Fatalf("new message with failed attachment = %v, want conflict", err)
}
page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10)
if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) {
@@ -163,7 +172,7 @@ func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
a := f.attachment(t, c.ID, alice, "ready")
a := f.attachment(t, c.ID, alice, "queued")
// A database fault at the join boundary must not leave a key or message.
_, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`)
if err != nil {
@@ -184,7 +193,7 @@ func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
a, b := f.attachment(t, c.ID, alice, "ready"), f.attachment(t, c.ID, alice, "ready")
a, b := f.attachment(t, c.ID, alice, "queued"), f.attachment(t, c.ID, alice, "queued")
const requests = 20
results := make(chan chat.Message, requests)
start := make(chan struct{})
@@ -238,7 +247,7 @@ func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) {
c := f.mustChat(t, alice)
var ids []uuid.UUID
for range 10 {
ids = append(ids, f.attachment(t, c.ID, alice, "ready"))
ids = append(ids, f.attachment(t, c.ID, alice, "queued"))
}
cfg := db.Pool.Config()
counter := &queryCounter{}
+31 -18
View File
@@ -18,7 +18,6 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/chat"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
@@ -138,9 +137,6 @@ const claimSQL = `
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
var stored chat.Message
err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil {
return err
}
for {
if err := ctx.Err(); err != nil {
return err
@@ -151,6 +147,11 @@ func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, t
return fmt.Errorf("insert message: %w", err)
}
if tag.RowsAffected() == 1 {
// Only a new message creates attachment links. A replay returns the
// committed message even if its attachment has since failed.
if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil {
return err
}
if len(m.AttachmentIDs) > 0 {
_, err := tx.Exec(ctx, `
INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal)
@@ -192,27 +193,39 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
if len(ids) == 0 {
return nil
}
// Shared row locks prevent deletion or a readiness change until joins commit.
// Lock attachment identities in the same order for every message. Pipeline
// version changes lock the attachment before replacing its current job.
rows, err := tx.Query(ctx, `
SELECT f.status
FROM attachments a
JOIN files f ON f.hash = a.file_hash
WHERE a.chat_id = $1 AND a.id = ANY($2::uuid[])
ORDER BY a.id
FOR SHARE OF a, f`, chatID, ids)
SELECT id FROM attachments WHERE chat_id=$1 AND id=ANY($2::uuid[])
ORDER BY id FOR SHARE`, chatID, ids)
if err != nil {
return fmt.Errorf("check attachments: %w", err)
return fmt.Errorf("lock attachments: %w", err)
}
statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus])
locked, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])
if err != nil {
return fmt.Errorf("read attachment status: %w", err)
return err
}
if len(statuses) != len(ids) {
if len(locked) != len(ids) {
return apperr.NotFound("attachment not found")
}
for _, status := range statuses {
if status != attachment.FileReady {
return apperr.Conflict("attachment is not ready")
// A separate statement sees the latest committed transition after a lock wait.
// Every referenced attachment must have a current job.
rows, err = tx.Query(ctx, `SELECT status FROM attachment_states WHERE id=ANY($1::uuid[]) ORDER BY id FOR SHARE`, ids)
if err != nil {
return err
}
states, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return err
}
if len(states) != len(ids) {
return apperr.Conflict("attachment processing state unavailable")
}
for _, state := range states {
switch state {
case "queued", "processing", "ready":
default:
return apperr.Conflict("attachment processing failed")
}
}
return nil
+12 -1
View File
@@ -2,17 +2,20 @@ package config
import (
"fmt"
"math"
"strconv"
"time"
)
type S3 struct {
Endpoint, Region, Bucket string
AccessKey, SecretKey string
MaxBytes, MaxPixels int64
StagingRetention time.Duration
}
func LoadS3(lookup Lookup) (S3, error) {
cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000}
cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000, StagingRetention: 24 * time.Hour}
for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} {
if v, ok := lookup(key); ok {
*dest = v
@@ -38,6 +41,14 @@ func LoadS3(lookup Lookup) (S3, error) {
*p.dest = n
}
}
if raw, ok := lookup("STAGING_RETENTION_SECONDS"); ok {
const maxSeconds = math.MaxInt64 / int64(time.Second)
n, err := strconv.ParseInt(raw, 10, 64)
if err != nil || n < 1 || n > maxSeconds {
return S3{}, fmt.Errorf("STAGING_RETENTION_SECONDS must be between 1 and %d", maxSeconds)
}
cfg.StagingRetention = time.Duration(n) * time.Second
}
return cfg, nil
}
+19
View File
@@ -3,6 +3,7 @@ package config_test
import (
"messenger/internal/platform/config"
"testing"
"time"
)
func TestS3LimitsAndSecretFiles(t *testing.T) {
@@ -25,3 +26,21 @@ func TestS3LimitsAndSecretFiles(t *testing.T) {
}
}
}
func TestS3RejectsInvalidStagingRetention(t *testing.T) {
values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")}
cfg, err := config.LoadS3(env(values))
if err != nil || cfg.StagingRetention != 24*time.Hour {
t.Fatalf("default retention %v %v", cfg.StagingRetention, err)
}
values["STAGING_RETENTION_SECONDS"] = "7200"
cfg, err = config.LoadS3(env(values))
if err != nil || cfg.StagingRetention != 2*time.Hour {
t.Fatalf("custom retention %v %v", cfg.StagingRetention, err)
}
for _, value := range []string{"0", "-1", "abc", "9223372036854775807"} {
values["STAGING_RETENTION_SECONDS"] = value
if _, err := config.LoadS3(env(values)); err == nil {
t.Errorf("accepted invalid retention %s", value)
}
}
}
+38
View File
@@ -0,0 +1,38 @@
// Package faultbarrier provides one-shot synchronization for controlled process-failure experiments.
package faultbarrier
import (
"context"
"os"
"time"
)
// Wait claims an armed phase and blocks until its hit file disappears.
// A nonempty id identifies a newly staged attachment before its job exists.
func Wait(ctx context.Context, base, id string) error {
if err := os.Rename(base+".arm", base+".hit"); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if id != "" {
if err := os.WriteFile(base+".id", []byte(id), 0600); err != nil {
return err
}
}
t := time.NewTicker(50 * time.Millisecond)
defer t.Stop()
for {
if _, err := os.Stat(base + ".hit"); os.IsNotExist(err) {
return nil
} else if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
}
}
@@ -0,0 +1,48 @@
package faultbarrier
import (
"context"
"os"
"path/filepath"
"testing"
"time"
)
func TestArmedBarrierRecordsIDAndWaitsForRelease(t *testing.T) {
base := filepath.Join(t.TempDir(), "digest-staged")
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := Wait(ctx, base, "unused"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(base + ".id"); !os.IsNotExist(err) {
t.Fatal("unarmed barrier created identity")
}
if err := os.WriteFile(base+".arm", nil, 0600); err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() { done <- Wait(ctx, base, "attachment-id") }()
for {
b, err := os.ReadFile(base + ".id")
if err == nil && string(b) == "attachment-id" {
break
}
select {
case <-ctx.Done():
t.Fatal("missing staged identity")
case <-time.After(10 * time.Millisecond):
}
}
select {
case <-done:
t.Fatal("continued before release")
default:
}
if err := os.Remove(base + ".hit"); err != nil {
t.Fatal(err)
}
if err := <-done; err != nil {
t.Fatal(err)
}
}
+2 -2
View File
@@ -44,7 +44,7 @@ type S3 struct {
func StartS3(ctx context.Context) (*S3, error) {
useDockerDesktopPipe()
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()}
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab5-s3-test-" + uuid.NewString()}
keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey}
for _, key := range keys {
b := make([]byte, 24)
@@ -199,7 +199,7 @@ func (s *S3) Close() {
if s.container != nil {
_ = testcontainers.TerminateContainer(s.container)
}
if strings.HasPrefix(s.volume, "lab4-s3-test-") {
if strings.HasPrefix(s.volume, "lab5-s3-test-") {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) })
+1 -1
View File
@@ -182,7 +182,7 @@ func exercise(ctx context.Context, o options, out, progress io.Writer) error {
fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved")
// These resources were created above solely for this run; no compose
// endpoint or pre-existing database/bucket is ever accepted by this tool.
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files`); err != nil {
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files CASCADE`); err != nil {
return err
}
if err = emptyBucket(ctx, s3.Store); err != nil {