Переведите приём и чтение вложений на асинхронный контракт
This commit is contained in:
@@ -65,7 +65,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
defer db.Close()
|
||||
r, err := attachment.NewReconciler(db, blobs, *grace)
|
||||
r, err := attachment.NewReconcilerWithRetention(db, blobs, *grace, s3Config.StagingRetention)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "reconcile:", err)
|
||||
return 2
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
@@ -117,10 +119,10 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data)))
|
||||
ts.expect(res, 201, "")
|
||||
ts.expect(res, 202, "")
|
||||
a := decode[attachment.Response](t, res)
|
||||
digest := sha256.Sum256(data)
|
||||
if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "ready" || a.ErrorCode != nil || len(a.Variants) != 0 {
|
||||
if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "queued" || a.ErrorCode != nil || len(a.Variants) != 0 {
|
||||
t.Fatalf("attachment: %+v", a)
|
||||
}
|
||||
base := "/api/v1/attachments/" + a.ID
|
||||
@@ -129,11 +131,9 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
|
||||
}
|
||||
ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "")
|
||||
got := ts.binary("GET", base+"/content", bob.token, "", nil, 0)
|
||||
ts.expect(got, 200, "")
|
||||
if !bytes.Equal(got.raw, data) {
|
||||
t.Fatal("download bytes changed")
|
||||
}
|
||||
ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 400, "invalid_request")
|
||||
ts.expect(got, 409, "conflict")
|
||||
ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 409, "conflict")
|
||||
ts.expect(ts.do("GET", base+"/content?variant=unknown", bob.token, nil, nil), 400, "invalid_request")
|
||||
msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey())
|
||||
ts.expect(msg, 201, "")
|
||||
if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID {
|
||||
@@ -168,7 +168,7 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
|
||||
if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 201, "")
|
||||
ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 202, "")
|
||||
ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found")
|
||||
// An interrupted read is never a 500 or a successfully published identity.
|
||||
before := 0
|
||||
@@ -183,6 +183,76 @@ func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) {
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after {
|
||||
t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err)
|
||||
}
|
||||
// Complete one claimed job with known bytes. Image encoding has its own
|
||||
// tests; this checks the published version, exact download and ACL on S3.
|
||||
repo := pipeline.NewStore(db.Pool)
|
||||
var job pipeline.Job
|
||||
found, err := repo.PublishOne(ctx, func(_ context.Context, j pipeline.Job) error { job = j; return nil })
|
||||
if err != nil || !found || job.AttachmentID.String() != a.ID {
|
||||
t.Fatalf("outbox: %+v %v", job, err)
|
||||
}
|
||||
claim, err := repo.Claim(ctx, job, time.Minute)
|
||||
if err != nil || claim == nil {
|
||||
t.Fatal("claim", err)
|
||||
}
|
||||
meta := decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil))
|
||||
if meta.Status != "processing" || len(meta.Variants) != 0 {
|
||||
t.Fatal("processing snapshot", meta)
|
||||
}
|
||||
if err := s3.Store.Copy(ctx, claim.SourceKey, attachment.BlobKey(claim.Hash)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodies := map[string][]byte{"original": data}
|
||||
var outputs []pipeline.Variant
|
||||
for _, name := range []string{"thumbnail", "crop", "optimized"} {
|
||||
body := []byte("published-" + name)
|
||||
mime := "image/jpeg"
|
||||
if name == "optimized" {
|
||||
mime = "image/webp"
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
key := pipeline.VariantKey(claim.AttachmentID, claim.PipelineVersion, name, digest[:])
|
||||
if err := s3.Store.Put(ctx, key, bytes.NewReader(body), int64(len(body)), mime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputs = append(outputs, pipeline.Variant{Name: name, Key: key, MIMEType: mime, Width: 1, Height: 1, SizeBytes: int64(len(body)), SHA256: digest[:]})
|
||||
bodies[name] = body
|
||||
}
|
||||
if ok, err := repo.Complete(ctx, claim, outputs); err != nil || !ok {
|
||||
t.Fatal("publish", err)
|
||||
}
|
||||
if err := repo.Reprocess(ctx, claim.AttachmentID, 2, func(ctx context.Context, key string) (bool, error) {
|
||||
_, err := s3.Store.Stat(ctx, key)
|
||||
if errors.Is(err, attachment.ErrObjectMissing) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta = decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil))
|
||||
if meta.Status != "ready" || len(meta.Variants) != 3 || meta.ErrorCode != nil || meta.SHA256 != a.SHA256 {
|
||||
t.Fatalf("ready snapshot during reprocess: %+v", meta)
|
||||
}
|
||||
for _, v := range meta.Variants {
|
||||
digest := sha256.Sum256(bodies[v.Name])
|
||||
if v.SHA256 != hex.EncodeToString(digest[:]) || v.SizeBytes != int64(len(bodies[v.Name])) {
|
||||
t.Fatal("variant checksum", v)
|
||||
}
|
||||
}
|
||||
for name, body := range bodies {
|
||||
target := base + "/content?variant=" + name
|
||||
ts.expect(ts.do("GET", target, eve.token, nil, nil), 404, "not_found")
|
||||
got := ts.binary("GET", target, bob.token, "", nil, 0)
|
||||
ts.expect(got, 200, "")
|
||||
if !bytes.Equal(got.raw, body) {
|
||||
t.Fatalf("wrong published bytes for %s", name)
|
||||
}
|
||||
}
|
||||
if err := s3.Store.Delete(ctx, outputs[0].Key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts.expect(ts.do("GET", base+"/content?variant="+outputs[0].Name, bob.token, nil, nil), 404, "not_found")
|
||||
if err := s3.Stop(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -9,19 +9,23 @@ import (
|
||||
type FileStatus string
|
||||
|
||||
const (
|
||||
FilePending FileStatus = "pending"
|
||||
FileReady FileStatus = "ready"
|
||||
FileFailed FileStatus = "failed"
|
||||
FileQueued FileStatus = "queued"
|
||||
FileProcessing FileStatus = "processing"
|
||||
FilePending FileStatus = "pending"
|
||||
FileReady FileStatus = "ready"
|
||||
FileFailed FileStatus = "failed"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID, ChatID, OwnerID uuid.UUID
|
||||
Filename string
|
||||
FileHash []byte
|
||||
SizeBytes int64
|
||||
MIMEType string
|
||||
Status FileStatus
|
||||
CreatedAt time.Time
|
||||
ID, ChatID, OwnerID uuid.UUID
|
||||
Filename string
|
||||
FileHash []byte
|
||||
SizeBytes int64
|
||||
MIMEType string
|
||||
Status FileStatus
|
||||
CreatedAt, UpdatedAt time.Time
|
||||
ErrorCode *string
|
||||
Variants []Variant
|
||||
}
|
||||
|
||||
type UploadInput struct {
|
||||
@@ -30,3 +34,12 @@ type UploadInput struct {
|
||||
MIMEType string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string
|
||||
ObjectKey string
|
||||
MIMEType string
|
||||
SizeBytes int64
|
||||
Width, Height int
|
||||
Hash []byte
|
||||
}
|
||||
|
||||
@@ -26,22 +26,37 @@ func (h *Handler) Register(protected, binaryProtected gin.IRouter) {
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []VariantResponse `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type VariantResponse struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
// Lab4's shared response contract requires variants=[], even without processing.
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func toResponse(a Attachment) Response {
|
||||
variants := make([]VariantResponse, 0, len(a.Variants))
|
||||
for _, v := range a.Variants {
|
||||
variants = append(variants, VariantResponse{
|
||||
Name: v.Name, ContentType: v.MIMEType, SizeBytes: v.SizeBytes,
|
||||
Width: v.Width, Height: v.Height, SHA256: hex.EncodeToString(v.Hash),
|
||||
})
|
||||
}
|
||||
return Response{
|
||||
ID: a.ID.String(),
|
||||
ChatID: a.ChatID.String(),
|
||||
@@ -51,9 +66,10 @@ func toResponse(a Attachment) Response {
|
||||
SizeBytes: a.SizeBytes,
|
||||
SHA256: hex.EncodeToString(a.FileHash),
|
||||
Status: string(a.Status),
|
||||
Variants: []any{},
|
||||
Variants: variants,
|
||||
ErrorCode: a.ErrorCode,
|
||||
CreatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
UpdatedAt: httpx.Timestamp(a.CreatedAt),
|
||||
UpdatedAt: httpx.Timestamp(a.UpdatedAt),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +105,7 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
httpx.JSON(c, http.StatusCreated, toResponse(a))
|
||||
httpx.JSON(c, http.StatusAccepted, toResponse(a))
|
||||
}
|
||||
func attachmentID(c *gin.Context) (uuid.UUID, error) {
|
||||
id, err := ident.Parse(c.Param("attachment_id"))
|
||||
@@ -117,12 +133,21 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
variant, present := c.GetQuery("variant")
|
||||
if present && variant != "original" {
|
||||
httpx.Error(c, apperr.Invalid("variant must be original"))
|
||||
return
|
||||
variant := "original"
|
||||
if values, ok := c.Request.URL.Query()["variant"]; ok {
|
||||
if len(values) != 1 {
|
||||
httpx.Error(c, apperr.Invalid("one variant is required"))
|
||||
return
|
||||
}
|
||||
variant = values[0]
|
||||
switch variant {
|
||||
case "original", "thumbnail", "crop", "optimized":
|
||||
default:
|
||||
httpx.Error(c, apperr.Invalid("unknown variant"))
|
||||
return
|
||||
}
|
||||
}
|
||||
a, r, err := h.s.Content(c.Request.Context(), actor, id)
|
||||
a, r, err := h.s.Content(c.Request.Context(), actor, id, variant)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
|
||||
@@ -9,8 +9,11 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
@@ -40,9 +43,10 @@ type Stats struct {
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
db *pgxpool.Pool
|
||||
blobs Blobs
|
||||
grace time.Duration
|
||||
db *pgxpool.Pool
|
||||
blobs Blobs
|
||||
grace time.Duration
|
||||
retention time.Duration
|
||||
}
|
||||
|
||||
type ReconcileOptions struct {
|
||||
@@ -51,6 +55,13 @@ type ReconcileOptions struct {
|
||||
}
|
||||
|
||||
func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) {
|
||||
return NewReconcilerWithRetention(db, blobs, grace, 24*time.Hour)
|
||||
}
|
||||
|
||||
func NewReconcilerWithRetention(db *pgxpool.Pool, blobs Blobs, grace, retention time.Duration) (*Reconciler, error) {
|
||||
if retention < time.Second {
|
||||
return nil, errors.New("reconcile: staging retention must be at least one second")
|
||||
}
|
||||
if db == nil || blobs == nil {
|
||||
return nil, errors.New("reconcile: database and blobs are required")
|
||||
}
|
||||
@@ -60,7 +71,7 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci
|
||||
if grace < 2*time.Minute {
|
||||
return nil, errors.New("reconcile: grace must be at least two minutes")
|
||||
}
|
||||
return &Reconciler{db: db, blobs: blobs, grace: grace}, nil
|
||||
return &Reconciler{db: db, blobs: blobs, grace: grace, retention: retention}, nil
|
||||
}
|
||||
|
||||
// Run streams findings and keeps at most one page of metadata in memory.
|
||||
@@ -78,7 +89,7 @@ func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit fun
|
||||
stats: &stats,
|
||||
cutoff: started.Add(-r.grace),
|
||||
}
|
||||
for _, prefix := range []string{"staging/", "blobs/"} {
|
||||
for _, prefix := range []string{"staging/", "variants/", "blobs/"} {
|
||||
if err = run.objects(prefix); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
@@ -147,10 +158,10 @@ func (s *reconcileRun) objects(prefix string) error {
|
||||
continue
|
||||
}
|
||||
var finding *Finding
|
||||
if prefix == "staging/" {
|
||||
finding, err = s.staging(obj)
|
||||
} else {
|
||||
if prefix == "blobs/" {
|
||||
finding, err = s.orphan(obj, known[obj.Key])
|
||||
} else {
|
||||
finding, err = s.jobObject(prefix, obj)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile %s: %w", obj.Key, err)
|
||||
@@ -178,26 +189,76 @@ func (s *reconcileRun) delete(key string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *reconcileRun) staging(obj Object) (*Finding, error) {
|
||||
f := &Finding{Kind: "stale_staging", Key: obj.Key, Action: "delete_object"}
|
||||
// Failed sources remain available for operator replay until the retention period after failure elapses.
|
||||
// Staging shared by several pipeline versions is retained while any active job needs it.
|
||||
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
|
||||
parts := strings.Split(strings.TrimPrefix(obj.Key, prefix), "/")
|
||||
id, idErr := uuid.Parse(parts[0])
|
||||
kind := "stale_staging"
|
||||
query := `SELECT EXISTS(SELECT 1 FROM attachment_jobs
|
||||
WHERE attachment_id=$1 AND source_key=$2 AND
|
||||
(state IN ('queued','processing') OR (state='failed' AND updated_at >= now()-$3::bigint*interval '1 second')))`
|
||||
args := []any{id, obj.Key, int64(s.r.retention / time.Second)}
|
||||
if prefix == "variants/" {
|
||||
if len(parts) != 4 || idErr != nil {
|
||||
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
|
||||
}
|
||||
version, err := strconv.Atoi(parts[1])
|
||||
if err != nil || version < 1 || version > math.MaxInt32 {
|
||||
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
|
||||
}
|
||||
kind = "orphan_variant"
|
||||
query = `SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$3 AND object_key=$2)
|
||||
OR EXISTS(SELECT 1 FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=$3 AND state IN ('queued','processing'))`
|
||||
args[2] = version
|
||||
}
|
||||
finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"}
|
||||
var keep bool
|
||||
s.stats.SQLQueries++
|
||||
if err := s.r.db.QueryRow(s.ctx, query, args...).Scan(&keep); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keep {
|
||||
return nil, nil
|
||||
}
|
||||
if !s.apply {
|
||||
return f, nil
|
||||
return finding, nil
|
||||
}
|
||||
current, err := s.stat(obj.Key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.old(current.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
if err = s.delete(obj.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.Applied = true
|
||||
return f, nil
|
||||
keyHash := sha256.Sum256([]byte(obj.Key))
|
||||
return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
// Intake locks the staging key before inserting an attachment. Replay,
|
||||
// reprocess and completion lock the existing attachment before changing job state.
|
||||
if idErr == nil {
|
||||
s.stats.SQLQueries++
|
||||
if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
if err := tx.QueryRow(opCtx, query, args...).Scan(&keep); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keep {
|
||||
return nil, nil
|
||||
}
|
||||
op := *s
|
||||
op.ctx = opCtx
|
||||
current, err := op.stat(obj.Key)
|
||||
if errors.Is(err, ErrObjectMissing) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.old(current.Modified) {
|
||||
return nil, nil
|
||||
}
|
||||
if err := op.delete(obj.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finding.Applied = true
|
||||
return finding, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
|
||||
|
||||
@@ -729,3 +729,47 @@ func TestReconcileLockRejectsMalformedHash(t *testing.T) {
|
||||
t.Fatal("malformed hash accepted")
|
||||
}
|
||||
}
|
||||
func TestGCRespectsJobSourcesAndVersionedOutputs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("job input", "pending", 48*time.Hour, true, false)
|
||||
var id uuid.UUID
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stage := "staging/" + id.String()
|
||||
output := "variants/" + id.String() + "/1/thumbnail/" + strings.Repeat("0", 64)
|
||||
old := time.Now().Add(-48 * time.Hour)
|
||||
f.b.add(stage, []byte("job input"), old)
|
||||
f.b.add(output, []byte("partial"), old)
|
||||
f.exec(`INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, stage)
|
||||
for _, apply := range []bool{false, true} {
|
||||
stats, _, err := f.run(apply, false)
|
||||
if err != nil || stats.Findings != 0 || !f.b.has(stage) || !f.b.has(output) {
|
||||
t.Fatalf("active job lost: %+v %v", stats, err)
|
||||
}
|
||||
}
|
||||
f.exec(`UPDATE attachment_jobs SET state='failed', error_code='retry_exhausted',updated_at=now()`)
|
||||
stats, _, err := f.run(true, false)
|
||||
if err != nil || stats.Repaired != 1 || !f.b.has(stage) || f.b.has(output) {
|
||||
t.Fatalf("failed retention: %+v %v", stats, err)
|
||||
}
|
||||
f.exec(`UPDATE attachment_jobs SET updated_at=now()-interval '3 hours'`)
|
||||
stats, _, err = f.run(true, false)
|
||||
if err != nil || stats.Repaired != 0 || !f.b.has(stage) {
|
||||
t.Fatalf("default failed retention: %+v %v", stats, err)
|
||||
}
|
||||
r, err := attachment.NewReconcilerWithRetention(db.Pool, f.b, time.Hour, 2*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
if err != nil || stats.Repaired != 1 || f.b.has(stage) {
|
||||
t.Fatalf("expired source: %+v %v", stats, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,17 +7,11 @@ import (
|
||||
)
|
||||
|
||||
type AttachmentRepository interface {
|
||||
// WithPublication holds the file's hash lock through the callback and commit.
|
||||
// Publication is valid only inside the callback. Any callback error rolls back.
|
||||
WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error
|
||||
// Enqueue checks the staging object under the collector's lock before committing its job.
|
||||
Enqueue(ctx context.Context, attachment Attachment, stage string, checkStage func() error) error
|
||||
GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error)
|
||||
}
|
||||
|
||||
type Publication interface {
|
||||
FileReady() bool
|
||||
SaveReady(ctx context.Context) error
|
||||
}
|
||||
|
||||
type ChatMembership interface {
|
||||
IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error)
|
||||
}
|
||||
|
||||
@@ -3,14 +3,17 @@ package attachment
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/faultbarrier"
|
||||
"messenger/internal/platform/ident"
|
||||
)
|
||||
|
||||
@@ -23,8 +26,9 @@ type Service struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
BarrierDir string
|
||||
}
|
||||
|
||||
// A nil blobs dependency allows the text API to run without configured S3.
|
||||
@@ -70,7 +74,11 @@ func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput
|
||||
}
|
||||
id := ident.New()
|
||||
stage := "staging/" + id.String()
|
||||
discard := true
|
||||
defer func() {
|
||||
if !discard {
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
// A failed cleanup leaves a staging object for the reconciler's grace-based collection.
|
||||
@@ -119,37 +127,30 @@ func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput
|
||||
FileHash: hash.Sum(nil),
|
||||
SizeBytes: input.SizeBytes,
|
||||
MIMEType: input.MIMEType,
|
||||
Status: FileReady,
|
||||
Status: FileQueued,
|
||||
UpdatedAt: ident.TimeOf(id),
|
||||
CreatedAt: ident.TimeOf(id),
|
||||
}
|
||||
if err := s.attachments.WithPublication(ctx, a, func(p Publication) error {
|
||||
if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil {
|
||||
return err
|
||||
if s.config.BarrierDir != "" {
|
||||
base := filepath.Join(s.config.BarrierDir, hex.EncodeToString(a.FileHash)+"-staged")
|
||||
if err := faultbarrier.Wait(ctx, base, a.ID.String()); err != nil {
|
||||
return Attachment{}, unavailable(err)
|
||||
}
|
||||
return p.SaveReady(ctx)
|
||||
}); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error {
|
||||
blobKey := BlobKey(a.FileHash)
|
||||
if fileReady {
|
||||
obj, err := s.blobs.Stat(ctx, blobKey)
|
||||
if err == nil && obj.Size == a.SizeBytes {
|
||||
return nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrObjectMissing) {
|
||||
// COMMIT can succeed even if its reply is lost. From this point only the
|
||||
// collector, after consulting durable job state, may delete this source.
|
||||
discard = false
|
||||
err = s.attachments.Enqueue(ctx, a, stage, func() error {
|
||||
obj, err := s.blobs.Stat(ctx, stage)
|
||||
if err != nil {
|
||||
return unavailable(err)
|
||||
}
|
||||
}
|
||||
// The hash is known after streaming. Copy the validated staging object to
|
||||
// its permanent CAS key while the repository's publication lock is held.
|
||||
if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil {
|
||||
return unavailable(err)
|
||||
}
|
||||
return nil
|
||||
if obj.Size != a.SizeBytes {
|
||||
return unavailable(errors.New("staging size changed"))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return a, err
|
||||
}
|
||||
|
||||
func (s *Service) validate(ctx context.Context, stage, mime string) (err error) {
|
||||
@@ -189,25 +190,38 @@ func (r *storageReader) Read(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) {
|
||||
a, err := s.attachments.GetForMember(ctx, id, actor)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if a.Status != FileReady {
|
||||
return Attachment{}, apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
return a, nil
|
||||
return s.attachments.GetForMember(ctx, id, actor)
|
||||
}
|
||||
|
||||
func (s *Service) Content(ctx context.Context, actor, id uuid.UUID) (Attachment, io.ReadCloser, error) {
|
||||
func (s *Service) Content(ctx context.Context, actor, id uuid.UUID, name string) (Attachment, io.ReadCloser, error) {
|
||||
a, err := s.Get(ctx, actor, id)
|
||||
if err != nil {
|
||||
return Attachment{}, nil, err
|
||||
}
|
||||
if a.Status != FileReady {
|
||||
return Attachment{}, nil, apperr.Conflict("attachment is not ready")
|
||||
}
|
||||
key := BlobKey(a.FileHash)
|
||||
if name != "original" {
|
||||
found := false
|
||||
for _, v := range a.Variants {
|
||||
if name == v.Name {
|
||||
key, a.MIMEType, a.SizeBytes = v.ObjectKey, v.MIMEType, v.SizeBytes
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return Attachment{}, nil, apperr.NotFound("variant not found")
|
||||
}
|
||||
}
|
||||
if s.blobs == nil {
|
||||
return Attachment{}, nil, apperr.Unavailable("object storage is unavailable")
|
||||
}
|
||||
r, err := s.blobs.Open(ctx, BlobKey(a.FileHash))
|
||||
r, err := s.blobs.Open(ctx, key)
|
||||
if name != "original" && errors.Is(err, ErrObjectMissing) {
|
||||
return Attachment{}, nil, apperr.NotFound("variant not found")
|
||||
}
|
||||
if err != nil {
|
||||
return Attachment{}, nil, unavailable(err)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package attachment_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
@@ -20,159 +19,79 @@ import (
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
type testBlobs struct {
|
||||
mu sync.Mutex
|
||||
data map[string][]byte
|
||||
copies int
|
||||
failCopy bool
|
||||
}
|
||||
|
||||
type blockingPublication struct {
|
||||
*testBlobs
|
||||
entered, resume chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockingPublication) Copy(ctx context.Context, from, to string) error {
|
||||
close(b.entered)
|
||||
select {
|
||||
case <-b.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
return b.testBlobs.Copy(ctx, from, to)
|
||||
}
|
||||
func (b *blockingPublication) Stat(ctx context.Context, key string) (attachment.Object, error) {
|
||||
o, e := b.testBlobs.Stat(ctx, key)
|
||||
o.Modified = time.Now().Add(-2 * time.Hour)
|
||||
return o, e
|
||||
}
|
||||
|
||||
func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
func TestCollectorRechecksStageAfterConcurrentEnqueue(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
actor, room := ident.New(), ident.New()
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'race_user','User','hash',now())`, actor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'race',now())`, room); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, room, actor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := reconcileSetup(t, db)
|
||||
f.exec(`INSERT INTO chat_members VALUES ($1,$2)`, f.chat, f.user)
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := sha256.Sum256(data)
|
||||
if _, err = db.Pool.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png','pending',now()-interval '2 hours')`, hash[:], len(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
entered := make(chan string, 1)
|
||||
resume := make(chan struct{})
|
||||
var once sync.Once
|
||||
f.b.stating = func(key string) {
|
||||
once.Do(func() {
|
||||
f.b.add(key, data, time.Now().Add(-2*time.Hour))
|
||||
entered <- key
|
||||
select {
|
||||
case <-resume:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
})
|
||||
}
|
||||
b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), f.b, attachment.Config{})
|
||||
uploaded := make(chan error, 1)
|
||||
go func() {
|
||||
input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
_, e := s.Upload(ctx, actor, input, bytes.NewReader(data))
|
||||
uploaded <- e
|
||||
_, err := s.Upload(ctx, f.user, attachment.UploadInput{ChatID: f.chat, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}, bytes.NewReader(data))
|
||||
uploaded <- err
|
||||
}()
|
||||
var key string
|
||||
select {
|
||||
case <-b.entered:
|
||||
case key = <-entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("publisher never reached copy")
|
||||
t.Fatal("upload never verified staging under lock")
|
||||
}
|
||||
r, err := attachment.NewReconciler(db.Pool, b, time.Hour)
|
||||
r, err := attachment.NewReconciler(db.Pool, f.b, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collected := make(chan error, 1)
|
||||
collect := make(chan error, 1)
|
||||
go func() {
|
||||
stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
if e == nil && stats.Repaired != 0 {
|
||||
e = errors.New("collector removed a newly referenced file")
|
||||
stats, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
if err == nil && stats.Repaired != 0 {
|
||||
err = errors.New("removed concurrent upload")
|
||||
}
|
||||
collected <- e
|
||||
collect <- err
|
||||
}()
|
||||
// Observe a real waiting advisory lock, rather than guessing which goroutine won.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
waiting := false
|
||||
for time.Now().Before(deadline) {
|
||||
var n int
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n > 0 {
|
||||
waiting = true
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
close(b.resume)
|
||||
if !waiting {
|
||||
t.Fatal("collector did not serialize with the publisher")
|
||||
}
|
||||
waitReconcileLock(t, ctx, db.Pool)
|
||||
close(resume)
|
||||
if err := <-uploaded; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-collected; err != nil {
|
||||
if err := <-collect; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.Stat(ctx, attachment.BlobKey(hash[:])); err != nil {
|
||||
t.Fatal("collector deleted published data", err)
|
||||
if !f.b.has(key) {
|
||||
t.Fatal("accepted source lost")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *testBlobs) Put(_ context.Context, k string, r io.Reader, n int64, m string) error {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
type lostCommitReply struct {
|
||||
attachment.AttachmentRepository
|
||||
}
|
||||
|
||||
func (r lostCommitReply) Enqueue(ctx context.Context, a attachment.Attachment, stage string, check func() error) error {
|
||||
if err := r.AttachmentRepository.Enqueue(ctx, a, stage, check); err != nil {
|
||||
return err
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.data[k] = data
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) Open(_ context.Context, k string) (io.ReadCloser, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
data, ok := b.data[k]
|
||||
if !ok {
|
||||
return nil, attachment.ErrObjectMissing
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
func (b *testBlobs) Stat(_ context.Context, k string) (attachment.Object, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
data, ok := b.data[k]
|
||||
if !ok {
|
||||
return attachment.Object{}, attachment.ErrObjectMissing
|
||||
}
|
||||
return attachment.Object{Key: k, Size: int64(len(data)), Modified: time.Now()}, nil
|
||||
}
|
||||
func (b *testBlobs) Copy(_ context.Context, from, to string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.failCopy {
|
||||
return errors.New("copy outage")
|
||||
}
|
||||
b.copies++
|
||||
b.data[to] = bytes.Clone(b.data[from])
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) Delete(_ context.Context, k string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.data, k)
|
||||
return nil
|
||||
}
|
||||
func (b *testBlobs) List(context.Context, string, string, int) ([]attachment.Object, error) {
|
||||
return nil, nil
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
@@ -197,7 +116,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b := &testBlobs{data: map[string][]byte{}}
|
||||
b := newReconcileBlobs()
|
||||
s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{})
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
@@ -220,13 +139,16 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
var files, atts int
|
||||
err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments)`).Scan(&files, &atts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
var files, atts, jobs int
|
||||
err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments),
|
||||
(SELECT count(*) FROM attachment_jobs WHERE state='queued' AND attempt=1 AND published_at IS NULL)`).Scan(&files, &atts, &jobs)
|
||||
if err != nil || files != 1 || atts != 10 || jobs != 10 || len(b.objects) != 10 {
|
||||
t.Fatalf("handoff: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err)
|
||||
}
|
||||
if files != 1 || atts != 10 || b.copies != 1 || len(b.data) != 1 {
|
||||
t.Fatalf("files=%d attachments=%d copies=%d objects=%d", files, atts, b.copies, len(b.data))
|
||||
for _, a := range results {
|
||||
if a.Status != attachment.FileQueued || !bytes.Equal(b.bodies["staging/"+a.ID.String()], data) {
|
||||
t.Fatal("missing queued source", a.ID)
|
||||
}
|
||||
}
|
||||
if results[0].ID == results[1].ID {
|
||||
t.Fatal("deduplication merged chat ACL identity")
|
||||
@@ -234,43 +156,26 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) {
|
||||
if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound {
|
||||
t.Fatalf("outsider got %v", err)
|
||||
}
|
||||
// A removed blob is republished from the new upload, preserving old references.
|
||||
b.mu.Lock()
|
||||
clear(b.data)
|
||||
b.mu.Unlock()
|
||||
input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
input := attachment.UploadInput{ChatID: c1, Filename: "lost.png", MIMEType: "image/png", SizeBytes: int64(len(data))}
|
||||
lost := attachment.NewService(lostCommitReply{store.NewAttachments(db.Pool)}, chatstore.NewChats(db.Pool), b, attachment.Config{})
|
||||
if _, err := lost.Upload(ctx, actor, input, bytes.NewReader(data)); !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
t.Fatal("lost commit reply:", err)
|
||||
}
|
||||
if b.copies != 2 {
|
||||
t.Fatalf("missing ready blob was not restored: copies=%d", b.copies)
|
||||
var source string
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT j.source_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.filename='lost.png'`).Scan(&source); err != nil || !bytes.Equal(b.bodies[source], data) {
|
||||
t.Fatal("committed job lost source", err)
|
||||
}
|
||||
b.failCopy = true
|
||||
other := bytes.Clone(data)
|
||||
other = append(other, 0)
|
||||
input.Filename, input.SizeBytes = "failed.png", int64(len(other))
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable {
|
||||
t.Fatalf("copy outage: %v", err)
|
||||
}
|
||||
if len(b.data) != 1 {
|
||||
t.Fatalf("failed publication left staging: %d objects", len(b.data))
|
||||
}
|
||||
b.failCopy = false
|
||||
// S3 publication succeeds, then PostgreSQL refuses the logical attachment.
|
||||
// Compensation may remove staging but must leave shared CAS data intact.
|
||||
_, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$;
|
||||
CREATE TRIGGER reject_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_attachment()`)
|
||||
_, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_job() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$;
|
||||
CREATE TRIGGER reject_job BEFORE INSERT ON attachment_jobs FOR EACH ROW EXECUTE FUNCTION reject_job()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.Filename = "rollback.png"
|
||||
other := append(bytes.Clone(data), 0)
|
||||
input.Filename, input.SizeBytes = "rollback.png", int64(len(other))
|
||||
if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil {
|
||||
t.Fatal("publication survived rejected metadata")
|
||||
t.Fatal("accepted rejected job")
|
||||
}
|
||||
if len(b.data) != 2 {
|
||||
t.Fatalf("expected original and recoverable orphan, got %d objects", len(b.data))
|
||||
}
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&files); err != nil || files != 1 {
|
||||
t.Fatalf("failed DB insert committed file metadata: %d %v", files, err)
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments),(SELECT count(*) FROM attachment_jobs)`).Scan(&files, &atts, &jobs); err != nil || files != 1 || atts != 11 || jobs != 11 || len(b.objects) != 12 {
|
||||
t.Fatalf("rollback: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,7 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"crypto/sha256"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -14,78 +13,77 @@ import (
|
||||
"messenger/internal/platform/postgres"
|
||||
)
|
||||
|
||||
type Attachments struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
type Attachments struct{ pool *pgxpool.Pool }
|
||||
|
||||
var _ attachment.AttachmentRepository = (*Attachments)(nil)
|
||||
|
||||
func NewAttachments(pool *pgxpool.Pool) *Attachments {
|
||||
return &Attachments{pool: pool}
|
||||
}
|
||||
func NewAttachments(pool *pgxpool.Pool) *Attachments { return &Attachments{pool: pool} }
|
||||
|
||||
// Publication and garbage collection use the same hash lock. The transaction
|
||||
// spans only publication, never client transfer or image decoding.
|
||||
func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error {
|
||||
func (r *Attachments) Enqueue(ctx context.Context, a attachment.Attachment, stage string, checkStage func() error) error {
|
||||
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil {
|
||||
// A request can pause after validation for longer than the GC grace.
|
||||
// Verify the source after serializing with deletion of this exact key.
|
||||
stageHash := sha256.Sum256([]byte(stage))
|
||||
if err := attachment.LockHash(ctx, tx, stageHash[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkStage(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil {
|
||||
return err
|
||||
}
|
||||
var status attachment.FileStatus
|
||||
err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ($1,$2,$3,'pending')
|
||||
ON CONFLICT (hash) DO NOTHING`, a.FileHash, a.SizeBytes, a.MIMEType); err != nil {
|
||||
return err
|
||||
}
|
||||
return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, a.ID, stage)
|
||||
return err
|
||||
}))
|
||||
}
|
||||
|
||||
type publication struct {
|
||||
tx pgx.Tx
|
||||
attachment attachment.Attachment
|
||||
ready bool
|
||||
}
|
||||
|
||||
func (p publication) FileReady() bool { return p.ready }
|
||||
|
||||
func (p publication) SaveReady(ctx context.Context) error {
|
||||
a := p.attachment
|
||||
_, err := p.tx.Exec(ctx, `
|
||||
INSERT INTO files (hash, size_bytes, mime_type, status)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (hash) DO UPDATE
|
||||
SET size_bytes = excluded.size_bytes,
|
||||
mime_type = excluded.mime_type,
|
||||
status = excluded.status`,
|
||||
a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish file: %w", err)
|
||||
}
|
||||
_, err = p.tx.Exec(ctx, `
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) {
|
||||
var a attachment.Attachment
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash,
|
||||
f.size_bytes, f.mime_type, f.status, a.created_at
|
||||
// A single statement returns one snapshot of the published version, even
|
||||
// when a worker completes a newer version during a download.
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash, f.size_bytes, f.mime_type,
|
||||
s.status, s.error_code, a.created_at, s.updated_at,
|
||||
coalesce(v.variant,''),coalesce(v.object_key,''),coalesce(v.mime_type,''),
|
||||
coalesce(v.size_bytes,0),coalesce(v.width,0),coalesce(v.height,0),v.sha256
|
||||
FROM attachments a
|
||||
JOIN files f ON f.hash = a.file_hash
|
||||
WHERE a.id = $1 AND EXISTS (
|
||||
SELECT 1 FROM chat_members m
|
||||
WHERE m.chat_id = a.chat_id AND m.user_id = $2
|
||||
)`, id, actor).
|
||||
Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash,
|
||||
&a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return attachment.Attachment{}, apperr.NotFound("attachment not found")
|
||||
JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2
|
||||
JOIN files f ON f.hash=a.file_hash
|
||||
JOIN attachment_states s ON s.id=a.id
|
||||
LEFT JOIN attachment_variants v ON v.attachment_id=a.id AND v.pipeline_version=a.published_version AND s.status='ready'
|
||||
WHERE a.id=$1 ORDER BY v.variant`, id, actor)
|
||||
if err != nil {
|
||||
return a, postgres.Classify(err)
|
||||
}
|
||||
return a, postgres.Classify(err)
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v attachment.Variant
|
||||
err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType,
|
||||
&a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
|
||||
&v.Name, &v.ObjectKey, &v.MIMEType, &v.SizeBytes, &v.Width, &v.Height, &v.Hash)
|
||||
if err != nil {
|
||||
return attachment.Attachment{}, postgres.Classify(err)
|
||||
}
|
||||
if v.Name != "" {
|
||||
a.Variants = append(a.Variants, v)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return attachment.Attachment{}, postgres.Classify(err)
|
||||
}
|
||||
if a.ID == uuid.Nil {
|
||||
return a, apperr.NotFound("attachment not found")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
@@ -25,7 +25,13 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
|
||||
INSERT INTO files (hash, size_bytes, mime_type, status, created_at)
|
||||
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], status, id, chatID, owner)
|
||||
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], "pending", id, chatID, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code,lease_token,lease_until)
|
||||
VALUES($1,$2,$3,CASE WHEN $3='failed' THEN 'processing_failed' END,CASE WHEN $3='processing' THEN gen_random_uuid() END,
|
||||
CASE WHEN $3='processing' THEN now()+interval '1 minute' END)`, id, "staging/"+id.String(), status)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -35,7 +41,7 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
|
||||
func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
a, b := f.attachment(t, c, f.bob, "ready"), f.attachment(t, c, f.alice, "ready")
|
||||
a, b := f.attachment(t, c, f.bob, "queued"), f.attachment(t, c, f.alice, "queued")
|
||||
r := messageRouter(f, f.alice)
|
||||
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a)
|
||||
first := sendJSON(r, c, "attachments", body)
|
||||
@@ -80,14 +86,46 @@ func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPReplayAfterAttachmentFailsAndMembershipChanges(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
a := f.attachment(t, c, f.alice, "queued")
|
||||
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, a)
|
||||
router := messageRouter(f, f.alice)
|
||||
first := sendJSON(router, c, "key", body)
|
||||
if first.Code != http.StatusCreated {
|
||||
t.Fatal(first.Code, first.Body)
|
||||
}
|
||||
if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state='failed',error_code='source_missing' WHERE attachment_id=$1`, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay := sendJSON(router, c, "key", body)
|
||||
if replay.Code != http.StatusCreated || replay.Body.String() != first.Body.String() {
|
||||
t.Fatalf("replay=%d %s, want original %s", replay.Code, replay.Body, first.Body)
|
||||
}
|
||||
changed := fmt.Sprintf(`{"text":"changed","attachment_ids":[%q]}`, ident.New())
|
||||
if r := sendJSON(router, c, "key", changed); r.Code != http.StatusConflict {
|
||||
t.Fatalf("changed body: %d %s", r.Code, r.Body)
|
||||
}
|
||||
if r := sendJSON(router, c, "new", body); r.Code != http.StatusConflict {
|
||||
t.Fatalf("new message: %d %s", r.Code, r.Body)
|
||||
}
|
||||
if _, err := db.Pool.Exec(f.ctx, `DELETE FROM chat_members WHERE chat_id=$1 AND user_id=$2`, c, f.alice); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := sendJSON(router, c, "key", body); r.Code != http.StatusNotFound {
|
||||
t.Fatalf("former member: %d %s", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
other := f.newChat(t, f.alice)
|
||||
ready := f.attachment(t, c, f.bob, "ready")
|
||||
foreign := f.attachment(t, other, f.alice, "ready")
|
||||
ready := f.attachment(t, c, f.bob, "queued")
|
||||
foreign := f.attachment(t, other, f.alice, "queued")
|
||||
failed := f.attachment(t, c, f.alice, "failed")
|
||||
pending := f.attachment(t, c, f.alice, "pending")
|
||||
pending := f.attachment(t, c, f.alice, "processing")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
actor, attachment uuid.UUID
|
||||
@@ -96,13 +134,13 @@ func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) {
|
||||
{"unknown", f.alice, ident.New(), http.StatusNotFound},
|
||||
{"foreign", f.alice, foreign, http.StatusNotFound},
|
||||
{"failed", f.alice, failed, http.StatusConflict},
|
||||
{"pending", f.alice, pending, http.StatusConflict},
|
||||
{"processing", f.alice, pending, http.StatusCreated},
|
||||
{"outsider", f.eve, ready, http.StatusNotFound},
|
||||
{"other-owner", f.alice, ready, http.StatusCreated},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment)
|
||||
rec := sendJSON(messageRouter(f, tc.actor), c, "same", body)
|
||||
rec := sendJSON(messageRouter(f, tc.actor), c, tc.name, body)
|
||||
if rec.Code != tc.want {
|
||||
t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want)
|
||||
}
|
||||
|
||||
@@ -30,7 +30,12 @@ func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string
|
||||
VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash)
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at)
|
||||
SELECT $3, $4, $5, 'image.png', hash, now() FROM file`,
|
||||
hash[:], status, id, chatID, owner)
|
||||
hash[:], "pending", id, chatID, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code)
|
||||
VALUES ($1,$2,$3,CASE WHEN $3='failed' THEN 'source_missing' END)`, id, "staging/"+id.String(), status)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -75,18 +80,18 @@ func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) {
|
||||
for _, status := range []string{"foreign", "failed", "pending", "mixed"} {
|
||||
for _, status := range []string{"foreign", "failed", "mixed"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice, bob := f.mustUser(t), f.mustUser(t)
|
||||
c := f.mustChat(t, alice, bob)
|
||||
other := f.mustChat(t, alice)
|
||||
ready := f.attachment(t, c.ID, bob, "ready")
|
||||
ready := f.attachment(t, c.ID, bob, "queued")
|
||||
var invalid uuid.UUID
|
||||
want := apperr.KindConflict
|
||||
switch status {
|
||||
case "foreign":
|
||||
invalid = f.attachment(t, other.ID, alice, "ready")
|
||||
invalid = f.attachment(t, other.ID, alice, "queued")
|
||||
want = apperr.KindNotFound
|
||||
case "mixed":
|
||||
ready = f.attachment(t, c.ID, bob, "failed")
|
||||
@@ -100,7 +105,7 @@ func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) {
|
||||
t.Fatalf("validation: %v; want %v", err, want)
|
||||
}
|
||||
assertCounts(t, 0, 0, 0)
|
||||
good := f.attachment(t, c.ID, bob, "ready")
|
||||
good := f.attachment(t, c.ID, bob, "queued")
|
||||
f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad")
|
||||
assertCounts(t, 1, 1, 1)
|
||||
})
|
||||
@@ -111,7 +116,7 @@ func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice, bob := f.mustUser(t), f.mustUser(t)
|
||||
c := f.mustChat(t, alice, bob)
|
||||
a, b := f.attachment(t, c.ID, bob, "ready"), f.attachment(t, c.ID, bob, "ready")
|
||||
a, b := f.attachment(t, c.ID, bob, "queued"), f.attachment(t, c.ID, bob, "queued")
|
||||
first := attachedMessage(t, c.ID, alice, b, a)
|
||||
got := f.appendOnce(t, first, "k")
|
||||
if !reflect.DeepEqual(got, first) {
|
||||
@@ -138,19 +143,23 @@ func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceRevalidatesAttachmentOnReplay(t *testing.T) {
|
||||
func TestAppendOnceReplaysAcceptedMessageAfterAttachmentFails(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a := f.attachment(t, c.ID, alice, "ready")
|
||||
a := f.attachment(t, c.ID, alice, "queued")
|
||||
first := attachedMessage(t, c.ID, alice, a)
|
||||
f.appendOnce(t, first, "k")
|
||||
if _, err := db.Pool.Exec(f.ctx, `UPDATE files SET status = 'failed'`); err != nil {
|
||||
if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state = 'failed',error_code='source_missing'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL)
|
||||
got := f.appendOnce(t, attachedMessage(t, c.ID, alice, a), "k")
|
||||
if !reflect.DeepEqual(got, first) {
|
||||
t.Fatalf("replay = %+v, want original %+v", got, first)
|
||||
}
|
||||
_, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "new", keyTTL)
|
||||
if err == nil || apperr.KindOf(err) != apperr.KindConflict {
|
||||
t.Fatalf("failed replay attachment = %v, want conflict", err)
|
||||
t.Fatalf("new message with failed attachment = %v, want conflict", err)
|
||||
}
|
||||
page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10)
|
||||
if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) {
|
||||
@@ -163,7 +172,7 @@ func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a := f.attachment(t, c.ID, alice, "ready")
|
||||
a := f.attachment(t, c.ID, alice, "queued")
|
||||
// A database fault at the join boundary must not leave a key or message.
|
||||
_, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`)
|
||||
if err != nil {
|
||||
@@ -184,7 +193,7 @@ func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
a, b := f.attachment(t, c.ID, alice, "ready"), f.attachment(t, c.ID, alice, "ready")
|
||||
a, b := f.attachment(t, c.ID, alice, "queued"), f.attachment(t, c.ID, alice, "queued")
|
||||
const requests = 20
|
||||
results := make(chan chat.Message, requests)
|
||||
start := make(chan struct{})
|
||||
@@ -238,7 +247,7 @@ func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) {
|
||||
c := f.mustChat(t, alice)
|
||||
var ids []uuid.UUID
|
||||
for range 10 {
|
||||
ids = append(ids, f.attachment(t, c.ID, alice, "ready"))
|
||||
ids = append(ids, f.attachment(t, c.ID, alice, "queued"))
|
||||
}
|
||||
cfg := db.Pool.Config()
|
||||
counter := &queryCounter{}
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
@@ -138,9 +137,6 @@ const claimSQL = `
|
||||
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
|
||||
var stored chat.Message
|
||||
err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
|
||||
if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@@ -151,6 +147,11 @@ func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, t
|
||||
return fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
// Only a new message creates attachment links. A replay returns the
|
||||
// committed message even if its attachment has since failed.
|
||||
if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(m.AttachmentIDs) > 0 {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal)
|
||||
@@ -192,27 +193,39 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Shared row locks prevent deletion or a readiness change until joins commit.
|
||||
// Lock attachment identities in the same order for every message. Pipeline
|
||||
// version changes lock the attachment before replacing its current job.
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT f.status
|
||||
FROM attachments a
|
||||
JOIN files f ON f.hash = a.file_hash
|
||||
WHERE a.chat_id = $1 AND a.id = ANY($2::uuid[])
|
||||
ORDER BY a.id
|
||||
FOR SHARE OF a, f`, chatID, ids)
|
||||
SELECT id FROM attachments WHERE chat_id=$1 AND id=ANY($2::uuid[])
|
||||
ORDER BY id FOR SHARE`, chatID, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check attachments: %w", err)
|
||||
return fmt.Errorf("lock attachments: %w", err)
|
||||
}
|
||||
statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus])
|
||||
locked, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])
|
||||
if err != nil {
|
||||
return fmt.Errorf("read attachment status: %w", err)
|
||||
return err
|
||||
}
|
||||
if len(statuses) != len(ids) {
|
||||
if len(locked) != len(ids) {
|
||||
return apperr.NotFound("attachment not found")
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status != attachment.FileReady {
|
||||
return apperr.Conflict("attachment is not ready")
|
||||
// A separate statement sees the latest committed transition after a lock wait.
|
||||
// Every referenced attachment must have a current job.
|
||||
rows, err = tx.Query(ctx, `SELECT status FROM attachment_states WHERE id=ANY($1::uuid[]) ORDER BY id FOR SHARE`, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
states, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(states) != len(ids) {
|
||||
return apperr.Conflict("attachment processing state unavailable")
|
||||
}
|
||||
for _, state := range states {
|
||||
switch state {
|
||||
case "queued", "processing", "ready":
|
||||
default:
|
||||
return apperr.Conflict("attachment processing failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -2,17 +2,20 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
Endpoint, Region, Bucket string
|
||||
AccessKey, SecretKey string
|
||||
MaxBytes, MaxPixels int64
|
||||
StagingRetention time.Duration
|
||||
}
|
||||
|
||||
func LoadS3(lookup Lookup) (S3, error) {
|
||||
cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000}
|
||||
cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000, StagingRetention: 24 * time.Hour}
|
||||
for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} {
|
||||
if v, ok := lookup(key); ok {
|
||||
*dest = v
|
||||
@@ -38,6 +41,14 @@ func LoadS3(lookup Lookup) (S3, error) {
|
||||
*p.dest = n
|
||||
}
|
||||
}
|
||||
if raw, ok := lookup("STAGING_RETENTION_SECONDS"); ok {
|
||||
const maxSeconds = math.MaxInt64 / int64(time.Second)
|
||||
n, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil || n < 1 || n > maxSeconds {
|
||||
return S3{}, fmt.Errorf("STAGING_RETENTION_SECONDS must be between 1 and %d", maxSeconds)
|
||||
}
|
||||
cfg.StagingRetention = time.Duration(n) * time.Second
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package config_test
|
||||
import (
|
||||
"messenger/internal/platform/config"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestS3LimitsAndSecretFiles(t *testing.T) {
|
||||
@@ -25,3 +26,21 @@ func TestS3LimitsAndSecretFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestS3RejectsInvalidStagingRetention(t *testing.T) {
|
||||
values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")}
|
||||
cfg, err := config.LoadS3(env(values))
|
||||
if err != nil || cfg.StagingRetention != 24*time.Hour {
|
||||
t.Fatalf("default retention %v %v", cfg.StagingRetention, err)
|
||||
}
|
||||
values["STAGING_RETENTION_SECONDS"] = "7200"
|
||||
cfg, err = config.LoadS3(env(values))
|
||||
if err != nil || cfg.StagingRetention != 2*time.Hour {
|
||||
t.Fatalf("custom retention %v %v", cfg.StagingRetention, err)
|
||||
}
|
||||
for _, value := range []string{"0", "-1", "abc", "9223372036854775807"} {
|
||||
values["STAGING_RETENTION_SECONDS"] = value
|
||||
if _, err := config.LoadS3(env(values)); err == nil {
|
||||
t.Errorf("accepted invalid retention %s", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// Package faultbarrier provides one-shot synchronization for controlled process-failure experiments.
|
||||
package faultbarrier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Wait claims an armed phase and blocks until its hit file disappears.
|
||||
// A nonempty id identifies a newly staged attachment before its job exists.
|
||||
func Wait(ctx context.Context, base, id string) error {
|
||||
if err := os.Rename(base+".arm", base+".hit"); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if id != "" {
|
||||
if err := os.WriteFile(base+".id", []byte(id), 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
t := time.NewTicker(50 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
if _, err := os.Stat(base + ".hit"); os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package faultbarrier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestArmedBarrierRecordsIDAndWaitsForRelease(t *testing.T) {
|
||||
base := filepath.Join(t.TempDir(), "digest-staged")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := Wait(ctx, base, "unused"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(base + ".id"); !os.IsNotExist(err) {
|
||||
t.Fatal("unarmed barrier created identity")
|
||||
}
|
||||
if err := os.WriteFile(base+".arm", nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- Wait(ctx, base, "attachment-id") }()
|
||||
for {
|
||||
b, err := os.ReadFile(base + ".id")
|
||||
if err == nil && string(b) == "attachment-id" {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("missing staged identity")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("continued before release")
|
||||
default:
|
||||
}
|
||||
if err := os.Remove(base + ".hit"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,7 @@ type S3 struct {
|
||||
|
||||
func StartS3(ctx context.Context) (*S3, error) {
|
||||
useDockerDesktopPipe()
|
||||
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()}
|
||||
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab5-s3-test-" + uuid.NewString()}
|
||||
keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey}
|
||||
for _, key := range keys {
|
||||
b := make([]byte, 24)
|
||||
@@ -199,7 +199,7 @@ func (s *S3) Close() {
|
||||
if s.container != nil {
|
||||
_ = testcontainers.TerminateContainer(s.container)
|
||||
}
|
||||
if strings.HasPrefix(s.volume, "lab4-s3-test-") {
|
||||
if strings.HasPrefix(s.volume, "lab5-s3-test-") {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) })
|
||||
|
||||
@@ -182,7 +182,7 @@ func exercise(ctx context.Context, o options, out, progress io.Writer) error {
|
||||
fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved")
|
||||
// These resources were created above solely for this run; no compose
|
||||
// endpoint or pre-existing database/bucket is ever accepted by this tool.
|
||||
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files`); err != nil {
|
||||
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files CASCADE`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = emptyBucket(ctx, s3.Store); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user