Files
mtusi2026lab6/internal/store/postgres/pipeline_integration_test.go

188 lines
5.8 KiB
Go

package postgres
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5"
"go.uber.org/zap"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/imageprocessor"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/streamqueue"
)
type faultStore struct {
*Store
before bool
after bool
}
// Complete injects a lost connection before or after the actual SQL commit.
func (s *faultStore) Complete(ctx context.Context, l pipeline.Lease, key string, variants []dto.Variant) (bool, error) {
if s.before {
s.before = false
return false, errors.New("injected connection loss before commit")
}
done, err := s.Store.Complete(ctx, l, key, variants)
if err == nil && s.after {
s.after = false
return done, errors.New("injected lost commit acknowledgement")
}
return done, err
}
// TestRealPipelineFaultWindows verifies real objects and SQL remain correct across uncertain commits.
func TestRealPipelineFaultWindows(t *testing.T) {
for _, fault := range []struct {
name string
before bool
after bool
}{{"after_S3_before_commit", true, false}, {"after_commit_before_ack", false, true}} {
t.Run(fault.name, func(t *testing.T) {
if os.Getenv("S3_ENDPOINT") == "" || os.Getenv("BROKER_ADDRESS") == "" {
t.Skip("isolated S3 and broker required")
}
s := testStore(t)
a := fixtureAttachment(t, s)
data, err := os.ReadFile("../../../tests/fixtures/sample.png")
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(data)
a.SHA256 = hex.EncodeToString(sum[:])
a.SizeBytes = int64(len(data))
_, err = s.pool.Exec(t.Context(), `UPDATE attachments SET size_bytes=@size,sha256=@sha WHERE id=@id`, pgx.StrictNamedArgs{"size": a.SizeBytes, "sha": a.SHA256, "id": a.ID})
if err != nil {
t.Fatal(err)
}
cfg, err := config.NewObjectStore()
if err != nil {
t.Fatal(err)
}
objects, err := objectstore.New(cfg)
if err != nil {
t.Fatal(err)
}
if err := objects.Put(t.Context(), a.ObjectKey, a.ContentType, bytes.NewReader(data), int64(len(data))); err != nil {
t.Fatal(err)
}
if _, err := s.QueueAttachment(t.Context(), a.ID); err != nil {
t.Fatal(err)
}
jobs, err := s.Due(t.Context(), 1)
if err != nil || len(jobs) != 1 {
t.Fatal("missing durable intent")
}
queue, err := streamqueue.NewScoped(os.Getenv("BROKER_ADDRESS"), a.ID.String(), "test-"+a.ID.String())
if err != nil {
t.Fatal(err)
}
defer func() {
if err := queue.Close(); err != nil {
t.Error(err)
}
}()
processor, err := imageprocessor.New()
if err != nil {
t.Fatal(err)
}
faulty := &faultStore{Store: s, before: fault.before, after: fault.after}
worker, err := pipeline.New(faulty, queue, objects, processor, zap.NewNop())
if err != nil {
t.Fatal(err)
}
// The real broker is exercised, while the exact delivery is controlled to avoid sharing consumer state.
if err := queue.Publish(t.Context(), jobs[0]); err != nil {
t.Fatal(err)
}
received, err := queue.Receive(t.Context())
if err != nil || received == nil {
t.Fatalf("delivery: %v", err)
}
delivery := *received
if err := worker.Handle(t.Context(), delivery); err != nil {
t.Fatal(err)
}
_, err = s.pool.Exec(t.Context(), `UPDATE attachment_jobs SET next_attempt_at=clock_timestamp()-interval '1 second' WHERE id=@id`, pgx.StrictNamedArgs{"id": jobs[0].ID})
if err != nil {
t.Fatal(err)
}
if err := worker.Handle(t.Context(), delivery); err != nil {
t.Fatal(err)
}
ready, err := s.Attachment(t.Context(), a.ID)
if err != nil || ready.Status != "ready" || len(ready.Variants) != 3 {
t.Fatalf("not ready: %v", err)
}
firstKey := ready.ObjectKey
if err := worker.Handle(t.Context(), delivery); err != nil {
t.Fatal(err)
}
again, err := s.Attachment(t.Context(), a.ID)
if err != nil || again.ObjectKey != firstKey {
t.Fatal("duplicate changed published effect")
}
reader, err := objects.Get(t.Context(), ready.ObjectKey)
if err != nil {
t.Fatal(err)
}
actual, err := io.ReadAll(reader)
closeErr := reader.Close()
if err != nil || closeErr != nil || !bytes.Equal(actual, data) {
t.Fatal("original changed")
}
t.Logf("fault=%s accepted=1 ready=1 variants=%d duplicate_effects=0", fault.name, len(ready.Variants))
})
}
}
// TestRetentionFencesReplay protects active work and makes interrupted cleanup repeatable.
func TestRetentionFencesReplay(t *testing.T) {
s := testStore(t)
a, job := queuedJob(t, s)
keys, err := s.ExpireStaging(t.Context())
if err != nil || len(keys) != 0 {
t.Fatal("active staging expired")
}
lease, err := s.Claim(t.Context(), job, time.Minute, 5)
if err != nil || lease == nil {
t.Fatal(err)
}
if ok, err := s.Complete(t.Context(), *lease, "result/original", nil); err != nil || !ok {
t.Fatal(err)
}
_, err = s.pool.Exec(t.Context(), `UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id=@id`, pgx.StrictNamedArgs{"id": a.ID})
if err != nil {
t.Fatal(err)
}
keys, err = s.ExpireStaging(t.Context())
if err != nil || len(keys) != 1 || keys[0] != a.ObjectKey {
t.Fatal("terminal staging not claimed")
}
if err := s.Replay(t.Context(), a.ID, 2); err == nil {
t.Fatal("expired source replay accepted")
}
again, err := s.ExpireStaging(t.Context())
if err != nil || len(again) != 1 {
t.Fatal("unacknowledged cleanup lost")
}
if err := s.StagingRemoved(t.Context(), a.ObjectKey); err != nil {
t.Fatal(err)
}
keys, err = s.ExpireStaging(t.Context())
if err != nil || len(keys) != 0 {
t.Fatal("confirmed deletion repeated")
}
}