forked from mtusi2026/mtusi2026lab6
188 lines
5.8 KiB
Go
188 lines
5.8 KiB
Go
package postgres
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"go.uber.org/zap"
|
|
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config"
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/imageprocessor"
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto"
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore"
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/streamqueue"
|
|
)
|
|
|
|
type faultStore struct {
|
|
*Store
|
|
before bool
|
|
after bool
|
|
}
|
|
|
|
// Complete injects a lost connection before or after the actual SQL commit.
|
|
func (s *faultStore) Complete(ctx context.Context, l pipeline.Lease, key string, variants []dto.Variant) (bool, error) {
|
|
if s.before {
|
|
s.before = false
|
|
return false, errors.New("injected connection loss before commit")
|
|
}
|
|
done, err := s.Store.Complete(ctx, l, key, variants)
|
|
if err == nil && s.after {
|
|
s.after = false
|
|
return done, errors.New("injected lost commit acknowledgement")
|
|
}
|
|
return done, err
|
|
}
|
|
|
|
// TestRealPipelineFaultWindows verifies real objects and SQL remain correct across uncertain commits.
|
|
func TestRealPipelineFaultWindows(t *testing.T) {
|
|
for _, fault := range []struct {
|
|
name string
|
|
before bool
|
|
after bool
|
|
}{{"after_S3_before_commit", true, false}, {"after_commit_before_ack", false, true}} {
|
|
t.Run(fault.name, func(t *testing.T) {
|
|
if os.Getenv("S3_ENDPOINT") == "" || os.Getenv("BROKER_ADDRESS") == "" {
|
|
t.Skip("isolated S3 and broker required")
|
|
}
|
|
s := testStore(t)
|
|
a := fixtureAttachment(t, s)
|
|
data, err := os.ReadFile("../../../tests/fixtures/sample.png")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(data)
|
|
a.SHA256 = hex.EncodeToString(sum[:])
|
|
a.SizeBytes = int64(len(data))
|
|
_, err = s.pool.Exec(t.Context(), `UPDATE attachments SET size_bytes=@size,sha256=@sha WHERE id=@id`, pgx.StrictNamedArgs{"size": a.SizeBytes, "sha": a.SHA256, "id": a.ID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg, err := config.NewObjectStore()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
objects, err := objectstore.New(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := objects.Put(t.Context(), a.ObjectKey, a.ContentType, bytes.NewReader(data), int64(len(data))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.QueueAttachment(t.Context(), a.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
jobs, err := s.Due(t.Context(), 1)
|
|
if err != nil || len(jobs) != 1 {
|
|
t.Fatal("missing durable intent")
|
|
}
|
|
queue, err := streamqueue.NewScoped(os.Getenv("BROKER_ADDRESS"), a.ID.String(), "test-"+a.ID.String())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
if err := queue.Close(); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
processor, err := imageprocessor.New()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
faulty := &faultStore{Store: s, before: fault.before, after: fault.after}
|
|
worker, err := pipeline.New(faulty, queue, objects, processor, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The real broker is exercised, while the exact delivery is controlled to avoid sharing consumer state.
|
|
if err := queue.Publish(t.Context(), jobs[0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
received, err := queue.Receive(t.Context())
|
|
if err != nil || received == nil {
|
|
t.Fatalf("delivery: %v", err)
|
|
}
|
|
delivery := *received
|
|
if err := worker.Handle(t.Context(), delivery); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = s.pool.Exec(t.Context(), `UPDATE attachment_jobs SET next_attempt_at=clock_timestamp()-interval '1 second' WHERE id=@id`, pgx.StrictNamedArgs{"id": jobs[0].ID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := worker.Handle(t.Context(), delivery); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ready, err := s.Attachment(t.Context(), a.ID)
|
|
if err != nil || ready.Status != "ready" || len(ready.Variants) != 3 {
|
|
t.Fatalf("not ready: %v", err)
|
|
}
|
|
firstKey := ready.ObjectKey
|
|
if err := worker.Handle(t.Context(), delivery); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := s.Attachment(t.Context(), a.ID)
|
|
if err != nil || again.ObjectKey != firstKey {
|
|
t.Fatal("duplicate changed published effect")
|
|
}
|
|
reader, err := objects.Get(t.Context(), ready.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
actual, err := io.ReadAll(reader)
|
|
closeErr := reader.Close()
|
|
if err != nil || closeErr != nil || !bytes.Equal(actual, data) {
|
|
t.Fatal("original changed")
|
|
}
|
|
t.Logf("fault=%s accepted=1 ready=1 variants=%d duplicate_effects=0", fault.name, len(ready.Variants))
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestRetentionFencesReplay protects active work and makes interrupted cleanup repeatable.
|
|
func TestRetentionFencesReplay(t *testing.T) {
|
|
s := testStore(t)
|
|
a, job := queuedJob(t, s)
|
|
keys, err := s.ExpireStaging(t.Context())
|
|
if err != nil || len(keys) != 0 {
|
|
t.Fatal("active staging expired")
|
|
}
|
|
lease, err := s.Claim(t.Context(), job, time.Minute, 5)
|
|
if err != nil || lease == nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ok, err := s.Complete(t.Context(), *lease, "result/original", nil); err != nil || !ok {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = s.pool.Exec(t.Context(), `UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id=@id`, pgx.StrictNamedArgs{"id": a.ID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys, err = s.ExpireStaging(t.Context())
|
|
if err != nil || len(keys) != 1 || keys[0] != a.ObjectKey {
|
|
t.Fatal("terminal staging not claimed")
|
|
}
|
|
if err := s.Replay(t.Context(), a.ID, 2); err == nil {
|
|
t.Fatal("expired source replay accepted")
|
|
}
|
|
again, err := s.ExpireStaging(t.Context())
|
|
if err != nil || len(again) != 1 {
|
|
t.Fatal("unacknowledged cleanup lost")
|
|
}
|
|
if err := s.StagingRemoved(t.Context(), a.ObjectKey); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys, err = s.ExpireStaging(t.Context())
|
|
if err != nil || len(keys) != 0 {
|
|
t.Fatal("confirmed deletion repeated")
|
|
}
|
|
}
|