375 lines
10 KiB
Go
375 lines
10 KiB
Go
package cache_test
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"mtusi-messenger/internal/platform/cache"
|
|
"mtusi-messenger/internal/platform/cache/mocks"
|
|
)
|
|
|
|
func newDistributed(t *testing.T, store cache.LockStore, cfg cache.DistributedConfig) *cache.DistributedCache {
|
|
t.Helper()
|
|
|
|
dc, err := cache.NewDistributed(store, cfg, nil)
|
|
require.NoError(t, err)
|
|
|
|
return dc
|
|
}
|
|
|
|
func expectFill(store *mocks.MockLockStore, key string) {
|
|
store.EXPECT().Get(mock.Anything, key).Return(nil, cache.ErrCacheMiss).Once()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:"+key, mock.AnythingOfType("string"), mock.Anything).Return(true, nil).Once()
|
|
store.EXPECT().Get(mock.Anything, key).Return(nil, cache.ErrCacheMiss).Once()
|
|
store.EXPECT().Set(mock.Anything, key, mock.Anything, mock.Anything).Return(nil).Once()
|
|
store.EXPECT().ReleaseLock(mock.Anything, "lock:"+key, mock.AnythingOfType("string")).Return(true, nil).Once()
|
|
}
|
|
|
|
// sharedWaitTimeout bounds every wait in these tests: a stuck helper must fail
|
|
// the test instead of hanging the whole package until -timeout fires.
|
|
const sharedWaitTimeout = 5 * time.Second
|
|
|
|
func awaitSignal(t *testing.T, ch <-chan struct{}, message string) {
|
|
t.Helper()
|
|
|
|
select {
|
|
case <-ch:
|
|
case <-time.After(sharedWaitTimeout):
|
|
t.Fatal("timed out: " + message)
|
|
}
|
|
}
|
|
|
|
func awaitValue[T any](t *testing.T, ch <-chan T) T {
|
|
t.Helper()
|
|
|
|
select {
|
|
case value := <-ch:
|
|
return value
|
|
case <-time.After(sharedWaitTimeout):
|
|
var zero T
|
|
|
|
t.Fatal("timed out waiting for the shared load to report its result")
|
|
|
|
return zero
|
|
}
|
|
}
|
|
|
|
func awaitWaitGroup(t *testing.T, wg *sync.WaitGroup) {
|
|
t.Helper()
|
|
|
|
done := make(chan struct{})
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
awaitSignal(t, done, "goroutines did not finish")
|
|
}
|
|
|
|
func TestDistributedCachesResult(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{})
|
|
ctx := context.Background()
|
|
|
|
var calls atomic.Int32
|
|
|
|
loader := func(context.Context) ([]byte, error) {
|
|
calls.Add(1)
|
|
|
|
return []byte("payload"), nil
|
|
}
|
|
|
|
expectFill(store, "key")
|
|
store.EXPECT().Get(mock.Anything, "key").Return([]byte("payload"), nil).Once()
|
|
|
|
first, err := dc.GetOrLoad(ctx, "key", time.Minute, loader)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("payload"), first)
|
|
|
|
second, err := dc.GetOrLoad(ctx, "key", time.Minute, loader)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("payload"), second)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
}
|
|
|
|
func TestDistributedDeduplicatesConcurrentMisses(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{})
|
|
ctx := context.Background()
|
|
|
|
var calls atomic.Int32
|
|
|
|
loader := func(context.Context) ([]byte, error) {
|
|
calls.Add(1)
|
|
time.Sleep(30 * time.Millisecond)
|
|
|
|
return []byte("payload"), nil
|
|
}
|
|
|
|
store.EXPECT().Get(mock.Anything, "shared").Return(nil, cache.ErrCacheMiss).Twice()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:shared", mock.AnythingOfType("string"), mock.Anything).Return(true, nil).Once()
|
|
store.EXPECT().Set(mock.Anything, "shared", mock.Anything, mock.Anything).Return(nil).Once()
|
|
store.EXPECT().ReleaseLock(mock.Anything, "lock:shared", mock.AnythingOfType("string")).Return(true, nil).Once()
|
|
|
|
const workers = 50
|
|
|
|
var (
|
|
wg sync.WaitGroup
|
|
failures atomic.Int32
|
|
)
|
|
|
|
wg.Add(workers)
|
|
|
|
for range workers {
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
value, err := dc.GetOrLoad(ctx, "shared", time.Minute, loader)
|
|
if err != nil || string(value) != "payload" {
|
|
failures.Add(1)
|
|
}
|
|
}()
|
|
}
|
|
|
|
awaitWaitGroup(t, &wg)
|
|
|
|
require.Zero(t, failures.Load())
|
|
require.Equal(t, int32(1), calls.Load(), "one instance must load once for concurrent misses")
|
|
}
|
|
|
|
func TestDistributedFallsBackOnWaitTimeout(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{WaitTimeout: 100 * time.Millisecond})
|
|
ctx := context.Background()
|
|
|
|
store.EXPECT().Get(mock.Anything, "key").Return(nil, cache.ErrCacheMiss).Maybe()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:key", mock.AnythingOfType("string"), mock.Anything).Return(false, nil).Once()
|
|
store.EXPECT().LockHeld(mock.Anything, "lock:key").Return(true, nil).Maybe()
|
|
|
|
var calls atomic.Int32
|
|
|
|
value, err := dc.GetOrLoad(ctx, "key", time.Minute, func(context.Context) ([]byte, error) {
|
|
calls.Add(1)
|
|
|
|
return []byte("fallback"), nil
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("fallback"), value)
|
|
require.Equal(t, int32(1), calls.Load(), "a wait timeout must fall back to a direct load")
|
|
}
|
|
|
|
func TestDistributedSharedLoadSurvivesCallerCancellation(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{})
|
|
leaderCtx, cancelLeader := context.WithCancel(context.Background())
|
|
defer cancelLeader()
|
|
|
|
var calls atomic.Int32
|
|
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
|
|
loader := func(loadCtx context.Context) ([]byte, error) {
|
|
calls.Add(1)
|
|
close(started)
|
|
<-release
|
|
|
|
if err := loadCtx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return []byte("value"), nil
|
|
}
|
|
|
|
store.EXPECT().Get(mock.Anything, "key").Return(nil, cache.ErrCacheMiss).Twice()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:key", mock.AnythingOfType("string"), mock.Anything).Return(true, nil).Once()
|
|
store.EXPECT().Set(mock.Anything, "key", mock.Anything, mock.Anything).Return(nil).Once()
|
|
store.EXPECT().ReleaseLock(mock.Anything, "lock:key", mock.AnythingOfType("string")).Return(true, nil).Once()
|
|
|
|
leaderErr := make(chan error, 1)
|
|
|
|
go func() {
|
|
_, err := dc.GetOrLoad(leaderCtx, "key", time.Minute, loader)
|
|
leaderErr <- err
|
|
}()
|
|
|
|
awaitSignal(t, started, "leader must reach the loader")
|
|
|
|
// The follower starts while the leader's loader is still blocked, so the
|
|
// singleflight call cannot complete: the signal below means DoChan was
|
|
// already called, i.e. the follower joined the leader's in-flight call
|
|
// instead of starting its own load.
|
|
followerJoined := make(chan struct{})
|
|
|
|
var (
|
|
wg sync.WaitGroup
|
|
followerV []byte
|
|
followerE error
|
|
)
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
ctx := newJoiningContext(followerJoined)
|
|
followerV, followerE = dc.GetOrLoad(ctx, "key", time.Minute, loader)
|
|
}()
|
|
|
|
awaitSignal(t, followerJoined, "follower must join the in-flight load")
|
|
|
|
cancelLeader()
|
|
close(release)
|
|
|
|
require.ErrorIs(t, awaitValue(t, leaderErr), context.Canceled)
|
|
awaitWaitGroup(t, &wg)
|
|
|
|
require.NoError(t, followerE)
|
|
require.Equal(t, []byte("value"), followerV)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
}
|
|
|
|
// joiningContext reports when GetOrLoad reaches the shared-result wait path. The
|
|
// delegate is kept as function fields so no struct holds a context.Context.
|
|
type joiningContext struct {
|
|
deadline func() (time.Time, bool)
|
|
done func() <-chan struct{}
|
|
err func() error
|
|
value func(any) any
|
|
|
|
once *sync.Once
|
|
joined chan struct{}
|
|
}
|
|
|
|
func newJoiningContext(joined chan struct{}) joiningContext {
|
|
base := context.Background()
|
|
|
|
return joiningContext{
|
|
deadline: base.Deadline,
|
|
done: base.Done,
|
|
err: base.Err,
|
|
value: base.Value,
|
|
once: new(sync.Once),
|
|
joined: joined,
|
|
}
|
|
}
|
|
|
|
func (c joiningContext) Deadline() (time.Time, bool) { return c.deadline() }
|
|
|
|
func (c joiningContext) Done() <-chan struct{} {
|
|
c.once.Do(func() { close(c.joined) })
|
|
|
|
return c.done()
|
|
}
|
|
|
|
func (c joiningContext) Err() error { return c.err() }
|
|
|
|
func (c joiningContext) Value(key any) any { return c.value(key) }
|
|
|
|
func TestDistributedRetriesWhenLockVanishes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{})
|
|
ctx := context.Background()
|
|
|
|
var released atomic.Bool
|
|
|
|
time.AfterFunc(120*time.Millisecond, func() { released.Store(true) })
|
|
|
|
store.EXPECT().Get(mock.Anything, "key").Return(nil, cache.ErrCacheMiss).Maybe()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:key", mock.AnythingOfType("string"), mock.Anything).Return(false, nil).Once()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:key", mock.AnythingOfType("string"), mock.Anything).Return(true, nil).Once()
|
|
store.EXPECT().LockHeld(mock.Anything, "lock:key").RunAndReturn(func(context.Context, string) (bool, error) {
|
|
return !released.Load(), nil
|
|
}).Maybe()
|
|
store.EXPECT().Set(mock.Anything, "key", mock.Anything, mock.Anything).Return(nil).Once()
|
|
store.EXPECT().ReleaseLock(mock.Anything, "lock:key", mock.AnythingOfType("string")).Return(true, nil).Once()
|
|
|
|
var calls atomic.Int32
|
|
|
|
value, err := dc.GetOrLoad(ctx, "key", time.Minute, func(context.Context) ([]byte, error) {
|
|
calls.Add(1)
|
|
|
|
return []byte("recovered"), nil
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("recovered"), value)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
}
|
|
|
|
func TestFillStoreCallsAreBoundedByDeadline(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := mocks.NewMockLockStore(t)
|
|
dc := newDistributed(t, store, cache.DistributedConfig{})
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
getBudgets []time.Duration
|
|
setBudget time.Duration
|
|
setHasDead bool
|
|
)
|
|
|
|
store.EXPECT().Get(mock.Anything, "key").RunAndReturn(func(ctx context.Context, _ string) ([]byte, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
getBudgets = append(getBudgets, time.Until(deadline))
|
|
}
|
|
|
|
return nil, cache.ErrCacheMiss
|
|
}).Twice()
|
|
store.EXPECT().AcquireLock(mock.Anything, "lock:key", mock.AnythingOfType("string"), mock.Anything).Return(true, nil).Once()
|
|
store.EXPECT().Set(mock.Anything, "key", mock.Anything, mock.Anything).
|
|
RunAndReturn(func(ctx context.Context, _ string, _ []byte, _ time.Duration) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
deadline, ok := ctx.Deadline()
|
|
setHasDead = ok
|
|
if ok {
|
|
setBudget = time.Until(deadline)
|
|
}
|
|
|
|
return nil
|
|
}).Once()
|
|
store.EXPECT().ReleaseLock(mock.Anything, "lock:key", mock.AnythingOfType("string")).Return(true, nil).Once()
|
|
|
|
_, err := dc.GetOrLoad(context.Background(), "key", time.Minute, func(context.Context) ([]byte, error) {
|
|
return []byte("value"), nil
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
require.True(t, setHasDead, "Set must receive a bounded context")
|
|
require.LessOrEqual(t, setBudget, 300*time.Millisecond)
|
|
|
|
minGetBudget := time.Hour
|
|
for _, budget := range getBudgets {
|
|
if budget < minGetBudget {
|
|
minGetBudget = budget
|
|
}
|
|
}
|
|
|
|
require.LessOrEqual(t, minGetBudget, 300*time.Millisecond, "fill's Get must receive a bounded context")
|
|
}
|