308 lines
6.6 KiB
Go
308 lines
6.6 KiB
Go
package cache
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
mathrand "math/rand/v2"
|
|
"strconv"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"golang.org/x/sync/singleflight"
|
|
)
|
|
|
|
const (
|
|
DefaultLockTTL = 10 * time.Second
|
|
DefaultWaitTimeout = 2 * time.Second
|
|
minLockTTL = time.Second
|
|
redisOpTimeout = 200 * time.Millisecond
|
|
releaseTimeout = 300 * time.Millisecond
|
|
)
|
|
|
|
type DistributedConfig struct {
|
|
LockTTL time.Duration
|
|
WaitTimeout time.Duration
|
|
}
|
|
|
|
type LockStore interface {
|
|
Cache
|
|
AcquireLock(ctx context.Context, key, token string, ttl time.Duration) (bool, error)
|
|
ReleaseLock(ctx context.Context, key, token string) (bool, error)
|
|
LockHeld(ctx context.Context, key string) (bool, error)
|
|
}
|
|
|
|
type DistributedCache struct {
|
|
store LockStore
|
|
lockTTL time.Duration
|
|
waitTimeout time.Duration
|
|
log *slog.Logger
|
|
group singleflight.Group
|
|
}
|
|
|
|
var fallbackTokenCounter atomic.Uint64
|
|
|
|
func NewDistributed(store LockStore, cfg DistributedConfig, log *slog.Logger) (*DistributedCache, error) {
|
|
if cfg.LockTTL <= 0 {
|
|
cfg.LockTTL = DefaultLockTTL
|
|
}
|
|
|
|
if cfg.WaitTimeout <= 0 {
|
|
cfg.WaitTimeout = DefaultWaitTimeout
|
|
}
|
|
|
|
if cfg.LockTTL <= minLockTTL {
|
|
return nil, fmt.Errorf("cache: lock ttl %s must be greater than %s", cfg.LockTTL, minLockTTL)
|
|
}
|
|
|
|
if cfg.WaitTimeout > cfg.LockTTL {
|
|
return nil, fmt.Errorf("cache: wait timeout %s must not exceed lock ttl %s", cfg.WaitTimeout, cfg.LockTTL)
|
|
}
|
|
|
|
return &DistributedCache{
|
|
store: store,
|
|
lockTTL: cfg.LockTTL,
|
|
waitTimeout: cfg.WaitTimeout,
|
|
log: log,
|
|
}, nil
|
|
}
|
|
|
|
func (d *DistributedCache) GetOrLoad(
|
|
ctx context.Context,
|
|
key string,
|
|
ttl time.Duration,
|
|
loader func(ctx context.Context) ([]byte, error),
|
|
) ([]byte, error) {
|
|
ch := d.group.DoChan(key, func() (any, error) {
|
|
lctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), d.lockTTL+d.waitTimeout)
|
|
defer cancel()
|
|
|
|
return d.getOrLoad(lctx, key, ttl, loader)
|
|
})
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case result := <-ch:
|
|
if result.Err != nil {
|
|
return nil, result.Err
|
|
}
|
|
|
|
value, ok := result.Val.([]byte)
|
|
if !ok {
|
|
return nil, errors.New("cache: unexpected singleflight result")
|
|
}
|
|
|
|
return bytes.Clone(value), nil
|
|
}
|
|
}
|
|
|
|
func (d *DistributedCache) getOrLoad(
|
|
ctx context.Context,
|
|
key string,
|
|
ttl time.Duration,
|
|
loader func(ctx context.Context) ([]byte, error),
|
|
) ([]byte, error) {
|
|
lockKey := "lock:" + key
|
|
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if value, err := d.store.Get(ctx, key); err == nil {
|
|
return value, nil
|
|
} else if !errors.Is(err, ErrCacheMiss) {
|
|
return loader(ctx)
|
|
}
|
|
|
|
token := randomToken()
|
|
|
|
acquired, err := d.store.AcquireLock(ctx, lockKey, token, d.lockTTL)
|
|
if err != nil {
|
|
return loader(ctx)
|
|
}
|
|
|
|
if acquired {
|
|
return d.fill(ctx, key, lockKey, token, ttl, loader)
|
|
}
|
|
|
|
value, outcome, err := d.await(ctx, key, lockKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
switch outcome {
|
|
case waitHit:
|
|
return value, nil
|
|
case waitDegraded:
|
|
return loader(ctx)
|
|
case waitTimeout:
|
|
return loader(ctx)
|
|
case waitRetry:
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
case waitNone:
|
|
return nil, errors.New("cache: unexpected wait outcome")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *DistributedCache) fill(
|
|
ctx context.Context,
|
|
key, lockKey, token string,
|
|
ttl time.Duration,
|
|
loader func(ctx context.Context) ([]byte, error),
|
|
) ([]byte, error) {
|
|
base := context.WithoutCancel(ctx)
|
|
|
|
defer func() {
|
|
rctx, cancel := context.WithTimeout(base, releaseTimeout)
|
|
defer cancel()
|
|
|
|
held, err := d.store.ReleaseLock(rctx, lockKey, token)
|
|
switch {
|
|
case err != nil:
|
|
d.warn(base, "lock release failed", key, err)
|
|
case !held:
|
|
d.warn(base, "lock expired before release: loader slower than lock ttl", key, nil)
|
|
}
|
|
}()
|
|
|
|
getCtx, cancelGet := context.WithTimeout(base, redisOpTimeout)
|
|
value, err := d.store.Get(getCtx, key)
|
|
cancelGet()
|
|
|
|
if err == nil {
|
|
return value, nil
|
|
}
|
|
|
|
loadCtx, cancel := context.WithTimeout(base, d.lockTTL-minLockTTL)
|
|
defer cancel()
|
|
|
|
value, err = loader(loadCtx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
setCtx, cancelSet := context.WithTimeout(base, redisOpTimeout)
|
|
setErr := d.store.Set(setCtx, key, value, ttl)
|
|
cancelSet()
|
|
|
|
if setErr != nil {
|
|
d.warn(base, "distributed cache write failed", key, setErr)
|
|
}
|
|
|
|
return value, nil
|
|
}
|
|
|
|
type waitOutcome int
|
|
|
|
const (
|
|
waitNone waitOutcome = iota
|
|
waitHit
|
|
waitRetry
|
|
waitTimeout
|
|
waitDegraded
|
|
)
|
|
|
|
func (d *DistributedCache) await(ctx context.Context, key, lockKey string) ([]byte, waitOutcome, error) {
|
|
wctx, cancel := context.WithTimeout(ctx, d.waitTimeout)
|
|
defer cancel()
|
|
|
|
for attempt := 1; ; attempt++ {
|
|
value, err := d.store.Get(wctx, key)
|
|
switch {
|
|
case err == nil:
|
|
return value, waitHit, nil
|
|
case ctx.Err() != nil:
|
|
return nil, waitNone, ctx.Err()
|
|
case wctx.Err() != nil:
|
|
return nil, waitTimeout, nil
|
|
case !errors.Is(err, ErrCacheMiss):
|
|
return nil, waitDegraded, nil
|
|
}
|
|
|
|
if held, err := d.store.LockHeld(wctx, lockKey); err == nil && !held {
|
|
return nil, waitRetry, nil
|
|
}
|
|
|
|
select {
|
|
case <-wctx.Done():
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, waitNone, err
|
|
}
|
|
|
|
return nil, waitTimeout, nil
|
|
case <-time.After(jittered(waitBackoff(attempt))):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *DistributedCache) Get(ctx context.Context, key string) ([]byte, error) {
|
|
return d.store.Get(ctx, key)
|
|
}
|
|
|
|
func (d *DistributedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
|
|
return d.store.Set(ctx, key, value, ttl)
|
|
}
|
|
|
|
func (d *DistributedCache) Delete(ctx context.Context, keys ...string) error {
|
|
return d.store.Delete(ctx, keys...)
|
|
}
|
|
|
|
func (d *DistributedCache) Ping(ctx context.Context) error {
|
|
return d.store.Ping(ctx)
|
|
}
|
|
|
|
func (d *DistributedCache) Close() error {
|
|
return d.store.Close()
|
|
}
|
|
|
|
func (d *DistributedCache) warn(ctx context.Context, msg, key string, err error) {
|
|
if d.log == nil {
|
|
return
|
|
}
|
|
|
|
attrs := []slog.Attr{slog.String("key", key)}
|
|
if err != nil {
|
|
attrs = append(attrs, slog.Any("error", err))
|
|
}
|
|
|
|
d.log.LogAttrs(ctx, slog.LevelWarn, msg, attrs...)
|
|
}
|
|
|
|
func waitBackoff(attempt int) time.Duration {
|
|
const base, maxWait = 50 * time.Millisecond, 500 * time.Millisecond
|
|
if attempt > 4 {
|
|
return maxWait
|
|
}
|
|
|
|
return base << (attempt - 1)
|
|
}
|
|
|
|
func jittered(d time.Duration) time.Duration {
|
|
half := d / 2
|
|
if half <= 0 {
|
|
return d
|
|
}
|
|
|
|
return half + mathrand.N(half)
|
|
}
|
|
|
|
func randomToken() string {
|
|
raw := make([]byte, 16)
|
|
if _, err := rand.Read(raw); err != nil {
|
|
counter := fallbackTokenCounter.Add(1)
|
|
|
|
return strconv.FormatInt(time.Now().UnixNano(), 10) + ":" + strconv.FormatUint(counter, 10)
|
|
}
|
|
|
|
return hex.EncodeToString(raw)
|
|
}
|