package cache import ( "bytes" "context" "crypto/rand" "encoding/hex" "errors" "fmt" "log/slog" mathrand "math/rand/v2" "strconv" "sync/atomic" "time" "golang.org/x/sync/singleflight" ) const ( DefaultLockTTL = 10 * time.Second DefaultWaitTimeout = 2 * time.Second minLockTTL = time.Second redisOpTimeout = 200 * time.Millisecond releaseTimeout = 300 * time.Millisecond ) type DistributedConfig struct { LockTTL time.Duration WaitTimeout time.Duration } type LockStore interface { Cache AcquireLock(ctx context.Context, key, token string, ttl time.Duration) (bool, error) ReleaseLock(ctx context.Context, key, token string) (bool, error) LockHeld(ctx context.Context, key string) (bool, error) } type DistributedCache struct { store LockStore lockTTL time.Duration waitTimeout time.Duration log *slog.Logger group singleflight.Group } var fallbackTokenCounter atomic.Uint64 func NewDistributed(store LockStore, cfg DistributedConfig, log *slog.Logger) (*DistributedCache, error) { if cfg.LockTTL <= 0 { cfg.LockTTL = DefaultLockTTL } if cfg.WaitTimeout <= 0 { cfg.WaitTimeout = DefaultWaitTimeout } if cfg.LockTTL <= minLockTTL { return nil, fmt.Errorf("cache: lock ttl %s must be greater than %s", cfg.LockTTL, minLockTTL) } if cfg.WaitTimeout > cfg.LockTTL { return nil, fmt.Errorf("cache: wait timeout %s must not exceed lock ttl %s", cfg.WaitTimeout, cfg.LockTTL) } return &DistributedCache{ store: store, lockTTL: cfg.LockTTL, waitTimeout: cfg.WaitTimeout, log: log, }, nil } func (d *DistributedCache) GetOrLoad( ctx context.Context, key string, ttl time.Duration, loader func(ctx context.Context) ([]byte, error), ) ([]byte, error) { ch := d.group.DoChan(key, func() (any, error) { lctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), d.lockTTL+d.waitTimeout) defer cancel() return d.getOrLoad(lctx, key, ttl, loader) }) select { case <-ctx.Done(): return nil, ctx.Err() case result := <-ch: if result.Err != nil { return nil, result.Err } value, ok := result.Val.([]byte) if !ok { return nil, errors.New("cache: unexpected singleflight result") } return bytes.Clone(value), nil } } func (d *DistributedCache) getOrLoad( ctx context.Context, key string, ttl time.Duration, loader func(ctx context.Context) ([]byte, error), ) ([]byte, error) { lockKey := "lock:" + key for { if err := ctx.Err(); err != nil { return nil, err } if value, err := d.store.Get(ctx, key); err == nil { return value, nil } else if !errors.Is(err, ErrCacheMiss) { return loader(ctx) } token := randomToken() acquired, err := d.store.AcquireLock(ctx, lockKey, token, d.lockTTL) if err != nil { return loader(ctx) } if acquired { return d.fill(ctx, key, lockKey, token, ttl, loader) } value, outcome, err := d.await(ctx, key, lockKey) if err != nil { return nil, err } switch outcome { case waitHit: return value, nil case waitDegraded: return loader(ctx) case waitTimeout: return loader(ctx) case waitRetry: if ctx.Err() != nil { return nil, ctx.Err() } case waitNone: return nil, errors.New("cache: unexpected wait outcome") } } } func (d *DistributedCache) fill( ctx context.Context, key, lockKey, token string, ttl time.Duration, loader func(ctx context.Context) ([]byte, error), ) ([]byte, error) { base := context.WithoutCancel(ctx) defer func() { rctx, cancel := context.WithTimeout(base, releaseTimeout) defer cancel() held, err := d.store.ReleaseLock(rctx, lockKey, token) switch { case err != nil: d.warn(base, "lock release failed", key, err) case !held: d.warn(base, "lock expired before release: loader slower than lock ttl", key, nil) } }() getCtx, cancelGet := context.WithTimeout(base, redisOpTimeout) value, err := d.store.Get(getCtx, key) cancelGet() if err == nil { return value, nil } loadCtx, cancel := context.WithTimeout(base, d.lockTTL-minLockTTL) defer cancel() value, err = loader(loadCtx) if err != nil { return nil, err } setCtx, cancelSet := context.WithTimeout(base, redisOpTimeout) setErr := d.store.Set(setCtx, key, value, ttl) cancelSet() if setErr != nil { d.warn(base, "distributed cache write failed", key, setErr) } return value, nil } type waitOutcome int const ( waitNone waitOutcome = iota waitHit waitRetry waitTimeout waitDegraded ) func (d *DistributedCache) await(ctx context.Context, key, lockKey string) ([]byte, waitOutcome, error) { wctx, cancel := context.WithTimeout(ctx, d.waitTimeout) defer cancel() for attempt := 1; ; attempt++ { value, err := d.store.Get(wctx, key) switch { case err == nil: return value, waitHit, nil case ctx.Err() != nil: return nil, waitNone, ctx.Err() case wctx.Err() != nil: return nil, waitTimeout, nil case !errors.Is(err, ErrCacheMiss): return nil, waitDegraded, nil } if held, err := d.store.LockHeld(wctx, lockKey); err == nil && !held { return nil, waitRetry, nil } select { case <-wctx.Done(): if err := ctx.Err(); err != nil { return nil, waitNone, err } return nil, waitTimeout, nil case <-time.After(jittered(waitBackoff(attempt))): } } } func (d *DistributedCache) Get(ctx context.Context, key string) ([]byte, error) { return d.store.Get(ctx, key) } func (d *DistributedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error { return d.store.Set(ctx, key, value, ttl) } func (d *DistributedCache) Delete(ctx context.Context, keys ...string) error { return d.store.Delete(ctx, keys...) } func (d *DistributedCache) Ping(ctx context.Context) error { return d.store.Ping(ctx) } func (d *DistributedCache) Close() error { return d.store.Close() } func (d *DistributedCache) warn(ctx context.Context, msg, key string, err error) { if d.log == nil { return } attrs := []slog.Attr{slog.String("key", key)} if err != nil { attrs = append(attrs, slog.Any("error", err)) } d.log.LogAttrs(ctx, slog.LevelWarn, msg, attrs...) } func waitBackoff(attempt int) time.Duration { const base, maxWait = 50 * time.Millisecond, 500 * time.Millisecond if attempt > 4 { return maxWait } return base << (attempt - 1) } func jittered(d time.Duration) time.Duration { half := d / 2 if half <= 0 { return d } return half + mathrand.N(half) } func randomToken() string { raw := make([]byte, 16) if _, err := rand.Read(raw); err != nil { counter := fallbackTokenCounter.Add(1) return strconv.FormatInt(time.Now().UnixNano(), 10) + ":" + strconv.FormatUint(counter, 10) } return hex.EncodeToString(raw) }