diff --git a/store/cache/cache.go b/store/cache/cache.go index 102f8add3..310942e49 100644 --- a/store/cache/cache.go +++ b/store/cache/cache.go @@ -66,10 +66,11 @@ func DefaultConfig() Config { // Cache is a thread-safe in-memory cache with TTL and memory management. type Cache struct { itemCount atomic.Int64 // Use atomic operations to track item count - data sync.Map + data sync.Map // stores *item; pointer identity guards removals config Config stopChan chan struct{} closedChan chan struct{} + closed atomic.Bool } // New creates a new memory cache with the given configuration. @@ -99,19 +100,29 @@ func (c *Cache) SetWithTTL(_ context.Context, key string, value any, ttl time.Du // Estimate size of the item (very rough approximation). size := estimateSize(value) - // Check if item already exists to avoid double counting. - if _, exists := c.data.Load(key); exists { - c.data.Delete(key) - } else { - // Only increment if this is a new key. - (&c.itemCount).Add(1) - } - - c.data.Store(key, item{ + entry := &item{ value: value, expiration: time.Now().Add(ttl), size: size, - }) + } + + for { + if old, ok := c.data.Load(key); ok { + // Replacing an existing entry does not change the count. + if c.data.CompareAndSwap(key, old, entry) { + break + } + continue + } + // Count the new entry before it is published, so a concurrent removal + // cannot decrement a count that has not been incremented. + (&c.itemCount).Add(1) + if _, loaded := c.data.LoadOrStore(key, entry); !loaded { + break + } + // Another writer published the key first. Undo the reservation and retry. + (&c.itemCount).Add(-1) + } // If we're over the max items, clean up old items. if c.config.MaxItems > 0 && (&c.itemCount).Load() > int64(c.config.MaxItems) { @@ -121,29 +132,37 @@ func (c *Cache) SetWithTTL(_ context.Context, key string, value any, ttl time.Du // Get retrieves a value from the cache. func (c *Cache) Get(_ context.Context, key string) (any, bool) { - value, ok := c.data.Load(key) - if !ok { - return nil, false - } - - itm, ok := value.(item) - if !ok { - // If the value is not of type item, it means it was corrupted or not set correctly. - c.data.Delete(key) - return nil, false - } - if time.Now().After(itm.expiration) { - c.data.Delete(key) - (&c.itemCount).Add(-1) - - if c.config.OnEviction != nil { - c.config.OnEviction(key, itm.value) + for { + value, ok := c.data.Load(key) + if !ok { + return nil, false } - return nil, false - } + itm, ok := value.(*item) + if !ok { + // If the value is not of type *item, it means it was corrupted or not set correctly. + if _, loaded := c.data.LoadAndDelete(key); loaded { + (&c.itemCount).Add(-1) + } + return nil, false + } + if !time.Now().After(itm.expiration) { + return itm.value, true + } - return itm.value, true + // Only account for the removal if we removed the exact entry we + // observed. A concurrent Set may have already replaced it with a + // fresher entry, which must survive. + if c.data.CompareAndDelete(key, value) { + (&c.itemCount).Add(-1) + + if c.config.OnEviction != nil { + c.config.OnEviction(key, itm.value) + } + return nil, false + } + // The expired entry was replaced while we looked at it; read the fresher one. + } } // Delete removes a value from the cache. @@ -152,7 +171,7 @@ func (c *Cache) Delete(_ context.Context, key string) { (&c.itemCount).Add(-1) if c.config.OnEviction != nil { - if itm, ok := value.(item); ok { + if itm, ok := value.(*item); ok { c.config.OnEviction(key, itm.value) } } @@ -161,21 +180,21 @@ func (c *Cache) Delete(_ context.Context, key string) { // Clear removes all values from the cache. func (c *Cache) Clear(_ context.Context) { - count := 0 c.data.Range(func(key, value any) bool { - if c.config.OnEviction != nil { - itm, ok := value.(item) - if ok { - if keyStr, ok := key.(string); ok { - c.config.OnEviction(keyStr, itm.value) + if _, loaded := c.data.LoadAndDelete(key); loaded { + (&c.itemCount).Add(-1) + + if c.config.OnEviction != nil { + itm, ok := value.(*item) + if ok { + if keyStr, ok := key.(string); ok { + c.config.OnEviction(keyStr, itm.value) + } } } } - c.data.Delete(key) - count++ return true }) - (&c.itemCount).Store(0) } // Size returns the number of items in the cache. @@ -185,15 +204,11 @@ func (c *Cache) Size() int64 { // Close stops the cache cleanup goroutine. func (c *Cache) Close() error { - select { - case <-c.stopChan: - // Already closed - return nil - default: + if c.closed.CompareAndSwap(false, true) { close(c.stopChan) <-c.closedChan // Wait for cleanup goroutine to exit - return nil } + return nil } // cleanupLoop periodically cleans up expired items. @@ -220,12 +235,16 @@ func (c *Cache) cleanup() { count := 0 c.data.Range(func(key, value any) bool { - itm, ok := value.(item) + itm, ok := value.(*item) if !ok { return true } if time.Now().After(itm.expiration) { - c.data.Delete(key) + // CompareAndDelete so a concurrently re-set fresher entry is + // neither removed nor counted. + if !c.data.CompareAndDelete(key, value) { + return true + } count++ if c.config.OnEviction != nil { @@ -250,32 +269,31 @@ func (c *Cache) cleanup() { } // cleanupOldest removes the oldest items if we're over the max items. +// +// itemCount can transiently over-count while other writers hold reservations +// for keys they have not published yet, so the decision to evict is based on +// the entries actually in the map, counted during the same scan that picks +// the eviction candidates. func (c *Cache) cleanupOldest() { // Remove 20% of max items at once threshold := max(c.config.MaxItems/5, 1) - currentCount := (&c.itemCount).Load() - - // If we're not over the threshold, don't do anything - if currentCount <= int64(c.config.MaxItems) { - return - } - // Find the oldest items type keyExpPair struct { key string - value any + entry *item expiration time.Time } candidates := make([]keyExpPair, 0, threshold) - + published := int64(0) c.data.Range(func(key, value any) bool { - itm, ok := value.(item) + itm, ok := value.(*item) if !ok { return true } + published++ if keyStr, ok := key.(string); ok && len(candidates) < threshold { - candidates = append(candidates, keyExpPair{keyStr, itm.value, itm.expiration}) + candidates = append(candidates, keyExpPair{keyStr, itm, itm.expiration}) return true } @@ -289,20 +307,29 @@ func (c *Cache) cleanupOldest() { // Replace it if this item is older if itm.expiration.Before(candidates[newestIdx].expiration) { - candidates[newestIdx] = keyExpPair{key.(string), itm.value, itm.expiration} + candidates[newestIdx] = keyExpPair{key.(string), itm, itm.expiration} } return true }) + // Not actually over the limit: the counter was inflated by in-flight reservations. + if published <= int64(c.config.MaxItems) { + return + } + // Delete the oldest items deletedCount := 0 for _, candidate := range candidates { - c.data.Delete(candidate.key) + // CompareAndDelete so an entry re-set or already removed since the + // scan is neither removed nor counted. + if !c.data.CompareAndDelete(candidate.key, candidate.entry) { + continue + } deletedCount++ if c.config.OnEviction != nil { - c.config.OnEviction(candidate.key, candidate.value) + c.config.OnEviction(candidate.key, candidate.entry.value) } } diff --git a/store/cache/cache_test.go b/store/cache/cache_test.go index 755857692..3154eb5d5 100644 --- a/store/cache/cache_test.go +++ b/store/cache/cache_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "testing" "time" ) @@ -204,3 +205,260 @@ func TestEvictionCallback(t *testing.T) { } evictedMu.Unlock() } + +func TestCacheConcurrentExpiryCountedOnce(t *testing.T) { + ctx := context.Background() + + config := DefaultConfig() + config.DefaultTTL = 15 * time.Millisecond + config.CleanupInterval = time.Hour // disable the background sweeper for determinism + config.MaxItems = 0 + var evictions atomic.Int64 + config.OnEviction = func(string, any) { + evictions.Add(1) + } + cache := New(config) + defer cache.Close() + + const rounds = 50 + const readers = 32 + for r := range rounds { + key := fmt.Sprintf("expiring-%d", r) + cache.Set(ctx, key, r) + time.Sleep(20 * time.Millisecond) // let the entry expire + + start := make(chan struct{}) + var wg sync.WaitGroup + for range readers { + wg.Go(func() { + <-start + cache.Get(ctx, key) + }) + } + close(start) + wg.Wait() + + if got := cache.Size(); got != 0 { + t.Fatalf("round %d: expired entry removal over-counted (size %d)", r, got) + } + } + if got := evictions.Load(); got != rounds { + t.Fatalf("expected %d eviction callbacks, got %d", rounds, got) + } +} + +func TestCacheOverwriteAccounting(t *testing.T) { + ctx := context.Background() + cache := NewDefault() + defer cache.Close() + + const workers = 16 + const writes = 200 + var wg sync.WaitGroup + for id := range workers { + wg.Go(func() { + for j := range writes { + key := fmt.Sprintf("key%d", j%8) + cache.Set(ctx, key, fmt.Sprintf("worker%d-%d", id, j)) + } + }) + } + wg.Wait() + + if got := cache.Size(); got != 8 { + t.Fatalf("expected 8 entries after overwrites, got %d", got) + } + for j := range 8 { + cache.Delete(ctx, fmt.Sprintf("key%d", j)) + } + if got := cache.Size(); got != 0 { + t.Fatalf("expected empty cache after deletes, got %d", got) + } +} + +func TestCacheExpiredGetKeepsFreshValue(t *testing.T) { + ctx := context.Background() + config := DefaultConfig() + config.CleanupInterval = time.Hour + cache := New(config) + defer cache.Close() + + for r := range 100 { + key := fmt.Sprintf("key-%d", r) + cache.SetWithTTL(ctx, key, "stale", 10*time.Millisecond) + time.Sleep(12 * time.Millisecond) + + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Go(func() { + <-start + cache.Get(ctx, key) // observes the expired entry and removes it + }) + wg.Go(func() { + <-start + cache.SetWithTTL(ctx, key, "fresh", time.Hour) + }) + close(start) + wg.Wait() + + if val, ok := cache.Get(ctx, key); !ok || val != "fresh" { + t.Fatalf("round %d: fresh value lost to a stale expiry removal (ok=%v)", r, ok) + } + } +} + +func TestCacheEvictionAfterConcurrentExpiry(t *testing.T) { + ctx := context.Background() + config := DefaultConfig() + config.DefaultTTL = 15 * time.Millisecond + config.CleanupInterval = time.Hour + config.MaxItems = 10 + cache := New(config) + defer cache.Close() + + // Live keys arrive alongside short-lived keys that expire and get read. + var wg sync.WaitGroup + for id := range 8 { + wg.Go(func() { + for j := range 25 { + key := fmt.Sprintf("w%d-%d", id, j) + cache.SetWithTTL(ctx, key, j, time.Hour) + if j%2 == 0 { + cache.SetWithTTL(ctx, fmt.Sprintf("tmp%d-%d", id, j), j, 5*time.Millisecond) + } + } + }) + } + wg.Wait() + time.Sleep(30 * time.Millisecond) + for id := range 8 { + for j := range 12 { + wg.Go(func() { + cache.Get(ctx, fmt.Sprintf("tmp%d-%d", id, j)) + }) + } + } + wg.Wait() + + if got := cache.Size(); got < 0 { + t.Fatalf("size drifted negative: %d", got) + } + live := 0 + for id := range 8 { + for j := range 25 { + if _, ok := cache.Get(ctx, fmt.Sprintf("w%d-%d", id, j)); ok { + live++ + } + } + } + if live == 0 { + t.Fatal("all live entries were lost") + } +} + +func TestCacheEvictionGuardIgnoresInflightReservations(t *testing.T) { + ctx := context.Background() + config := DefaultConfig() + config.CleanupInterval = time.Hour + config.MaxItems = 2 + var evictions atomic.Int64 + config.OnEviction = func(string, any) { + evictions.Add(1) + } + cache := New(config) + defer cache.Close() + + cache.Set(ctx, "x", 1) + + // Simulate a concurrent writer that reserved a count for a key it has + // not published yet. + (&cache.itemCount).Add(1) + + // Publishing another key must not evict live entries: only two entries + // are actually in the cache, which is within the limit. + cache.Set(ctx, "k", 2) + + if _, ok := cache.Get(ctx, "x"); !ok { + t.Fatal("live entry evicted by a guard that counted an in-flight reservation") + } + if got := evictions.Load(); got != 0 { + t.Fatalf("expected no eviction callbacks, got %d", got) + } + + // Roll the simulated reservation back and verify a real overflow still + // triggers eviction. + (&cache.itemCount).Add(-1) + cache.Set(ctx, "y", 3) + if cache.Size() > int64(config.MaxItems) { + t.Fatalf("cache size %d exceeds limit %d after a real overflow", cache.Size(), config.MaxItems) + } + if v, ok := cache.Get(ctx, "y"); !ok || v != 3 { + t.Fatalf("newest entry y was evicted by an overflow cleanup (ok=%v, v=%v)", ok, v) + } +} + +func TestCacheConcurrentClose(t *testing.T) { + cache := NewDefault() + + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + if err := cache.Close(); err != nil { + t.Errorf("Close returned error: %v", err) + } + }) + } + wg.Wait() +} + +func TestCacheCloseWaitsForCleanup(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + config := DefaultConfig() + config.CleanupInterval = time.Millisecond + config.OnEviction = func(string, any) { + close(started) + <-release + } + cache := New(config) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + go func() { _ = cache.Close() }() + select { + case <-cache.closedChan: + case <-time.After(5 * time.Second): + t.Error("cleanup goroutine did not exit") + } + }) + cache.SetWithTTL(context.Background(), "expired", "value", -time.Second) + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not enter the eviction callback") + } + go func() { done <- cache.Close() }() + select { + case <-cache.stopChan: + case <-time.After(5 * time.Second): + t.Fatal("Close did not signal the cleanup goroutine") + } + select { + case err := <-done: + t.Fatalf("Close returned before cleanup finished: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case err := <-done: + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close did not return after cleanup finished") + } +}