From 09f94b47762ea801c41d91b6d775fd480908e84d Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 3 Mar 2020 17:10:13 -0700
Subject: [PATCH] Clear the cache's space more efficiently
---
CHANGELOG.md | 6 +++++-
internal_cache/media_cache.go | 22 +++++++++++-----------
2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f0bf92b8..66d765af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,13 +9,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added
-* Added support for a `forKinds: ["all"]` option on datastores.
## [1.0.2] - March 3, 2020
+### Added
+
+* Added support for a `forKinds: ["all"]` option on datastores.
+
### Fixed
* Fixed a bug with the cache where it would never expire old entries unless it was pressed for space.
+* Fixed a bug with the cache where the minimum cache time trigger would not work.
## [1.0.1] - February 27, 2020
diff --git a/internal_cache/media_cache.go b/internal_cache/media_cache.go
index 22f613e7..e584f198 100644
--- a/internal_cache/media_cache.go
+++ b/internal_cache/media_cache.go
@@ -70,12 +70,11 @@ func Get() *MediaCache {
go func() {
rctx := rcontext.Initial().LogWithFields(logrus.Fields{"task": "cache_cleanup"})
for _ = range instance.cleanupTimer.C {
- rctx.Log.Info("Cleanup timer fired at")
+ rctx.Log.Info("Cache cleanup timer fired")
maxSize := config.Get().Downloads.Cache.MaxSizeBytes
- maxDownloads := config.Get().Downloads.Cache.MinDownloads * 10
- b := instance.clearSpace(maxSize, maxDownloads, maxSize, rctx)
- rctx.Log.Infof("Cleared %d bytes from cache during cleanup", b)
+ b := instance.clearSpace(maxSize, math.MaxInt32, maxSize, true, rctx)
+ rctx.Log.Infof("Cleared %d bytes from cache during cleanup (%d bytes remain)", b, instance.size)
}
}()
}
@@ -204,7 +203,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
// We need to clean up some space
maxSizeClear := int64(math.Ceil(float64(mediaSize) * 1.25))
ctx.Log.Info(fmt.Sprintf("Attempting to clear %d bytes from media cache (max evict size %d bytes)", mediaSize, maxSizeClear))
- clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, ctx)
+ clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, false, ctx)
ctx.Log.Info(fmt.Sprintf("Cleared %d bytes from media cache", clearedSpace))
freeSpace += clearedSpace
if freeSpace >= mediaSize {
@@ -233,7 +232,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
// set the maximum file size that can be cleared to the size of the cache which
// essentially allows us to remove anything.
downloadsLessThan := config.Get().Downloads.Cache.MinDownloads * 4
- overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, ctx) // metrics handled internally
+ overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, true, ctx) // metrics handled internally
ctx.Log.Infof("Cleared %d bytes from media cache", overageCleared)
}
@@ -253,7 +252,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
return nil, nil
}
-func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, ctx rcontext.RequestContext) int64 {
+func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, deleteEvenIfNotEnough bool, ctx rcontext.RequestContext) int64 {
// This should never happen, but we'll protect against it anyways. If we clear negative space we
// end up assuming that a very small amount being cleared is enough space for the file we're about
// to put in, which results in the cache growing beyond the file size limit.
@@ -271,9 +270,6 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
var preppedSpace int64 = 0
for k, item := range c.cache.Items() {
record := item.Object.(*cachedFile)
- if int64(record.Contents.Len()) >= withSizeLessThan {
- continue // file too large, cannot evict
- }
var recordId string
if record.thumbnail != nil {
@@ -282,6 +278,10 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
recordId = record.media.Sha256Hash
}
+ if int64(record.Contents.Len()) >= withSizeLessThan {
+ continue // file too large, cannot evict
+ }
+
downloads := c.tracker.NumDownloads(recordId)
if downloads >= withDownloadsLessThan {
continue // too many downloads, cannot evict
@@ -299,7 +299,7 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
}
}
- if preppedSpace < neededBytes {
+ if preppedSpace < neededBytes && !deleteEvenIfNotEnough {
// not enough space prepared - don't evict anything
return 0
}
--
GitLab