diff --git a/CHANGELOG.md b/CHANGELOG.md index f0bf92b812f48557aa1e7b77e2dc9228ac6ab48e..66d765aff2cbaf549ae69a8e122b697c6fb28666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added -* Added support for a `forKinds: ["all"]` option on datastores. ## [1.0.2] - March 3, 2020 +### Added + +* Added support for a `forKinds: ["all"]` option on datastores. + ### Fixed * Fixed a bug with the cache where it would never expire old entries unless it was pressed for space. +* Fixed a bug with the cache where the minimum cache time trigger would not work. ## [1.0.1] - February 27, 2020 diff --git a/internal_cache/media_cache.go b/internal_cache/media_cache.go index 22f613e771e57905f1b15ff3a476d34415e06516..e584f198b0b1fa80ada14483c8a36e74061fce09 100644 --- a/internal_cache/media_cache.go +++ b/internal_cache/media_cache.go @@ -70,12 +70,11 @@ func Get() *MediaCache { go func() { rctx := rcontext.Initial().LogWithFields(logrus.Fields{"task": "cache_cleanup"}) for _ = range instance.cleanupTimer.C { - rctx.Log.Info("Cleanup timer fired at") + rctx.Log.Info("Cache cleanup timer fired") maxSize := config.Get().Downloads.Cache.MaxSizeBytes - maxDownloads := config.Get().Downloads.Cache.MinDownloads * 10 - b := instance.clearSpace(maxSize, maxDownloads, maxSize, rctx) - rctx.Log.Infof("Cleared %d bytes from cache during cleanup", b) + b := instance.clearSpace(maxSize, math.MaxInt32, maxSize, true, rctx) + rctx.Log.Infof("Cleared %d bytes from cache during cleanup (%d bytes remain)", b, instance.size) } }() } @@ -204,7 +203,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn // We need to clean up some space maxSizeClear := int64(math.Ceil(float64(mediaSize) * 1.25)) ctx.Log.Info(fmt.Sprintf("Attempting to clear %d bytes from media cache (max evict size %d bytes)", mediaSize, maxSizeClear)) - clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, ctx) + clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, false, ctx) ctx.Log.Info(fmt.Sprintf("Cleared %d bytes from media cache", clearedSpace)) freeSpace += clearedSpace if freeSpace >= mediaSize { @@ -233,7 +232,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn // set the maximum file size that can be cleared to the size of the cache which // essentially allows us to remove anything. downloadsLessThan := config.Get().Downloads.Cache.MinDownloads * 4 - overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, ctx) // metrics handled internally + overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, true, ctx) // metrics handled internally ctx.Log.Infof("Cleared %d bytes from media cache", overageCleared) } @@ -253,7 +252,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn return nil, nil } -func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, ctx rcontext.RequestContext) int64 { +func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, deleteEvenIfNotEnough bool, ctx rcontext.RequestContext) int64 { // This should never happen, but we'll protect against it anyways. If we clear negative space we // end up assuming that a very small amount being cleared is enough space for the file we're about // to put in, which results in the cache growing beyond the file size limit. @@ -271,9 +270,6 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi var preppedSpace int64 = 0 for k, item := range c.cache.Items() { record := item.Object.(*cachedFile) - if int64(record.Contents.Len()) >= withSizeLessThan { - continue // file too large, cannot evict - } var recordId string if record.thumbnail != nil { @@ -282,6 +278,10 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi recordId = record.media.Sha256Hash } + if int64(record.Contents.Len()) >= withSizeLessThan { + continue // file too large, cannot evict + } + downloads := c.tracker.NumDownloads(recordId) if downloads >= withDownloadsLessThan { continue // too many downloads, cannot evict @@ -299,7 +299,7 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi } } - if preppedSpace < neededBytes { + if preppedSpace < neededBytes && !deleteEvenIfNotEnough { // not enough space prepared - don't evict anything return 0 }