diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a4ff23a696cb8af40d33ed8f0011378e26d3ec5..b2e0fbd756fca113b6992c387ef37201640a5a81 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added
* Added support for [MSC2448](https://github.com/matrix-org/matrix-doc/pull/2448).
-* Added support for a `forKinds: ["all"]` option on datastores.
* Added support for specifying a `region` to the S3 provider.
* Pass-through the `Accept-Language` header for URL previews, with options to set a default.
* Experimental support for IPFS.
@@ -23,6 +22,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
* Updated to Go 1.14
+## [1.0.2] - March 3, 2020
+
+### Added
+
+* Added support for a `forKinds: ["all"]` option on datastores.
+
+### Fixed
+
+* Fixed a bug with the cache where it would never expire old entries unless it was pressed for space.
+* Fixed a bug with the cache where the minimum cache time trigger would not work.
+
## [1.0.1] - February 27, 2020
### Fixed
@@ -67,7 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
* Various other features that would be expected like maximum/minimum size controls, rate limiting, etc. Check out the
sample config for a better idea of what else is possible.
-[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.1...HEAD
+[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.2...HEAD
+[1.0.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.1...v1.0.2
[1.0.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0...v1.0.1
[1.0.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0-rc.2...v1.0.0
[1.0.0-rc.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0-rc.1...v1.0.0-rc.2
diff --git a/cmd/media_repo/main.go b/cmd/media_repo/main.go
index d26fc9b62c473c8bdb468e4d5e01d9a540d9c9b7..3419a384ee4de3a2244d484192b60f8f6a16451c 100644
--- a/cmd/media_repo/main.go
+++ b/cmd/media_repo/main.go
@@ -12,6 +12,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/logging"
"github.com/turt2live/matrix-media-repo/common/runtime"
"github.com/turt2live/matrix-media-repo/common/version"
+ "github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/tasks"
)
@@ -73,6 +74,7 @@ func main() {
logrus.Info("Stopping recurring tasks...")
tasks.StopAll()
+ internal_cache.Get().Stop()
}
// Set up a listener for SIGINT
diff --git a/config.sample.yaml b/config.sample.yaml
index 382a4a9a43b319b6641faff512dd894611fc15f8..4163993db9d1b01674e81405d6c6988dbc130fd9 100644
--- a/config.sample.yaml
+++ b/config.sample.yaml
@@ -191,7 +191,7 @@ downloads:
failureCacheMinutes: 5
# The cache control settings for downloads. This can help speed up downloads for users by
- # keeping popular media in the cache.
+ # keeping popular media in the cache. This cache is also used for thumbnails.
cache:
enabled: true
diff --git a/controllers/download_controller/download_controller.go b/controllers/download_controller/download_controller.go
index 20d08a7c8545ced013efb63cd47f8ea6eaffa529..6eaea8749c037192121d6543a2c1c91d36d57562 100644
--- a/controllers/download_controller/download_controller.go
+++ b/controllers/download_controller/download_controller.go
@@ -98,7 +98,6 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
}
localCache.Set(origin+"/"+mediaId, media, cache.DefaultExpiration)
- internal_cache.Get().IncrementDownloads(media.Sha256Hash)
cached, err := internal_cache.Get().GetMedia(media, ctx)
if err != nil {
diff --git a/controllers/thumbnail_controller/thumbnail_controller.go b/controllers/thumbnail_controller/thumbnail_controller.go
index ee2ee86a33ec06bf91083ae4cd9077a17947bcac..0858ad44bf9901e9d05b4ca668a2a2a51879d306 100644
--- a/controllers/thumbnail_controller/thumbnail_controller.go
+++ b/controllers/thumbnail_controller/thumbnail_controller.go
@@ -159,7 +159,7 @@ func GetThumbnail(origin string, mediaId string, desiredWidth int, desiredHeight
}, nil
}
- ctx.Log.Info("Reading thumbnail from disk")
+ ctx.Log.Info("Reading thumbnail from datastore")
mediaStream, err := datastore.DownloadStream(ctx, thumbnail.DatastoreId, thumbnail.Location)
if err != nil {
return nil, err
diff --git a/internal_cache/media_cache.go b/internal_cache/media_cache.go
index a3e7a248d7f0a365401b885243e2bc0f02aec8f3..709e5078f9cab5f2e784be9f4ceac5fea1af39c3 100644
--- a/internal_cache/media_cache.go
+++ b/internal_cache/media_cache.go
@@ -27,6 +27,7 @@ type MediaCache struct {
tracker *download_tracker.DownloadTracker
size int64
enabled bool
+ cleanupTimer *time.Ticker
}
type cachedFile struct {
@@ -63,7 +64,19 @@ func Get() *MediaCache {
cache: cache.New(trackedMinutes, trackedMinutes*2),
cooldownCache: cache.New(maxCooldown*2, maxCooldown*2),
tracker: download_tracker.New(config.Get().Downloads.Cache.TrackedMinutes),
+ cleanupTimer: time.NewTicker(5 * time.Minute),
}
+
+ go func() {
+ rctx := rcontext.Initial().LogWithFields(logrus.Fields{"task": "cache_cleanup"})
+ for _ = range instance.cleanupTimer.C {
+ rctx.Log.Info("Cache cleanup timer fired")
+ maxSize := config.Get().Downloads.Cache.MaxSizeBytes
+
+ b := instance.clearSpace(maxSize, math.MaxInt32, maxSize, true, rctx)
+ rctx.Log.Infof("Cleared %d bytes from cache during cleanup (%d bytes remain)", b, instance.size)
+ }
+ }()
}
})
@@ -82,6 +95,10 @@ func (c *MediaCache) Reset() {
c.tracker.Reset()
}
+func (c *MediaCache) Stop() {
+ c.cleanupTimer.Stop()
+}
+
func (c *MediaCache) IncrementDownloads(fileHash string) {
if !c.enabled {
return
@@ -186,7 +203,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
// We need to clean up some space
maxSizeClear := int64(math.Ceil(float64(mediaSize) * 1.25))
ctx.Log.Info(fmt.Sprintf("Attempting to clear %d bytes from media cache (max evict size %d bytes)", mediaSize, maxSizeClear))
- clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, ctx)
+ clearedSpace := c.clearSpace(mediaSize, downloads, maxSizeClear, false, ctx)
ctx.Log.Info(fmt.Sprintf("Cleared %d bytes from media cache", clearedSpace))
freeSpace += clearedSpace
if freeSpace >= mediaSize {
@@ -215,7 +232,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
// set the maximum file size that can be cleared to the size of the cache which
// essentially allows us to remove anything.
downloadsLessThan := config.Get().Downloads.Cache.MinDownloads * 4
- overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, ctx) // metrics handled internally
+ overageCleared := c.clearSpace(overage, downloadsLessThan, maxSpace, true, ctx) // metrics handled internally
ctx.Log.Infof("Cleared %d bytes from media cache", overageCleared)
}
@@ -235,7 +252,7 @@ func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn
return nil, nil
}
-func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, ctx rcontext.RequestContext) int64 {
+func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64, deleteEvenIfNotEnough bool, ctx rcontext.RequestContext) int64 {
// This should never happen, but we'll protect against it anyways. If we clear negative space we
// end up assuming that a very small amount being cleared is enough space for the file we're about
// to put in, which results in the cache growing beyond the file size limit.
@@ -253,9 +270,6 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
var preppedSpace int64 = 0
for k, item := range c.cache.Items() {
record := item.Object.(*cachedFile)
- if int64(record.Contents.Len()) >= withSizeLessThan {
- continue // file too large, cannot evict
- }
var recordId string
if record.thumbnail != nil {
@@ -264,6 +278,10 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
recordId = record.media.Sha256Hash
}
+ if int64(record.Contents.Len()) >= withSizeLessThan {
+ continue // file too large, cannot evict
+ }
+
downloads := c.tracker.NumDownloads(recordId)
if downloads >= withDownloadsLessThan {
continue // too many downloads, cannot evict
@@ -281,7 +299,7 @@ func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, wi
}
}
- if preppedSpace < neededBytes {
+ if preppedSpace < neededBytes && !deleteEvenIfNotEnough {
// not enough space prepared - don't evict anything
return 0
}
@@ -346,14 +364,14 @@ func (c *MediaCache) checkExpiration(cd *cooldown, recordId string) bool {
func (c *MediaCache) flagEvicted(recordId string) {
logrus.Info("Flagging " + recordId + " as evicted (overwriting any previous cooldowns)")
- duration := int64(config.Get().Downloads.Cache.MinEvictedTimeSeconds) * 1000
- c.cooldownCache.Set(recordId, &cooldown{isEviction: true, expiresTs: duration}, cache.DefaultExpiration)
+ expireTs := (int64(config.Get().Downloads.Cache.MinEvictedTimeSeconds) * 1000) + util.NowMillis()
+ c.cooldownCache.Set(recordId, &cooldown{isEviction: true, expiresTs: expireTs}, cache.DefaultExpiration)
}
func (c *MediaCache) flagCached(recordId string) {
logrus.Info("Flagging " + recordId + " as joining the cache (overwriting any previous cooldowns)")
- duration := int64(config.Get().Downloads.Cache.MinCacheTimeSeconds) * 1000
- c.cooldownCache.Set(recordId, &cooldown{isEviction: false, expiresTs: duration}, cache.DefaultExpiration)
+ expireTs := (int64(config.Get().Downloads.Cache.MinCacheTimeSeconds) * 1000) + util.NowMillis()
+ c.cooldownCache.Set(recordId, &cooldown{isEviction: false, expiresTs: expireTs}, cache.DefaultExpiration)
}
func (c *cooldown) IsExpired() bool {