From 4ab640beb1859d1c05035f2bcfe38318c56be0c4 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sat, 16 Jun 2018 17:05:10 -0600
Subject: [PATCH] First draft for the new middle layer
Downloads go through quite a few layers, and some of those layers are copy/pasted. Needs some refactoring, but it works.
Part of #58
---
.../matrix-media-repo/api/r0/download.go | 6 +-
.../download_controller.go | 88 ++++++
.../download_resource_handler.go | 183 ++++++++++++
.../remote_media_downloader.go | 110 +++++++
.../internal_cache/cache_types.go | 23 ++
.../internal_cache/media_cache.go | 273 ++++++++++++++++++
.../media_service/resource_handler.go | 2 +-
.../thumbnail_service/resource_handler.go | 2 +-
.../services/url_service/resource_handler.go | 2 +-
.../resource_handler/handler.go | 0
10 files changed, 682 insertions(+), 7 deletions(-)
create mode 100644 src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
create mode 100644 src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_resource_handler.go
create mode 100644 src/github.com/turt2live/matrix-media-repo/controllers/download_controller/remote_media_downloader.go
create mode 100644 src/github.com/turt2live/matrix-media-repo/internal_cache/cache_types.go
create mode 100644 src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
rename src/github.com/turt2live/matrix-media-repo/{old_middle_layer => util}/resource_handler/handler.go (100%)
diff --git a/src/github.com/turt2live/matrix-media-repo/api/r0/download.go b/src/github.com/turt2live/matrix-media-repo/api/r0/download.go
index e27d41de..adff9cc4 100644
--- a/src/github.com/turt2live/matrix-media-repo/api/r0/download.go
+++ b/src/github.com/turt2live/matrix-media-repo/api/r0/download.go
@@ -9,7 +9,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/old_middle_layer/media_cache"
+ "github.com/turt2live/matrix-media-repo/controllers/download_controller"
)
type DownloadMediaResponse struct {
@@ -43,9 +43,7 @@ func DownloadMedia(r *http.Request, log *logrus.Entry, user api.UserInfo) interf
"allowRemote": downloadRemote,
})
- mediaCache := media_cache.Create(r.Context(), log)
-
- streamedMedia, err := mediaCache.GetMedia(server, mediaId, downloadRemote)
+ streamedMedia, err := download_controller.GetMedia(server, mediaId, downloadRemote, r.Context(), log)
if err != nil {
if err == common.ErrMediaNotFound {
return api.NotFoundError()
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
new file mode 100644
index 00000000..0d079331
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
@@ -0,0 +1,88 @@
+package download_controller
+
+import (
+ "context"
+ "database/sql"
+ "os"
+ "time"
+
+ "github.com/patrickmn/go-cache"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/internal_cache"
+ "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/types"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+var localCache = cache.New(30*time.Second, 60*time.Second)
+
+func GetMedia(origin string, mediaId string, downloadRemote bool, ctx context.Context, log *logrus.Entry) (*types.StreamedMedia, error) {
+ media, err := FindMediaRecord(origin, mediaId, downloadRemote, ctx, log)
+ if err != nil {
+ return nil, err
+ }
+
+ localCache.Set(origin+"/"+mediaId, media, cache.DefaultExpiration)
+ internal_cache.Get().IncrementDownloads(media.Sha256Hash)
+
+ cached, err := internal_cache.Get().GetMedia(media, log)
+ if err != nil {
+ return nil, err
+ }
+ if cached != nil && cached.Contents != nil {
+ return &types.StreamedMedia{
+ Media: media,
+ Stream: util.BufferToStream(cached.Contents),
+ }, nil
+ }
+
+ log.Info("Reading media from disk")
+ stream, err := os.Open(media.Location)
+ if err != nil {
+ return nil, err
+ }
+
+ return &types.StreamedMedia{Media: media, Stream: stream}, nil
+}
+
+func FindMediaRecord(origin string, mediaId string, downloadRemote bool, ctx context.Context, log *logrus.Entry) (*types.Media, error) {
+ db := storage.GetDatabase().GetMediaStore(ctx, log)
+
+ var media *types.Media
+ item, found := localCache.Get(origin + "/" + mediaId)
+ if found {
+ media = item.(*types.Media)
+ } else {
+ log.Info("Getting media record from database")
+ dbMedia, err := db.Get(origin, mediaId)
+ if err != nil {
+ if err == sql.ErrNoRows {
+ if util.IsServerOurs(origin) {
+ log.Warn("Media not found")
+ return nil, common.ErrMediaNotFound
+ }
+ }
+
+ if !downloadRemote {
+ log.Warn("Remote media not being downloaded")
+ return nil, common.ErrMediaNotFound
+ }
+
+ result := <-getResourceHandler().DownloadRemoteMedia(origin, mediaId)
+ if result.err != nil {
+ return nil, result.err
+ }
+ media = result.media
+ } else {
+ media = dbMedia
+ }
+ }
+
+ if media == nil {
+ log.Warn("Despite all efforts, a media record could not be found")
+ return nil, common.ErrMediaNotFound
+ }
+
+ return media, nil
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_resource_handler.go b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_resource_handler.go
new file mode 100644
index 00000000..17db3393
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_resource_handler.go
@@ -0,0 +1,183 @@
+package download_controller
+
+import (
+ "context"
+ "io"
+ "os"
+ "sync"
+
+ "github.com/ryanuber/go-glob"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/types"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/resource_handler"
+)
+
+type mediaResourceHandler struct {
+ resourceHandler *resource_handler.ResourceHandler
+}
+
+type downloadRequest struct {
+ origin string
+ mediaId string
+}
+
+type downloadResponse struct {
+ media *types.Media
+ err error
+}
+
+var resHandler *mediaResourceHandler
+var resHandlerLock = &sync.Once{}
+
+func getResourceHandler() (*mediaResourceHandler) {
+ if resHandler == nil {
+ resHandlerLock.Do(func() {
+ handler, err := resource_handler.New(config.Get().Downloads.NumWorkers, downloadResourceWorkFn)
+ if err != nil {
+ panic(err)
+ }
+
+ resHandler = &mediaResourceHandler{handler}
+ })
+ }
+
+ return resHandler
+}
+
+func (h *mediaResourceHandler) DownloadRemoteMedia(origin string, mediaId string) chan *downloadResponse {
+ resultChan := make(chan *downloadResponse)
+ go func() {
+ reqId := "remote_download:" + origin + "_" + mediaId
+ result := <-h.resourceHandler.GetResource(reqId, &downloadRequest{origin, mediaId})
+ resultChan <- result.(*downloadResponse)
+ }()
+ return resultChan
+}
+
+func downloadResourceWorkFn(request *resource_handler.WorkRequest) interface{} {
+ info := request.Metadata.(*downloadRequest)
+ log := logrus.WithFields(logrus.Fields{
+ "worker_requestId": request.Id,
+ "worker_requestOrigin": info.origin,
+ "worker_requestMediaId": info.mediaId,
+ })
+ log.Info("Downloading remote media")
+
+ ctx := context.TODO() // TODO: Should we use a real context?
+
+ downloader := newRemoteMediaDownloader(ctx, log)
+ downloaded, err := downloader.Download(info.origin, info.mediaId)
+ if err != nil {
+ return &downloadResponse{err: err}
+ }
+
+ defer downloaded.Contents.Close()
+
+ media, err := storeMedia(downloaded.Contents, downloaded.ContentType, downloaded.DesiredFilename, info.origin, info.mediaId, ctx, log)
+ if err != nil {
+ return &downloadResponse{err: err}
+ }
+
+ return &downloadResponse{media, err}
+}
+
+func storeMedia(contents io.Reader, contentType string, filename string, origin string, mediaId string, ctx context.Context, log *logrus.Entry) (*types.Media, error) {
+ fileLocation, err := storage.PersistFile(contents, ctx, log)
+ if err != nil {
+ return nil, err
+ }
+
+ fileMime, err := util.GetMimeType(fileLocation)
+ if err != nil {
+ log.Error("Error while checking content type of file: ", err.Error())
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ for _, allowedType := range config.Get().Uploads.AllowedTypes {
+ if !glob.Glob(allowedType, fileMime) {
+ log.Warn("Content type " + fileMime +" (reported as " + contentType+") is not allowed to be uploaded")
+
+ os.Remove(fileLocation) // delete temp file
+ return nil, common.ErrMediaNotAllowed
+ }
+ }
+
+ hash, err := storage.GetFileHash(fileLocation)
+ if err != nil {
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ db := storage.GetDatabase().GetMediaStore(ctx, log)
+ records, err := db.GetByHash(hash)
+ if err != nil {
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ if len(records) > 0 {
+ log.Info("Duplicate media for hash ", hash)
+
+ // We'll use the location from the first record
+ media := records[0]
+ media.Origin = origin
+ media.MediaId = mediaId
+ media.UserId = ""
+ media.UploadName = filename
+ media.ContentType = contentType
+ media.CreationTs = util.NowMillis()
+
+ err = db.Insert(media)
+ if err != nil {
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ // If the media's file exists, we'll delete the temp file
+ // If the media's file doesn't exist, we'll move the temp file to where the media expects it to be
+ exists, err := util.FileExists(media.Location)
+ if err != nil || !exists {
+ // We'll assume an error means it doesn't exist
+ os.Rename(fileLocation, media.Location)
+ } else {
+ os.Remove(fileLocation)
+ }
+
+ return media, nil
+ }
+
+ // The media doesn't already exist - save it as new
+
+ fileSize, err := util.FileSize(fileLocation)
+ if err != nil {
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ log.Info("Persisting new media record")
+
+ media := &types.Media{
+ Origin: origin,
+ MediaId: mediaId,
+ UploadName: filename,
+ ContentType: contentType,
+ UserId: "",
+ Sha256Hash: hash,
+ SizeBytes: fileSize,
+ Location: fileLocation,
+ CreationTs: util.NowMillis(),
+ }
+
+ err = db.Insert(media)
+ if err != nil {
+ os.Remove(fileLocation) // delete temp file
+ return nil, err
+ }
+
+ return media, nil
+}
\ No newline at end of file
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/remote_media_downloader.go b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/remote_media_downloader.go
new file mode 100644
index 00000000..15c00c82
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/remote_media_downloader.go
@@ -0,0 +1,110 @@
+package download_controller
+
+import (
+ "context"
+ "errors"
+ "io"
+ "mime"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/patrickmn/go-cache"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/matrix"
+)
+
+type downloadedMedia struct {
+ Contents io.ReadCloser
+ DesiredFilename string
+ ContentType string
+}
+
+type remoteMediaDownloader struct {
+ ctx context.Context
+ log *logrus.Entry
+}
+
+var downloadErrorsCache *cache.Cache
+var downloadErrorCacheSingletonLock = &sync.Once{}
+
+func newRemoteMediaDownloader(ctx context.Context, log *logrus.Entry) *remoteMediaDownloader {
+ return &remoteMediaDownloader{ctx, log}
+}
+
+func (r *remoteMediaDownloader) Download(server string, mediaId string) (*downloadedMedia, error) {
+ if downloadErrorsCache == nil {
+ downloadErrorCacheSingletonLock.Do(func() {
+ cacheTime := time.Duration(config.Get().Downloads.FailureCacheMinutes) * time.Minute
+ downloadErrorsCache = cache.New(cacheTime, cacheTime*2)
+ })
+ }
+
+ cacheKey := server + "/" + mediaId
+ item, found := downloadErrorsCache.Get(cacheKey)
+ if found {
+ r.log.Warn("Returning cached error for remote media download failure")
+ return nil, item.(error)
+ }
+
+ baseUrl, err := matrix.GetServerApiUrl(server)
+ if err != nil {
+ downloadErrorsCache.Set(cacheKey, err, cache.DefaultExpiration)
+ return nil, err
+ }
+
+ downloadUrl := baseUrl + "/_matrix/media/v1/download/" + server + "/" + mediaId + "?allow_remote=false"
+ resp, err := matrix.FederatedGet(downloadUrl, server)
+ if err != nil {
+ downloadErrorsCache.Set(cacheKey, err, cache.DefaultExpiration)
+ return nil, err
+ }
+
+ if resp.StatusCode == 404 {
+ r.log.Info("Remote media not found")
+
+ err = common.ErrMediaNotFound
+ downloadErrorsCache.Set(cacheKey, err, cache.DefaultExpiration)
+ return nil, err
+ } else if resp.StatusCode != 200 {
+ r.log.Info("Unknown error fetching remote media; received status code " + strconv.Itoa(resp.StatusCode))
+
+ err = errors.New("could not fetch remote media")
+ downloadErrorsCache.Set(cacheKey, err, cache.DefaultExpiration)
+ return nil, err
+ }
+
+ var contentLength int64 = 0
+ if resp.Header.Get("Content-Length") != "" {
+ contentLength, err = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ r.log.Warn("Missing Content-Length header on response - continuing anyway")
+ }
+
+ if contentLength > 0 && config.Get().Downloads.MaxSizeBytes > 0 && contentLength > config.Get().Downloads.MaxSizeBytes {
+ r.log.Warn("Attempted to download media that was too large")
+
+ err = common.ErrMediaTooLarge
+ downloadErrorsCache.Set(cacheKey, err, cache.DefaultExpiration)
+ return nil, err
+ }
+
+ request := &downloadedMedia{
+ ContentType: resp.Header.Get("Content-Type"),
+ Contents: resp.Body,
+ // DesiredFilename (calculated below)
+ }
+
+ _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
+ if err == nil && params["filename"] != "" {
+ request.DesiredFilename = params["filename"]
+ }
+
+ r.log.Info("Persisting downloaded media")
+ return request, nil
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/internal_cache/cache_types.go b/src/github.com/turt2live/matrix-media-repo/internal_cache/cache_types.go
new file mode 100644
index 00000000..efbd7b14
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/internal_cache/cache_types.go
@@ -0,0 +1,23 @@
+package internal_cache
+
+import (
+ "bytes"
+
+ "github.com/turt2live/matrix-media-repo/types"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+type cachedFile struct {
+ media *types.Media
+ thumbnail *types.Thumbnail
+ Contents *bytes.Buffer
+}
+
+type cooldown struct {
+ isEviction bool
+ expiresTs int64
+}
+
+func (c *cooldown) IsExpired() bool {
+ return util.NowMillis() >= c.expiresTs
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go b/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
new file mode 100644
index 00000000..6b80667e
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
@@ -0,0 +1,273 @@
+package internal_cache
+
+import (
+ "bytes"
+ "container/list"
+ "fmt"
+ "io/ioutil"
+ "sync"
+ "time"
+
+ "github.com/patrickmn/go-cache"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/types"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/download_tracker"
+)
+
+type MediaCache struct {
+ cache *cache.Cache
+ cooldownCache *cache.Cache
+ tracker *download_tracker.DownloadTracker
+ size int64
+ enabled bool
+}
+
+var instance *MediaCache
+var lock = &sync.Once{}
+
+func Get() (*MediaCache) {
+ if instance != nil {
+ return instance
+ }
+
+ lock.Do(func() {
+ if !config.Get().Downloads.Cache.Enabled {
+ logrus.Warn("Cache is disabled - setting up a dummy instance")
+ instance = &MediaCache{enabled: false}
+ } else {
+ logrus.Info("Setting up cache")
+ trackedMinutes := time.Duration(config.Get().Downloads.Cache.TrackedMinutes) * time.Minute
+ maxCooldownSec := util.MaxInt(config.Get().Downloads.Cache.MinEvictedTimeSeconds, config.Get().Downloads.Cache.MinCacheTimeSeconds)
+ maxCooldown := time.Duration(maxCooldownSec) * time.Second
+ instance = &MediaCache{
+ enabled: true,
+ size: 0,
+ cache: cache.New(trackedMinutes, trackedMinutes*2),
+ cooldownCache: cache.New(maxCooldown*2, maxCooldown*2),
+ tracker: download_tracker.New(config.Get().Downloads.Cache.TrackedMinutes),
+ }
+ }
+ })
+
+ return instance
+}
+
+func (c *MediaCache) Reset() {
+ if !c.enabled {
+ return
+ }
+
+ logrus.Warn("Resetting media cache")
+ c.cache.Flush()
+ c.cooldownCache.Flush()
+ c.size = 0
+ c.tracker.Reset()
+}
+
+func (c *MediaCache) IncrementDownloads(fileHash string) {
+ if !c.enabled {
+ return
+ }
+
+ logrus.Info("File " + fileHash + " has been downloaded")
+ c.tracker.Increment(fileHash)
+}
+
+func (c *MediaCache) GetMedia(media *types.Media, log *logrus.Entry) (*cachedFile, error) {
+ if !c.enabled {
+ return nil, nil
+ }
+
+ cacheFn := func() (*cachedFile, error) {
+ data, err := ioutil.ReadFile(media.Location)
+ if err != nil {
+ return nil, err
+ }
+
+ return &cachedFile{media: media, Contents: bytes.NewBuffer(data)}, nil
+ }
+
+ return c.updateItemInCache(media.Sha256Hash, media.SizeBytes, cacheFn, log)
+}
+
+func (c *MediaCache) updateItemInCache(recordId string, mediaSize int64, cacheFn func() (*cachedFile, error), log *logrus.Entry) (*cachedFile, error) {
+ downloads := c.tracker.NumDownloads(recordId)
+ enoughDownloads := downloads >= config.Get().Downloads.Cache.MinDownloads
+ canCache := c.canJoinCache(recordId)
+ item, found := c.cache.Get(recordId)
+
+ // No longer eligible for the cache - delete item
+ // The cached bytes will leave memory over time
+ if found && !enoughDownloads {
+ log.Info("Removing media from cache because it does not have enough downloads")
+ c.cache.Delete(recordId)
+ c.flagEvicted(recordId)
+ return nil, nil
+ }
+
+ // Eligible for the cache, but not in it currently (and not on cooldown)
+ if !found && enoughDownloads && canCache {
+ maxSpace := config.Get().Downloads.Cache.MaxSizeBytes
+ usedSpace := c.size
+ freeSpace := maxSpace - usedSpace
+ mediaSize := mediaSize
+
+ // Don't bother checking for space if it won't fit anyways
+ if mediaSize > maxSpace {
+ log.Warn("Media too large to cache")
+ return nil, nil
+ }
+
+ if freeSpace >= mediaSize {
+ // Perfect! It'll fit - just cache it
+ log.Info("Caching file in memory")
+ c.size = usedSpace + mediaSize
+ c.flagCached(recordId)
+
+ cachedItem, err := cacheFn()
+ if err != nil {
+ return nil, err
+ }
+ c.cache.Set(recordId, cachedItem, cache.DefaultExpiration)
+ }
+
+ // We need to clean up some space
+ neededSize := (usedSpace + mediaSize) - maxSpace
+ log.Info(fmt.Sprintf("Attempting to clear %d bytes from media cache", neededSize))
+ clearedSpace := c.clearSpace(neededSize, downloads, mediaSize)
+ log.Info(fmt.Sprintf("Cleared %d bytes from media cache", clearedSpace))
+ freeSpace += clearedSpace
+ if freeSpace >= mediaSize {
+ // Now it'll fit - cache it
+ log.Info("Caching file in memory")
+ c.size = usedSpace + mediaSize
+ c.flagCached(recordId)
+
+ cachedItem, err := cacheFn()
+ if err != nil {
+ return nil, err
+ }
+ c.cache.Set(recordId, cachedItem, cache.DefaultExpiration)
+ }
+
+ log.Warn("Unable to clear enough space for file to be cached")
+ return nil, nil
+ }
+
+ // By now the media should be in the correct state (cached or not)
+ if found {
+ return item.(*cachedFile), nil
+ }
+ return nil, nil
+}
+
+func (c *MediaCache) clearSpace(neededBytes int64, withDownloadsLessThan int, withSizeLessThan int64) int64 {
+ type removable struct {
+ cacheKey string
+ recordId string
+ }
+
+ keysToClear := list.New()
+ var preppedSpace int64 = 0
+ for k, item := range c.cache.Items() {
+ record := item.Object.(*cachedFile)
+ if int64(record.Contents.Len()) >= withSizeLessThan {
+ continue // file too large, cannot evict
+ }
+
+ var recordId string
+ if record.thumbnail != nil {
+ recordId = *record.thumbnail.Sha256Hash
+ } else {
+ recordId = record.media.Sha256Hash
+ }
+
+ downloads := c.tracker.NumDownloads(recordId)
+ if downloads >= withDownloadsLessThan {
+ continue // too many downloads, cannot evict
+ }
+
+ if !c.canLeaveCache(recordId) {
+ continue // on cooldown, cannot evict
+ }
+
+ // Small enough and has an appropriate file size
+ preppedSpace += int64(record.Contents.Len())
+ keysToClear.PushBack(&removable{k, recordId})
+ if preppedSpace >= neededBytes {
+ break // cleared enough space - clear it out
+ }
+ }
+
+ if preppedSpace < neededBytes {
+ // not enough space prepared - don't evict anything
+ return 0
+ }
+
+ for e := keysToClear.Front(); e != nil; e = e.Next() {
+ toRemove := e.Value.(*removable)
+ c.cache.Delete(toRemove.cacheKey)
+ c.flagEvicted(toRemove.recordId)
+ }
+
+ return preppedSpace
+}
+
+func (c *MediaCache) canJoinCache(recordId string) bool {
+ item, found := c.cooldownCache.Get(recordId)
+ if !found {
+ return true // No cooldown means we're probably fine
+ }
+
+ cd := item.(*cooldown)
+ if !cd.isEviction {
+ return true // It should already be in the cache anyways
+ }
+
+ return c.checkExpiration(cd, recordId)
+}
+
+func (c *MediaCache) canLeaveCache(recordId string) bool {
+ item, found := c.cooldownCache.Get(recordId)
+ if !found {
+ return true // No cooldown means we're probably fine
+ }
+
+ cd := item.(*cooldown)
+ if cd.isEviction {
+ return true // It should already be outside the cache anyways
+ }
+
+ return c.checkExpiration(cd, recordId)
+}
+
+func (c *MediaCache) checkExpiration(cd *cooldown, recordId string) bool {
+ cdType := "Joined cache"
+ if cd.isEviction {
+ cdType = "Eviction"
+ }
+
+ expired := cd.IsExpired()
+ if expired {
+ logrus.Info(cdType + " cooldown for " + recordId + " has expired")
+ c.cooldownCache.Delete(recordId) // cleanup
+ return true
+ }
+
+ logrus.Warn(cdType + " cooldown on " + recordId + " is still active")
+ return false
+}
+
+func (c *MediaCache) flagEvicted(recordId string) {
+ logrus.Info("Flagging " + recordId + " as evicted (overwriting any previous cooldowns)")
+ duration := int64(config.Get().Downloads.Cache.MinEvictedTimeSeconds) * 1000
+ c.cooldownCache.Set(recordId, &cooldown{isEviction: true, expiresTs: duration}, cache.DefaultExpiration)
+}
+
+func (c *MediaCache) flagCached(recordId string) {
+ logrus.Info("Flagging " + recordId + " as joining the cache (overwriting any previous cooldowns)")
+ duration := int64(config.Get().Downloads.Cache.MinCacheTimeSeconds) * 1000
+ c.cooldownCache.Set(recordId, &cooldown{isEviction: false, expiresTs: duration}, cache.DefaultExpiration)
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/media_service/resource_handler.go b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/media_service/resource_handler.go
index b823277d..f60fc813 100644
--- a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/media_service/resource_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/media_service/resource_handler.go
@@ -6,7 +6,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/old_middle_layer/resource_handler"
+ "github.com/turt2live/matrix-media-repo/util/resource_handler"
"github.com/turt2live/matrix-media-repo/types"
)
diff --git a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/thumbnail_service/resource_handler.go b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/thumbnail_service/resource_handler.go
index 0a68c8c8..4b913b1e 100644
--- a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/thumbnail_service/resource_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/thumbnail_service/resource_handler.go
@@ -7,7 +7,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/old_middle_layer/resource_handler"
+ "github.com/turt2live/matrix-media-repo/util/resource_handler"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
)
diff --git a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/url_service/resource_handler.go b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/url_service/resource_handler.go
index 5a34e108..6e490de4 100644
--- a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/url_service/resource_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/old_middle_layer/services/url_service/resource_handler.go
@@ -9,7 +9,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/old_middle_layer/resource_handler"
+ "github.com/turt2live/matrix-media-repo/util/resource_handler"
"github.com/turt2live/matrix-media-repo/old_middle_layer/services/media_service"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
diff --git a/src/github.com/turt2live/matrix-media-repo/old_middle_layer/resource_handler/handler.go b/src/github.com/turt2live/matrix-media-repo/util/resource_handler/handler.go
similarity index 100%
rename from src/github.com/turt2live/matrix-media-repo/old_middle_layer/resource_handler/handler.go
rename to src/github.com/turt2live/matrix-media-repo/util/resource_handler/handler.go
--
GitLab