From 923cd39470c8c76a45e44e9fe30e48f8fc5cb2c5 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Thu, 10 Feb 2022 22:15:42 -0700
Subject: [PATCH] Move download cache call to earlier in the download process
---
CHANGELOG.md | 5 ++--
.../download_controller.go | 23 ++++++++++---------
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e992d1ac..07974e9b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
* New config option to set user agent when requesting URL previews.
* Added support for `image/jxl` thumbnailing.
-* Built-in early support for content ranges (being able to skip around in audio and video).
+* Built-in early support for content ranges (being able to skip around in audio and video). This is only available if
+ caching is enabled.
### Removed
@@ -19,13 +20,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
* Support for the Redis config under `features` has been removed. It is now only available at the top level of the
config. See the sample config for more details.
-
### Fixed
* Fixed media being permanently lost when transferring to an (effectively) readonly S3 datastore.
* Purging non-existent files now won't cause errors.
* Fixed HEIF/HEIC thumbnailing. Note that this thumbnail type might cause increased memory usage.
* Ensure endpoints register in a stable way, making them predictably available.
+* Reduced download hits to datastores when using Redis cache.
### Changed
diff --git a/controllers/download_controller/download_controller.go b/controllers/download_controller/download_controller.go
index fd403c48..4d7f354b 100644
--- a/controllers/download_controller/download_controller.go
+++ b/controllers/download_controller/download_controller.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"github.com/getsentry/sentry-go"
+ "io"
"io/ioutil"
"time"
@@ -103,16 +104,6 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
}
localCache.Set(origin+"/"+mediaId, media, cache.DefaultExpiration)
-
- cached, err := internal_cache.Get().GetMedia(media.Sha256Hash, internal_cache.StreamerForMedia(media), ctx)
- if err != nil {
- return nil, err
- }
- if cached != nil && cached.Contents != nil {
- cleanup.DumpAndCloseStream(minMedia.Stream) // close the other stream first
- minMedia.Stream = ioutil.NopCloser(cached.Contents)
- return minMedia, nil
- }
}
if minMedia.Stream != nil {
@@ -235,10 +226,20 @@ func FindMinimalMediaRecord(origin string, mediaId string, downloadRemote bool,
return nil, common.ErrMediaNotFound
}
- mediaStream, err := datastore.DownloadStream(ctx, media.DatastoreId, media.Location)
+ var mediaStream io.ReadCloser
+
+ cached, err := internal_cache.Get().GetMedia(media.Sha256Hash, internal_cache.StreamerForMedia(media), ctx)
if err != nil {
return nil, err
}
+ if cached != nil && cached.Contents != nil {
+ mediaStream = ioutil.NopCloser(cached.Contents)
+ } else {
+ mediaStream, err = datastore.DownloadStream(ctx, media.DatastoreId, media.Location)
+ if err != nil {
+ return nil, err
+ }
+ }
return &types.MinimalMedia{
Origin: media.Origin,
--
GitLab