From 81bfd75601fbda60c25c9a5819b3f4758587f6e7 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sun, 24 Feb 2019 16:54:07 -0700
Subject: [PATCH] Support downloading from s3 (and other datastores)
---
.../download_controller.go | 19 ++-----
.../maintainance_controller.go | 11 ++--
.../preview_resource_handler.go | 7 +--
.../thumbnail_controller.go | 10 ++--
.../thumbnail_resource_handler.go | 50 +++++++++----------
.../upload_controller/upload_controller.go | 12 ++---
.../internal_cache/media_cache.go | 10 ++--
.../storage/datastore/datastore.go | 33 +++++++++++-
.../storage/datastore/datastore_ref.go | 26 +++++++---
.../storage/datastore/ds_file/file_store.go | 5 +-
.../storage/datastore/ds_s3/s3_store.go | 12 +++--
.../matrix-media-repo/storage/ds_utils.go | 15 ------
.../storage/startup_migrations.go | 2 +-
.../matrix-media-repo/types/datastore.go | 8 ---
.../matrix-media-repo/util/random.go | 4 +-
.../matrix-media-repo/util/util_exif/exif.go | 11 ++--
16 files changed, 122 insertions(+), 113 deletions(-)
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
index e163615e..7017eb19 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"errors"
- "os"
"time"
"github.com/patrickmn/go-cache"
@@ -12,6 +11,7 @@ import (
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
)
@@ -92,16 +92,12 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
}
log.Info("Reading media from disk")
- filePath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
- if err != nil {
- return nil, err
- }
- stream, err := os.Open(filePath)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
return nil, err
}
- minMedia.Stream = stream
+ minMedia.Stream = mediaStream
return minMedia, nil
}
@@ -151,12 +147,7 @@ func FindMinimalMediaRecord(origin string, mediaId string, downloadRemote bool,
return nil, common.ErrMediaNotFound
}
- filePath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
- if err != nil {
- return nil, err
- }
-
- stream, err := os.Open(filePath)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
return nil, err
}
@@ -167,7 +158,7 @@ func FindMinimalMediaRecord(origin string, mediaId string, downloadRemote bool,
ContentType: media.ContentType,
UploadName: media.UploadName,
SizeBytes: media.SizeBytes,
- Stream: stream,
+ Stream: mediaStream,
KnownMedia: media,
}, nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/maintenance_controller/maintainance_controller.go b/src/github.com/turt2live/matrix-media-repo/controllers/maintenance_controller/maintainance_controller.go
index 3836f00e..6acc43ff 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/maintenance_controller/maintainance_controller.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/maintenance_controller/maintainance_controller.go
@@ -3,10 +3,10 @@ package maintenance_controller
import (
"context"
"fmt"
- "os"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/util"
)
@@ -39,13 +39,14 @@ func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Ent
continue
}
- // Delete the file first
- filePath, err := storage.ResolveMediaLocation(context.TODO(), &logrus.Entry{}, media.DatastoreId, media.Location)
+ ds, err := datastore.LocateDatastore(context.TODO(), &logrus.Entry{}, media.DatastoreId)
if err != nil {
- log.Error("Error resolving datastore path for media " + media.Origin + "/" + media.MediaId + " because: " + err.Error())
+ log.Error("Error finding datastore for media " + media.Origin + "/" + media.MediaId + " because: " + err.Error())
continue
}
- err = os.Remove(filePath)
+
+ // Delete the file first
+ err = ds.DeleteObject(media.Location)
if err != nil {
log.Warn("Cannot remove media " + media.Origin + "/" + media.MediaId + " because: " + err.Error())
} else {
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_resource_handler.go b/src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_resource_handler.go
index 7cf35e99..4b52c21a 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_resource_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_resource_handler.go
@@ -12,6 +12,7 @@ import (
"github.com/turt2live/matrix-media-repo/controllers/preview_controller/previewers"
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
"github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
"github.com/turt2live/matrix-media-repo/util/resource_handler"
@@ -98,11 +99,11 @@ func urlPreviewWorkFn(request *resource_handler.WorkRequest) interface{} {
if err != nil {
log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
} else {
- filePath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
- log.Warn("Non-fatal error resolving datastore path: " + err.Error())
+ log.Warn("Non-fatal error streaming datastore file: " + err.Error())
} else {
- img, err := imaging.Open(filePath)
+ img, err := imaging.Decode(mediaStream)
if err != nil {
log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
} else {
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_controller.go b/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_controller.go
index 5715df6c..222d2ff9 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_controller.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_controller.go
@@ -8,7 +8,6 @@ import (
"image"
"image/color"
"math"
- "os"
"time"
"github.com/disintegration/imaging"
@@ -22,6 +21,7 @@ import (
"github.com/turt2live/matrix-media-repo/controllers/download_controller"
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
"golang.org/x/image/font/gofont/gosmallcaps"
@@ -158,16 +158,12 @@ func GetThumbnail(origin string, mediaId string, desiredWidth int, desiredHeight
}
log.Info("Reading thumbnail from disk")
- filePath, err := storage.ResolveMediaLocation(ctx, log, thumbnail.DatastoreId, thumbnail.Location)
- if err != nil {
- return nil, err
- }
- stream, err := os.Open(filePath)
+ mediaStream, err := datastore.DownloadStream(ctx, log, thumbnail.DatastoreId, thumbnail.Location)
if err != nil {
return nil, err
}
- return &types.StreamedThumbnail{Thumbnail: thumbnail, Stream: stream}, nil
+ return &types.StreamedThumbnail{Thumbnail: thumbnail, Stream: mediaStream}, nil
}
func GetOrGenerateThumbnail(media *types.Media, width int, height int, animated bool, method string, ctx context.Context, log *logrus.Entry) (*types.Thumbnail, error) {
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_resource_handler.go b/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_resource_handler.go
index a3a432a9..978d993b 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_resource_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_resource_handler.go
@@ -8,6 +8,7 @@ import (
"image"
"image/draw"
"image/gif"
+ "io"
"io/ioutil"
"math"
"os"
@@ -145,12 +146,12 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
} else if canAnimate && !animated {
src, err = pickImageFrame(media, ctx, log)
} else {
- mediaPath, err2 := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+ mediaStream, err2 := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err2 != nil {
- log.Error("Error resolving datastore path: ", err2)
+ log.Error("Error getting file: ", err2)
return nil, err2
}
- src, err = imaging.Open(mediaPath)
+ src, err = imaging.Decode(mediaStream)
}
if err != nil {
@@ -220,19 +221,13 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
// Animated GIFs are a bit more special because we need to do it frame by frame.
// This is fairly resource intensive. The calling code is responsible for limiting this case.
- mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
log.Error("Error resolving datastore path: ", err)
return nil, err
}
- inputFile, err := os.Open(mediaPath)
- if err != nil {
- log.Error("Error generating animated thumbnail: " + err.Error())
- return nil, err
- }
- defer inputFile.Close()
- g, err := gif.DecodeAll(inputFile)
+ g, err := gif.DecodeAll(mediaStream)
if err != nil {
log.Error("Error generating animated thumbnail: " + err.Error())
return nil, err
@@ -342,21 +337,32 @@ func thumbnailFrame(src image.Image, method string, width int, height int, filte
}
func svgToImage(media *types.Media, ctx context.Context, log *logrus.Entry) (image.Image, error) {
- tempFile := path.Join(os.TempDir(), "media_repo."+media.Origin+"."+media.MediaId+".png")
- defer os.Remove(tempFile)
+ tempFile1 := path.Join(os.TempDir(), "media_repo."+media.Origin+"."+media.MediaId+".1.png")
+ tempFile2 := path.Join(os.TempDir(), "media_repo."+media.Origin+"."+media.MediaId+".2.png")
+
+ defer os.Remove(tempFile1)
+ defer os.Remove(tempFile2)
// requires imagemagick
- mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
- log.Error("Error resolving datastore path: ", err)
+ log.Error("Error streaming file: ", err)
return nil, err
}
- err = exec.Command("convert", mediaPath, tempFile).Run()
+
+ f, err := os.Open(tempFile1)
if err != nil {
return nil, err
}
+ io.Copy(f, mediaStream)
+ f.Close()
- b, err := ioutil.ReadFile(tempFile)
+ err = exec.Command("convert", tempFile1, tempFile2).Run()
+ if err != nil {
+ return nil, err
+ }
+
+ b, err := ioutil.ReadFile(tempFile2)
if err != nil {
return nil, err
}
@@ -366,19 +372,13 @@ func svgToImage(media *types.Media, ctx context.Context, log *logrus.Entry) (ima
}
func pickImageFrame(media *types.Media, ctx context.Context, log *logrus.Entry) (image.Image, error) {
- mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+ mediaStream, err := datastore.DownloadStream(ctx, log, media.DatastoreId, media.Location)
if err != nil {
log.Error("Error resolving datastore path: ", err)
return nil, err
}
- inputFile, err := os.Open(mediaPath)
- if err != nil {
- log.Error("Error picking frame: " + err.Error())
- return nil, err
- }
- defer inputFile.Close()
- g, err := gif.DecodeAll(inputFile)
+ g, err := gif.DecodeAll(mediaStream)
if err != nil {
log.Error("Error picking frame: " + err.Error())
return nil, err
diff --git a/src/github.com/turt2live/matrix-media-repo/controllers/upload_controller/upload_controller.go b/src/github.com/turt2live/matrix-media-repo/controllers/upload_controller/upload_controller.go
index 4be3f058..93c4e38f 100644
--- a/src/github.com/turt2live/matrix-media-repo/controllers/upload_controller/upload_controller.go
+++ b/src/github.com/turt2live/matrix-media-repo/controllers/upload_controller/upload_controller.go
@@ -152,7 +152,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
db := storage.GetDatabase().GetMediaStore(ctx, log)
records, err := db.GetByHash(info.Sha256Hash)
if err != nil {
- ds.DeleteObject(info) // delete temp object
+ ds.DeleteObject(info.Location) // delete temp object
return nil, err
}
@@ -166,7 +166,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
for _, record := range records {
if record.UserId == userId && record.Origin == origin && record.ContentType == contentType {
log.Info("User has already uploaded this media before - returning unaltered media record")
- ds.DeleteObject(info) // delete temp object
+ ds.DeleteObject(info.Location) // delete temp object
trackUploadAsLastAccess(ctx, log, record)
return record, nil
}
@@ -184,7 +184,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
err = db.Insert(media)
if err != nil {
- ds.DeleteObject(info) // delete temp object
+ ds.DeleteObject(info.Location) // delete temp object
return nil, err
}
@@ -200,7 +200,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
// // We'll assume an error means it doesn't exist
// os.Rename(fileLocation, targetPath)
//} else {
- // ds.DeleteObject(info)
+ // ds.DeleteObject(info.Location)
//}
trackUploadAsLastAccess(ctx, log, media)
@@ -210,7 +210,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
// The media doesn't already exist - save it as new
if info.SizeBytes <= 0 {
- ds.DeleteObject(info)
+ ds.DeleteObject(info.Location)
return nil, errors.New("file has no contents")
}
@@ -231,7 +231,7 @@ func StoreDirect(contents io.ReadCloser, contentType string, filename string, us
err = db.Insert(media)
if err != nil {
- ds.DeleteObject(info) // delete temp object
+ ds.DeleteObject(info.Location) // delete temp object
return nil, err
}
diff --git a/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go b/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
index db82da74..3573fcba 100644
--- a/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
+++ b/src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go
@@ -14,7 +14,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/metrics"
- "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
"github.com/turt2live/matrix-media-repo/util/download_tracker"
@@ -97,11 +97,11 @@ func (c *MediaCache) GetMedia(media *types.Media, log *logrus.Entry) (*cachedFil
}
cacheFn := func() (*cachedFile, error) {
- filePath, err := storage.ResolveMediaLocation(context.TODO(), log, media.DatastoreId, media.Location)
+ mediaStream, err := datastore.DownloadStream(context.TODO(), log, media.DatastoreId, media.Location)
if err != nil {
return nil, err
}
- data, err := ioutil.ReadFile(filePath)
+ data, err := ioutil.ReadAll(mediaStream)
if err != nil {
return nil, err
}
@@ -119,11 +119,11 @@ func (c *MediaCache) GetThumbnail(thumbnail *types.Thumbnail, log *logrus.Entry)
}
cacheFn := func() (*cachedFile, error) {
- filePath, err := storage.ResolveMediaLocation(context.TODO(), log, thumbnail.DatastoreId, thumbnail.Location)
+ mediaStream, err := datastore.DownloadStream(context.TODO(), log, thumbnail.DatastoreId, thumbnail.Location)
if err != nil {
return nil, err
}
- data, err := ioutil.ReadFile(filePath)
+ data, err := ioutil.ReadAll(mediaStream)
if err != nil {
return nil, err
}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go
index 516ce1ad..0fe1cc61 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go
@@ -3,6 +3,7 @@ package datastore
import (
"context"
"fmt"
+ "io"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -11,7 +12,37 @@ import (
"github.com/turt2live/matrix-media-repo/types"
)
-// TODO: Download (get stream) from DS
+func LocateDatastore(ctx context.Context, log *logrus.Entry, datastoreId string) (*DatastoreRef, error) {
+ ds, err := storage.GetDatabase().GetMediaStore(ctx, log).GetDatastore(datastoreId)
+ if err != nil {
+ return nil, err
+ }
+
+ conf, err := GetDatastoreConfig(ds)
+ if err != nil {
+ return nil, err
+ }
+
+ return newDatastoreRef(ds, conf), nil
+}
+
+func DownloadStream(ctx context.Context, log *logrus.Entry, datastoreId string, location string) (io.ReadCloser, error) {
+ ref, err := LocateDatastore(ctx, log, datastoreId)
+ if err != nil {
+ return nil, err
+ }
+ return ref.DownloadFile(location)
+}
+
+func GetDatastoreConfig(ds *types.Datastore) (config.DatastoreConfig, error) {
+ for _, dsConf := range config.Get().DataStores {
+ if dsConf.Type == ds.Type && GetUriForDatastore(dsConf) == ds.Uri {
+ return dsConf, nil
+ }
+ }
+
+ return config.DatastoreConfig{}, errors.New("datastore not found")
+}
func GetUriForDatastore(dsConf config.DatastoreConfig) string {
if dsConf.Type == "file" {
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore_ref.go b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore_ref.go
index 977709e2..72b2aad7 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore_ref.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore_ref.go
@@ -4,6 +4,8 @@ import (
"context"
"errors"
"io"
+ "os"
+ "path"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
@@ -32,10 +34,6 @@ func newDatastoreRef(ds *types.Datastore, config config.DatastoreConfig) *Datast
}
}
-func (d *DatastoreRef) ResolveFilePath(location string) string {
- return d.datastore.ResolveFilePath(location)
-}
-
func (d *DatastoreRef) UploadFile(file io.ReadCloser, ctx context.Context, log *logrus.Entry) (*types.ObjectInfo, error) {
log = log.WithFields(logrus.Fields{"datastoreId": d.DatastoreId, "datastoreUri": d.Uri})
@@ -52,16 +50,30 @@ func (d *DatastoreRef) UploadFile(file io.ReadCloser, ctx context.Context, log *
}
}
-func (d *DatastoreRef) DeleteObject(objectInfo *types.ObjectInfo) error {
+func (d *DatastoreRef) DeleteObject(location string) error {
if d.Type == "file" {
- return ds_file.DeletePersistedFile(d.Uri, objectInfo)
+ return ds_file.DeletePersistedFile(d.Uri, location)
} else if d.Type == "s3" {
s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
if err != nil {
return err
}
- return s3.DeleteObject(objectInfo)
+ return s3.DeleteObject(location)
} else {
return errors.New("unknown datastore type")
}
}
+
+func (d *DatastoreRef) DownloadFile(location string) (io.ReadCloser, error) {
+ if d.Type == "file" {
+ return os.Open(path.Join(d.Uri, location))
+ } else if d.Type == "s3" {
+ s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
+ if err != nil {
+ return nil,err
+ }
+ return s3.DownloadObject(location)
+ } else {
+ return nil, errors.New("unknown datastore type")
+ }
+}
\ No newline at end of file
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_file/file_store.go b/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_file/file_store.go
index d7a3602e..bb7ca4ef 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_file/file_store.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_file/file_store.go
@@ -106,7 +106,6 @@ func PersistFile(basePath string, file io.ReadCloser, ctx context.Context, log *
}, nil
}
-func DeletePersistedFile(basePath string, info *types.ObjectInfo) error {
- filePath := path.Join(basePath, info.Location)
- return os.Remove(filePath)
+func DeletePersistedFile(basePath string, location string) error {
+ return os.Remove(path.Join(basePath, location))
}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3/s3_store.go b/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3/s3_store.go
index 6daca34c..8690de71 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3/s3_store.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3/s3_store.go
@@ -100,7 +100,7 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, ctx context.Context, log *l
}
if hashErr != nil {
- s.DeleteObject(obj)
+ s.DeleteObject(obj.Location)
return nil, hashErr
}
@@ -111,6 +111,12 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, ctx context.Context, log *l
return obj, nil
}
-func (s *s3Datastore) DeleteObject(objectInfo *types.ObjectInfo) error {
- return s.client.RemoveObject(s.bucket, objectInfo.Location)
+func (s *s3Datastore) DeleteObject(location string) error {
+ logrus.Info("Deleting object from bucket ", s.bucket, ": ", location)
+ return s.client.RemoveObject(s.bucket, location)
+}
+
+func (s *s3Datastore) DownloadObject(location string) (io.ReadCloser, error) {
+ logrus.Info("Downloading object from bucket ", s.bucket, ": ", location)
+ return s.client.GetObject(s.bucket, location, minio.GetObjectOptions{})
}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go b/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
index 2833432a..a9635da9 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
@@ -3,7 +3,6 @@ package storage
import (
"context"
"database/sql"
- "errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/storage/stores"
@@ -59,17 +58,3 @@ func getOrCreateDatastoreWithMediaService(mediaService *stores.MediaStore, baseP
return datastore, nil
}
-
-func ResolveMediaLocation(ctx context.Context, log *logrus.Entry, datastoreId string, location string) (string, error) {
- svc := GetDatabase().GetMediaStore(ctx, log)
- ds, err := svc.GetDatastore(datastoreId)
- if err != nil {
- return "", err
- }
-
- if ds.Type != "file" {
- return "", errors.New("unrecognized datastore type: " + ds.Type)
- }
-
- return ds.ResolveFilePath(location), nil
-}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/startup_migrations.go b/src/github.com/turt2live/matrix-media-repo/storage/startup_migrations.go
index 25d67d9b..c6ed4d74 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/startup_migrations.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/startup_migrations.go
@@ -28,7 +28,7 @@ func populateThumbnailHashes(db *Database) error {
logrus.Error("Unrecognized datastore type for thumbnail ", thumb.Origin, " ", thumb.MediaId)
continue
}
- location := datastore.ResolveFilePath(thumb.Location)
+ location := path.Join(datastore.Uri, thumb.Location)
hash, err := util.GetFileHash(location)
if err != nil {
diff --git a/src/github.com/turt2live/matrix-media-repo/types/datastore.go b/src/github.com/turt2live/matrix-media-repo/types/datastore.go
index 323aa2b0..79802398 100644
--- a/src/github.com/turt2live/matrix-media-repo/types/datastore.go
+++ b/src/github.com/turt2live/matrix-media-repo/types/datastore.go
@@ -1,15 +1,7 @@
package types
-import (
- "path"
-)
-
type Datastore struct {
DatastoreId string
Type string
Uri string
}
-
-func (d *Datastore) ResolveFilePath(location string) string {
- return path.Join(d.Uri, location)
-}
diff --git a/src/github.com/turt2live/matrix-media-repo/util/random.go b/src/github.com/turt2live/matrix-media-repo/util/random.go
index e2cb48a7..d2f1ed99 100644
--- a/src/github.com/turt2live/matrix-media-repo/util/random.go
+++ b/src/github.com/turt2live/matrix-media-repo/util/random.go
@@ -1,8 +1,8 @@
package util
import (
- "crypto/md5"
"crypto/rand"
+ "crypto/sha1"
"encoding/hex"
)
@@ -21,7 +21,7 @@ func GenerateRandomString(nBytes int) (string, error) {
return "", err
}
- hasher := md5.New()
+ hasher := sha1.New()
hasher.Write(b)
return hex.EncodeToString(hasher.Sum(nil)), nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/util/util_exif/exif.go b/src/github.com/turt2live/matrix-media-repo/util/util_exif/exif.go
index 49d10398..eef4352c 100644
--- a/src/github.com/turt2live/matrix-media-repo/util/util_exif/exif.go
+++ b/src/github.com/turt2live/matrix-media-repo/util/util_exif/exif.go
@@ -3,12 +3,11 @@ package util_exif
import (
"context"
"fmt"
- "os"
"github.com/pkg/errors"
"github.com/rwcarlsen/goexif/exif"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
)
@@ -23,16 +22,12 @@ func GetExifOrientation(media *types.Media) (*ExifOrientation, error) {
return nil, errors.New("image is not a jpeg")
}
- filePath, err := storage.ResolveMediaLocation(context.TODO(), &logrus.Entry{}, media.DatastoreId, media.Location)
- if err != nil {
- return nil, err
- }
- file, err := os.Open(filePath)
+ mediaStream, err := datastore.DownloadStream(context.TODO(), &logrus.Entry{}, media.DatastoreId, media.Location)
if err != nil {
return nil, err
}
- exifData, err := exif.Decode(file)
+ exifData, err := exif.Decode(mediaStream)
if err != nil {
return nil, err
}
--
GitLab