From 2f6d9cdf03359a60379334388c6e67724904b6f8 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Mon, 8 Jan 2018 21:29:44 -0700
Subject: [PATCH] Use more pointers; Make the db a singleton; Go back to
passing context/logs around
---
.../client/error_handlers.go | 6 +-
.../matrix-media-repo/client/r0/download.go | 32 ++++--
.../matrix-media-repo/client/r0/identicon.go | 13 +--
.../client/r0/preview_url.go | 13 ++-
.../matrix-media-repo/client/r0/thumbnail.go | 44 +++++---
.../matrix-media-repo/client/r0/upload.go | 12 +-
.../cmd/import_synapse/main.go | 23 +---
.../matrix-media-repo/cmd/media_repo/main.go | 31 +-----
.../rcontext/request_info.go | 14 ---
.../services/handlers/opengraph_previewer.go | 45 ++++----
.../handlers/remote_media_downloader.go | 38 ++++---
.../services/handlers/thumbnailer.go | 44 ++++----
.../services/media_service.go | 66 +++++------
.../services/thumbnail_service.go | 45 ++++----
.../matrix-media-repo/services/url_service.go | 105 +++++++++---------
.../matrix-media-repo/storage/file_store.go | 4 +-
.../matrix-media-repo/storage/storage.go | 38 +++++--
.../storage/stores/media_store.go | 10 +-
.../storage/stores/thumbnail_store.go | 6 +-
.../matrix-media-repo/types/media.go | 4 +
.../util/{errcodes => errs}/errorcodes.go | 2 +-
.../util/{ => errs}/errors.go | 2 +-
.../turt2live/matrix-media-repo/util/mxc.go | 7 --
23 files changed, 302 insertions(+), 302 deletions(-)
delete mode 100644 src/github.com/turt2live/matrix-media-repo/rcontext/request_info.go
rename src/github.com/turt2live/matrix-media-repo/util/{errcodes => errs}/errorcodes.go (92%)
rename src/github.com/turt2live/matrix-media-repo/util/{ => errs}/errors.go (95%)
delete mode 100644 src/github.com/turt2live/matrix-media-repo/util/mxc.go
diff --git a/src/github.com/turt2live/matrix-media-repo/client/error_handlers.go b/src/github.com/turt2live/matrix-media-repo/client/error_handlers.go
index dabbe61d..506a02ae 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/error_handlers.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/error_handlers.go
@@ -3,13 +3,13 @@ package client
import (
"net/http"
- "github.com/turt2live/matrix-media-repo/rcontext"
+ "github.com/sirupsen/logrus"
)
-func NotFoundHandler(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func NotFoundHandler(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
return NotFoundError()
}
-func MethodNotAllowedHandler(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func MethodNotAllowedHandler(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
return MethodNotAllowed()
}
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/download.go b/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
index 624b0832..59ca6841 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
@@ -1,25 +1,27 @@
package r0
import (
+ "io"
"net/http"
+ "os"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type DownloadMediaResponse struct {
ContentType string
Filename string
SizeBytes int64
- Location string
+ Data io.ReadCloser
}
-func DownloadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
- if !ValidateUserCanDownload(r, i) {
+func DownloadMedia(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
+ if !ValidateUserCanDownload(r) {
return client.AuthFailed()
}
@@ -29,22 +31,22 @@ func DownloadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInf
mediaId := params["mediaId"]
filename := params["filename"]
- i.Log = i.Log.WithFields(logrus.Fields{
+ log = log.WithFields(logrus.Fields{
"mediaId": mediaId,
"server": server,
"filename": filename,
})
- svc := services.CreateMediaService(i)
+ svc := services.NewMediaService(r.Context(), log)
media, err := svc.GetMedia(server, mediaId)
if err != nil {
- if err == util.ErrMediaNotFound {
+ if err == errs.ErrMediaNotFound {
return client.NotFoundError()
- } else if err == util.ErrMediaTooLarge {
+ } else if err == errs.ErrMediaTooLarge {
return client.RequestTooLarge()
}
- i.Log.Error("Unexpected error locating media: " + err.Error())
+ log.Error("Unexpected error locating media: " + err.Error())
return client.InternalServerError("Unexpected Error")
}
@@ -52,21 +54,27 @@ func DownloadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInf
filename = media.UploadName
}
+ fstream, err := os.Open(media.Location)
+ if err != nil {
+ log.Error("Unexpected error opening media: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
+ }
+
return &DownloadMediaResponse{
ContentType: media.ContentType,
Filename: filename,
SizeBytes: media.SizeBytes,
- Location: media.Location,
+ Data: fstream,
}
}
-func ValidateUserCanDownload(r *http.Request, i rcontext.RequestInfo) (bool) {
+func ValidateUserCanDownload(r *http.Request) (bool) {
hs := util.GetHomeserverConfig(r.Host)
if !hs.DownloadRequiresAuth {
return true // no auth required == can access
}
accessToken := util.GetAccessTokenFromRequest(r)
- userId, err := util.GetUserIdFromToken(i.Context, r.Host, accessToken)
+ userId, err := util.GetUserIdFromToken(r.Context(), r.Host, accessToken)
return userId != "" && err != nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/identicon.go b/src/github.com/turt2live/matrix-media-repo/client/r0/identicon.go
index fa511a06..ca030225 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/identicon.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/identicon.go
@@ -14,18 +14,17 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
)
type IdenticonResponse struct {
Avatar io.Reader
}
-func Identicon(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func Identicon(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
if !config.Get().Identicons.Enabled {
return client.NotFoundError()
}
- if !ValidateUserCanDownload(r, i) {
+ if !ValidateUserCanDownload(r) {
return client.AuthFailed()
}
@@ -52,7 +51,7 @@ func Identicon(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) i
}
}
- i.Log = i.Log.WithFields(logrus.Fields{
+ log = log.WithFields(logrus.Fields{
"identiconWidth": width,
"identiconHeight": height,
"identiconSeed": seed,
@@ -76,18 +75,18 @@ func Identicon(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) i
},
}
- i.Log.Info("Generating identicon")
+ log.Info("Generating identicon")
img := sig.Make(width, false, []byte(hashed))
if width != height {
// Resize to the desired height
- i.Log.Info("Resizing image to fit height")
+ log.Info("Resizing image to fit height")
img = imaging.Resize(img, width, height, imaging.Lanczos)
}
imgData := &bytes.Buffer{}
err = imaging.Encode(imgData, img, imaging.PNG)
if err != nil {
- i.Log.Error("Error generating image:" + err.Error())
+ log.Error("Error generating image:" + err.Error())
return client.InternalServerError("error generating identicon")
}
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/preview_url.go b/src/github.com/turt2live/matrix-media-repo/client/r0/preview_url.go
index 69d99442..3283f26e 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/preview_url.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/preview_url.go
@@ -5,11 +5,12 @@ import (
"strconv"
"strings"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type MatrixOpenGraph struct {
@@ -25,7 +26,7 @@ type MatrixOpenGraph struct {
ImageHeight int `json:"og:image:height,omitempty"`
}
-func PreviewUrl(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func PreviewUrl(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
if !config.Get().UrlPreviews.Enabled {
return client.NotFoundError()
}
@@ -45,7 +46,7 @@ func PreviewUrl(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo)
if tsStr != "" {
ts, err = strconv.ParseInt(tsStr, 10, 64)
if err != nil {
- i.Log.Error("Error parsing ts: " + err.Error())
+ log.Error("Error parsing ts: " + err.Error())
return client.BadRequest(err.Error())
}
}
@@ -58,12 +59,12 @@ func PreviewUrl(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo)
return client.BadRequest("Scheme not accepted")
}
- svc := services.CreateUrlService(i)
+ svc := services.NewUrlService(r.Context(), log)
preview, err := svc.GetPreview(urlStr, r.Host, userId, ts)
if err != nil {
- if err == util.ErrMediaNotFound || err == util.ErrHostNotFound {
+ if err == errs.ErrMediaNotFound || err == errs.ErrHostNotFound {
return client.NotFoundError()
- } else if err == util.ErrInvalidHost || err == util.ErrHostBlacklisted {
+ } else if err == errs.ErrInvalidHost || err == errs.ErrHostBlacklisted {
return client.BadRequest(err.Error())
} else {
return client.InternalServerError("unexpected error during request")
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go b/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
index 12482f73..d7e8cce1 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
@@ -2,19 +2,19 @@ package r0
import (
"net/http"
+ "os"
"strconv"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
-func ThumbnailMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
- if !ValidateUserCanDownload(r, i) {
+func ThumbnailMedia(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
+ if !ValidateUserCanDownload(r) {
return client.AuthFailed()
}
@@ -23,7 +23,7 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestIn
server := params["server"]
mediaId := params["mediaId"]
- i.Log = i.Log.WithFields(logrus.Fields{
+ log = log.WithFields(logrus.Fields{
"mediaId": mediaId,
"server": server,
})
@@ -53,45 +53,57 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestIn
method = "crop"
}
- i.Log = i.Log.WithFields(logrus.Fields{
+ log = log.WithFields(logrus.Fields{
"requestedWidth": width,
"requestedHeight": height,
"requestedMethod": method,
})
- mediaSvc := services.CreateMediaService(i)
- thumbSvc := services.CreateThumbnailService(i)
+ mediaSvc := services.NewMediaService(r.Context(), log)
+ thumbSvc := services.NewThumbnailService(r.Context(), log)
media, err := mediaSvc.GetMedia(server, mediaId)
if err != nil {
- if err == util.ErrMediaNotFound {
+ if err == errs.ErrMediaNotFound {
return client.NotFoundError()
- } else if err == util.ErrMediaTooLarge {
+ } else if err == errs.ErrMediaTooLarge {
return client.RequestTooLarge()
}
- i.Log.Error("Unexpected error locating media: " + err.Error())
+ log.Error("Unexpected error locating media: " + err.Error())
return client.InternalServerError("Unexpected Error")
}
thumb, err := thumbSvc.GetThumbnail(media, width, height, method)
if err != nil {
- if err == util.ErrMediaTooLarge {
- i.Log.Warn("Media too large to thumbnail, returning source image instead")
+ fstream, err := os.Open(media.Location)
+ if err != nil {
+ log.Error("Unexpected error opening media: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
+ }
+
+ if err == errs.ErrMediaTooLarge {
+ log.Warn("Media too large to thumbnail, returning source image instead")
return &DownloadMediaResponse{
ContentType: media.ContentType,
SizeBytes: media.SizeBytes,
- Location: media.Location,
+ Data: fstream,
Filename: "thumbnail",
}
}
- i.Log.Error("Unexpected error getting thumbnail: " + err.Error())
+ log.Error("Unexpected error getting thumbnail: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
+ }
+
+ fstream, err := os.Open(thumb.Location)
+ if err != nil {
+ log.Error("Unexpected error opening thumbnail media: " + err.Error())
return client.InternalServerError("Unexpected Error")
}
return &DownloadMediaResponse{
ContentType: thumb.ContentType,
SizeBytes: thumb.SizeBytes,
- Location: thumb.Location,
+ Data: fstream,
Filename: "thumbnail",
}
}
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go b/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
index 80e29358..da2900c9 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
@@ -7,7 +7,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services"
"github.com/turt2live/matrix-media-repo/util"
)
@@ -16,7 +15,7 @@ type MediaUploadedResponse struct {
ContentUri string `json:"content_uri"`
}
-func UploadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func UploadMedia(w http.ResponseWriter, r *http.Request, log *logrus.Entry) interface{} {
accessToken := util.GetAccessTokenFromRequest(r)
userId, err := util.GetUserIdFromToken(r.Context(), r.Host, accessToken)
if err != nil || userId == "" {
@@ -28,7 +27,7 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo)
filename = "upload.bin"
}
- i.Log = i.Log.WithFields(logrus.Fields{
+ log = log.WithFields(logrus.Fields{
"filename": filename,
"userId": userId,
})
@@ -38,7 +37,7 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo)
contentType = "application/octet-stream" // binary
}
- svc := services.CreateMediaService(i)
+ svc := services.NewMediaService(r.Context(), log)
if svc.IsTooLarge(r.ContentLength, r.Header.Get("Content-Length")) {
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
@@ -51,10 +50,9 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo)
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
defer r.Body.Close()
- i.Log.Error("Unexpected error storing media: " + err.Error())
+ log.Error("Unexpected error storing media: " + err.Error())
return client.InternalServerError("Unexpected Error")
}
- mxc := util.MediaToMxc(&media)
- return &MediaUploadedResponse{mxc}
+ return &MediaUploadedResponse{media.MxcUri()}
}
diff --git a/src/github.com/turt2live/matrix-media-repo/cmd/import_synapse/main.go b/src/github.com/turt2live/matrix-media-repo/cmd/import_synapse/main.go
index 5dc6f5a4..8c3f8874 100644
--- a/src/github.com/turt2live/matrix-media-repo/cmd/import_synapse/main.go
+++ b/src/github.com/turt2live/matrix-media-repo/cmd/import_synapse/main.go
@@ -13,9 +13,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/logging"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services"
- "github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/synapse"
)
@@ -48,11 +46,6 @@ func main() {
logrus.Info("Setting up for importing...")
- db, err := storage.OpenDatabase(config.Get().Database.Postgres)
- if err != nil {
- panic(err)
- }
-
connectionString := "postgres://" + *postgresUsername + ":" + realPsqlPassword + "@" + *postgresHost + ":" + strconv.Itoa(*postgresPort) + "/" + *postgresDatabase + "?sslmode=disable"
csApiUrl := *baseUrl
if csApiUrl[len(csApiUrl)-1:] == "/" {
@@ -77,27 +70,19 @@ func main() {
percent := int((float32(i+1) / float32(len(records))) * 100)
record := records[i]
- info := rcontext.RequestInfo{
- Log: logrus.WithFields(logrus.Fields{
- "mediaId": record.MediaId,
- }),
- Context: ctx,
- Db: *db,
- }
-
- info.Log.Info(fmt.Sprintf("Downloading %s (%d/%d %d%%)", record.MediaId, i+1, len(records), percent))
+ logrus.Info(fmt.Sprintf("Downloading %s (%d/%d %d%%)", record.MediaId, i+1, len(records), percent))
body, err := downloadMedia(csApiUrl, *serverName, record.MediaId)
if err != nil {
- info.Log.Error(err.Error())
+ logrus.Error(err.Error())
continue
}
- svc := services.CreateMediaService(info)
+ svc := services.NewMediaService(ctx, logrus.WithFields(logrus.Fields{}))
_, err = svc.StoreMedia(body, record.ContentType, record.UploadName, record.UserId, *serverName, record.MediaId)
if err != nil {
- info.Log.Error(err.Error())
+ logrus.Error(err.Error())
continue
}
diff --git a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
index 884c4140..a7b7c22e 100644
--- a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
+++ b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"net/http"
- "os"
"reflect"
"strconv"
"strings"
@@ -18,8 +17,6 @@ import (
"github.com/turt2live/matrix-media-repo/client/r0"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/logging"
- "github.com/turt2live/matrix-media-repo/rcontext"
- "github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/util"
)
@@ -30,12 +27,11 @@ type requestCounter struct {
}
type Handler struct {
- h func(http.ResponseWriter, *http.Request, rcontext.RequestInfo) interface{}
+ h func(http.ResponseWriter, *http.Request, *log.Entry) interface{}
opts HandlerOpts
}
type HandlerOpts struct {
- db storage.Database
reqCounter *requestCounter
}
@@ -56,13 +52,8 @@ func main() {
log.Info("Starting media repository...")
- db, err := storage.OpenDatabase(config.Get().Database.Postgres)
- if err != nil {
- panic(err)
- }
-
counter := requestCounter{}
- hOpts := HandlerOpts{*db, &counter}
+ hOpts := HandlerOpts{&counter}
optionsHandler := Handler{optionsRequest, hOpts}
uploadHandler := Handler{r0.UploadMedia, hOpts}
@@ -146,11 +137,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var res interface{} = client.AuthFailed()
if util.IsServerOurs(r.Host) {
contextLog.Info("Server is owned by us, processing request")
- res = h.h(w, r, rcontext.RequestInfo{
- Log: contextLog,
- Context: r.Context(),
- Db: h.opts.db,
- })
+ res = h.h(w, r, contextLog)
if res == nil {
res = &EmptyResponse{}
}
@@ -194,14 +181,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("Content-Disposition", "inline; filename=\""+result.Filename+"\"")
w.Header().Set("Content-Length", fmt.Sprint(result.SizeBytes))
- f, err := os.Open(result.Location)
- if err != nil {
- w.Header().Set("Content-Type", "application/json")
- http.Error(w, UnkErrJson, http.StatusInternalServerError)
- break
- }
- defer f.Close()
- io.Copy(w, f)
+ defer result.Data.Close()
+ io.Copy(w, result.Data)
break
case *r0.IdenticonResponse:
w.Header().Set("Content-Type", "image/png")
@@ -221,6 +202,6 @@ func (c *requestCounter) GetNextId() string {
return "REQ-" + strId
}
-func optionsRequest(w http.ResponseWriter, r *http.Request, i rcontext.RequestInfo) interface{} {
+func optionsRequest(w http.ResponseWriter, r *http.Request, log *log.Entry) interface{} {
return &EmptyResponse{}
}
diff --git a/src/github.com/turt2live/matrix-media-repo/rcontext/request_info.go b/src/github.com/turt2live/matrix-media-repo/rcontext/request_info.go
deleted file mode 100644
index 1f95b7f7..00000000
--- a/src/github.com/turt2live/matrix-media-repo/rcontext/request_info.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package rcontext
-
-import (
- "context"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/storage"
-)
-
-type RequestInfo struct {
- Context context.Context
- Log *logrus.Entry
- Db storage.Database
-}
diff --git a/src/github.com/turt2live/matrix-media-repo/services/handlers/opengraph_previewer.go b/src/github.com/turt2live/matrix-media-repo/services/handlers/opengraph_previewer.go
index b889b532..f3b6daf5 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/handlers/opengraph_previewer.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/handlers/opengraph_previewer.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "context"
"errors"
"io"
"io/ioutil"
@@ -14,8 +15,7 @@ import (
"github.com/dyatlov/go-opengraph/opengraph"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type OpenGraphResult struct {
@@ -24,8 +24,7 @@ type OpenGraphResult struct {
Type string
Description string
Title string
- Image OpenGraphImage
- HasImage bool
+ Image *OpenGraphImage
}
type OpenGraphImage struct {
@@ -37,22 +36,27 @@ type OpenGraphImage struct {
}
type OpenGraphUrlPreviewer struct {
- Info rcontext.RequestInfo
+ ctx context.Context
+ log *logrus.Entry
+}
+
+func NewOpenGraphPreviewer(ctx context.Context, log *logrus.Entry) *OpenGraphUrlPreviewer {
+ return &OpenGraphUrlPreviewer{ctx, log}
}
func (p *OpenGraphUrlPreviewer) GeneratePreview(urlStr string) (OpenGraphResult, error) {
- html, err := downloadContent(urlStr, p.Info.Log)
+ html, err := downloadContent(urlStr, p.log)
if err != nil {
- p.Info.Log.Error("Error downloading content: " + err.Error())
+ p.log.Error("Error downloading content: " + err.Error())
// We'll consider it not found for the sake of processing
- return OpenGraphResult{}, util.ErrMediaNotFound
+ return OpenGraphResult{}, errs.ErrMediaNotFound
}
og := opengraph.NewOpenGraph()
err = og.ProcessHTML(strings.NewReader(html))
if err != nil {
- p.Info.Log.Error("Error getting OpenGraph: " + err.Error())
+ p.log.Error("Error getting OpenGraph: " + err.Error())
return OpenGraphResult{}, err
}
@@ -77,25 +81,24 @@ func (p *OpenGraphUrlPreviewer) GeneratePreview(urlStr string) (OpenGraphResult,
if og.Images != nil && len(og.Images) > 0 {
baseUrl, err := url.Parse(urlStr)
if err != nil {
- p.Info.Log.Error("Non-fatal error getting thumbnail (parsing base url): " + err.Error())
+ p.log.Error("Non-fatal error getting thumbnail (parsing base url): " + err.Error())
return *graph, nil
}
imgUrl, err := url.Parse(og.Images[0].URL)
if err != nil {
- p.Info.Log.Error("Non-fatal error getting thumbnail (parsing image url): " + err.Error())
+ p.log.Error("Non-fatal error getting thumbnail (parsing image url): " + err.Error())
return *graph, nil
}
imgAbsUrl := baseUrl.ResolveReference(imgUrl)
- img, err := downloadImage(imgAbsUrl.String(), p.Info)
+ img, err := downloadImage(imgAbsUrl.String(), p.log)
if err != nil {
- p.Info.Log.Error("Non-fatal error getting thumbnail (downloading image): " + err.Error())
+ p.log.Error("Non-fatal error getting thumbnail (downloading image): " + err.Error())
return *graph, nil
}
graph.Image = img
- graph.HasImage = true
}
return *graph, nil
@@ -113,7 +116,7 @@ func downloadContent(urlStr string, log *logrus.Entry) (string, error) {
}
if config.Get().UrlPreviews.MaxPageSizeBytes > 0 && resp.ContentLength >= 0 && resp.ContentLength > config.Get().UrlPreviews.MaxPageSizeBytes {
- return "", util.ErrMediaTooLarge
+ return "", errs.ErrMediaTooLarge
}
var reader io.Reader
@@ -133,15 +136,15 @@ func downloadContent(urlStr string, log *logrus.Entry) (string, error) {
return html, nil
}
-func downloadImage(imageUrl string, i rcontext.RequestInfo) (OpenGraphImage, error) {
- i.Log.Info("Getting image from " + imageUrl)
+func downloadImage(imageUrl string, log *logrus.Entry) (*OpenGraphImage, error) {
+ log.Info("Getting image from " + imageUrl)
resp, err := http.Get(imageUrl)
if err != nil {
- return OpenGraphImage{}, err
+ return nil, err
}
if resp.StatusCode != http.StatusOK {
- i.Log.Warn("Received status code " + strconv.Itoa(resp.StatusCode))
- return OpenGraphImage{}, errors.New("error during transfer")
+ log.Warn("Received status code " + strconv.Itoa(resp.StatusCode))
+ return nil, errors.New("error during transfer")
}
image := &OpenGraphImage{
@@ -156,7 +159,7 @@ func downloadImage(imageUrl string, i rcontext.RequestInfo) (OpenGraphImage, err
image.Filename = params["filename"]
}
- return *image, nil
+ return image, nil
}
func calcTitle(html string) string {
diff --git a/src/github.com/turt2live/matrix-media-repo/services/handlers/remote_media_downloader.go b/src/github.com/turt2live/matrix-media-repo/services/handlers/remote_media_downloader.go
index bbb80c12..e1865940 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/handlers/remote_media_downloader.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/handlers/remote_media_downloader.go
@@ -1,16 +1,16 @@
package handlers
import (
+ "context"
"errors"
"io"
"mime"
"strconv"
"github.com/matrix-org/gomatrixserverlib"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
- "github.com/turt2live/matrix-media-repo/storage/stores"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type DownloadedMedia struct {
@@ -20,33 +20,37 @@ type DownloadedMedia struct {
}
type RemoteMediaDownloader struct {
- MediaStore stores.MediaStore
- Info rcontext.RequestInfo
+ ctx context.Context
+ log *logrus.Entry
}
-func (r *RemoteMediaDownloader) Download(server string, mediaId string) (DownloadedMedia, error) {
+func NewRemoteMediaDownloader(ctx context.Context, log *logrus.Entry) *RemoteMediaDownloader {
+ return &RemoteMediaDownloader{ctx, log}
+}
+
+func (r *RemoteMediaDownloader) Download(server string, mediaId string) (*DownloadedMedia, error) {
mtxClient := gomatrixserverlib.NewClient()
mtxServer := gomatrixserverlib.ServerName(server)
- resp, err := mtxClient.CreateMediaDownloadRequest(r.Info.Context, mtxServer, mediaId)
+ resp, err := mtxClient.CreateMediaDownloadRequest(r.ctx, mtxServer, mediaId)
if err != nil {
- return DownloadedMedia{}, err
+ return nil, err
}
if resp.StatusCode == 404 {
- r.Info.Log.Info("Remote media not found")
- return DownloadedMedia{}, util.ErrMediaNotFound
+ r.log.Info("Remote media not found")
+ return nil, errs.ErrMediaNotFound
} else if resp.StatusCode != 200 {
- r.Info.Log.Info("Unknown error fetching remote media; received status code " + strconv.Itoa(resp.StatusCode))
- return DownloadedMedia{}, errors.New("could not fetch remote media")
+ r.log.Info("Unknown error fetching remote media; received status code " + strconv.Itoa(resp.StatusCode))
+ return nil, errors.New("could not fetch remote media")
}
contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
if err != nil {
- return DownloadedMedia{}, err
+ return nil, err
}
if config.Get().Downloads.MaxSizeBytes > 0 && contentLength > config.Get().Downloads.MaxSizeBytes {
- r.Info.Log.Warn("Attempted to download media that was too large")
- return DownloadedMedia{}, util.ErrMediaTooLarge
+ r.log.Warn("Attempted to download media that was too large")
+ return nil, errs.ErrMediaTooLarge
}
request := &DownloadedMedia{
@@ -60,6 +64,6 @@ func (r *RemoteMediaDownloader) Download(server string, mediaId string) (Downloa
request.DesiredFilename = params["filename"]
}
- r.Info.Log.Info("Persisting downloaded media")
- return *request, nil
+ r.log.Info("Persisting downloaded media")
+ return request, nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/services/handlers/thumbnailer.go b/src/github.com/turt2live/matrix-media-repo/services/handlers/thumbnailer.go
index 52585230..1136bbe3 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/handlers/thumbnailer.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/handlers/thumbnailer.go
@@ -2,12 +2,12 @@ package handlers
import (
"bytes"
+ "context"
"errors"
"github.com/disintegration/imaging"
- "github.com/turt2live/matrix-media-repo/rcontext"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/storage"
- "github.com/turt2live/matrix-media-repo/storage/stores"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
)
@@ -19,16 +19,18 @@ type GeneratedThumbnail struct {
}
type Thumbnailer struct {
- ThumbnailStore stores.ThumbnailStore
- Info rcontext.RequestInfo
+ ctx context.Context
+ log *logrus.Entry
}
-func (t *Thumbnailer) GenerateThumbnail(media types.Media, width int, height int, method string) (GeneratedThumbnail, error) {
- thumb := &GeneratedThumbnail{}
+func NewThumbnailer(ctx context.Context, log *logrus.Entry) *Thumbnailer {
+ return &Thumbnailer{ctx, log}
+}
+func (t *Thumbnailer) GenerateThumbnail(media *types.Media, width int, height int, method string) (*GeneratedThumbnail, error) {
src, err := imaging.Open(media.Location)
if err != nil {
- return *thumb, err
+ return nil, err
}
srcWidth := src.Bounds().Max.X
@@ -39,16 +41,18 @@ func (t *Thumbnailer) GenerateThumbnail(media types.Media, width int, height int
if aspectRatio == targetAspectRatio {
// Highly unlikely, but if the aspect ratios match then just resize
method = "scale"
- t.Info.Log.Info("Aspect ratio is the same, converting method to 'scale'")
+ t.log.Info("Aspect ratio is the same, converting method to 'scale'")
}
+ thumb := &GeneratedThumbnail{}
+
if srcWidth <= width && srcHeight <= height {
// Image is too small - don't upscale
thumb.ContentType = media.ContentType
thumb.DiskLocation = media.Location
thumb.SizeBytes = media.SizeBytes
- t.Info.Log.Warn("Image too small, returning raw image")
- return *thumb, nil
+ t.log.Warn("Image too small, returning raw image")
+ return thumb, nil
}
if method == "scale" {
@@ -56,34 +60,34 @@ func (t *Thumbnailer) GenerateThumbnail(media types.Media, width int, height int
} else if method == "crop" {
src = imaging.Fill(src, width, height, imaging.Center, imaging.Lanczos)
} else {
- t.Info.Log.Error("Unrecognized thumbnail method: " + method)
- return *thumb, errors.New("unrecognized method: " + method)
+ t.log.Error("Unrecognized thumbnail method: " + method)
+ return nil, errors.New("unrecognized method: " + method)
}
// Put the image bytes into a memory buffer
imgData := &bytes.Buffer{}
err = imaging.Encode(imgData, src, imaging.PNG)
if err != nil {
- t.Info.Log.Error("Unexpected error encoding thumbnail: " + err.Error())
- return *thumb, err
+ t.log.Error("Unexpected error encoding thumbnail: " + err.Error())
+ return nil, err
}
// Reset the buffer pointer and store the file
- location, err := storage.PersistFile(imgData, t.Info.Context, &t.Info.Db)
+ location, err := storage.PersistFile(imgData, t.ctx)
if err != nil {
- t.Info.Log.Error("Unexpected error saving thumbnail: " + err.Error())
- return *thumb, err
+ t.log.Error("Unexpected error saving thumbnail: " + err.Error())
+ return nil, err
}
fileSize, err := util.FileSize(location)
if err != nil {
- t.Info.Log.Error("Unexpected error getting the size of the thumbnail: " + err.Error())
- return *thumb, err
+ t.log.Error("Unexpected error getting the size of the thumbnail: " + err.Error())
+ return nil, err
}
thumb.DiskLocation = location
thumb.ContentType = "image/png"
thumb.SizeBytes = fileSize
- return *thumb, nil
+ return thumb, nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/services/media_service.go b/src/github.com/turt2live/matrix-media-repo/services/media_service.go
index 7068314a..bd250a5f 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/media_service.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/media_service.go
@@ -1,6 +1,7 @@
package services
import (
+ "context"
"database/sql"
"io"
"os"
@@ -8,31 +9,33 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services/handlers"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/stores"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type MediaService struct {
store *stores.MediaStore
- i rcontext.RequestInfo
+ ctx context.Context
+ log *logrus.Entry
}
-func CreateMediaService(i rcontext.RequestInfo) (*MediaService) {
- return &MediaService{i.Db.GetMediaStore(i.Context, i.Log), i}
+func NewMediaService(ctx context.Context, log *logrus.Entry) (*MediaService) {
+ store := storage.GetDatabase().GetMediaStore(ctx, log)
+ return &MediaService{store, ctx, log}
}
-func (s *MediaService) GetMedia(server string, mediaId string) (types.Media, error) {
- s.i.Log.Info("Looking up media")
+func (s *MediaService) GetMedia(server string, mediaId string) (*types.Media, error) {
+ s.log.Info("Looking up media")
media, err := s.store.Get(server, mediaId)
if err != nil {
if err == sql.ErrNoRows {
if util.IsServerOurs(server) {
- s.i.Log.Warn("Media not found")
- return media, util.ErrMediaNotFound
+ s.log.Warn("Media not found")
+ return media, errs.ErrMediaNotFound
}
}
@@ -42,10 +45,10 @@ func (s *MediaService) GetMedia(server string, mediaId string) (types.Media, err
exists, err := util.FileExists(media.Location)
if !exists || err != nil {
if util.IsServerOurs(server) {
- s.i.Log.Error("Media not found in file store when we expected it to")
- return media, util.ErrMediaNotFound
+ s.log.Error("Media not found in file store when we expected it to")
+ return media, errs.ErrMediaNotFound
} else {
- s.i.Log.Warn("Media appears to have been deleted - redownloading")
+ s.log.Warn("Media appears to have been deleted - redownloading")
return s.downloadRemoteMedia(server, mediaId)
}
}
@@ -53,16 +56,13 @@ func (s *MediaService) GetMedia(server string, mediaId string) (types.Media, err
return media, nil
}
-func (s *MediaService) downloadRemoteMedia(server string, mediaId string) (types.Media, error) {
- s.i.Log.Info("Attempting to download remote media")
- downloader := &handlers.RemoteMediaDownloader{
- Info: s.i,
- MediaStore: *s.store,
- }
+func (s *MediaService) downloadRemoteMedia(server string, mediaId string) (*types.Media, error) {
+ s.log.Info("Attempting to download remote media")
+ downloader := handlers.NewRemoteMediaDownloader(s.ctx, s.log)
downloaded, err := downloader.Download(server, mediaId)
if err != nil {
- return types.Media{}, err
+ return nil, err
}
defer downloaded.Contents.Close()
@@ -79,7 +79,7 @@ func (s *MediaService) IsTooLarge(contentLength int64, contentLengthHeader strin
if contentLengthHeader != "" {
parsed, err := strconv.ParseInt(contentLengthHeader, 10, 64)
if err != nil {
- s.i.Log.Warn("Invalid content length header given; assuming too large. Value received: " + contentLengthHeader)
+ s.log.Warn("Invalid content length header given; assuming too large. Value received: " + contentLengthHeader)
return true // Invalid header
}
@@ -89,7 +89,7 @@ func (s *MediaService) IsTooLarge(contentLength int64, contentLengthHeader strin
return false // We can only assume
}
-func (s *MediaService) UploadMedia(contents io.ReadCloser, contentType string, filename string, userId string, host string) (types.Media, error) {
+func (s *MediaService) UploadMedia(contents io.ReadCloser, contentType string, filename string, userId string, host string) (*types.Media, error) {
defer contents.Close()
var data io.Reader
if config.Get().Uploads.MaxSizeBytes > 0 {
@@ -101,41 +101,41 @@ func (s *MediaService) UploadMedia(contents io.ReadCloser, contentType string, f
return s.StoreMedia(data, contentType, filename, userId, host, "")
}
-func (s *MediaService) StoreMedia(contents io.Reader, contentType string, filename string, userId string, host string, mediaId string) (types.Media, error) {
+func (s *MediaService) StoreMedia(contents io.Reader, contentType string, filename string, userId string, host string, mediaId string) (*types.Media, error) {
isGeneratedId := false
if mediaId == "" {
mediaId = generateMediaId()
isGeneratedId = true
}
- log := s.i.Log.WithFields(logrus.Fields{
+ log := s.log.WithFields(logrus.Fields{
"mediaService_mediaId": mediaId,
"mediaService_host": host,
"mediaService_mediaIdIsGenerated": isGeneratedId,
})
// Store the file in a temporary location
- fileLocation, err := storage.PersistFile(contents, s.i.Context, &s.i.Db)
+ fileLocation, err := storage.PersistFile(contents, s.ctx)
if err != nil {
- return types.Media{}, err
+ return nil, err
}
hash, err := storage.GetFileHash(fileLocation)
if err != nil {
defer os.Remove(fileLocation) // attempt cleanup
- return types.Media{}, err
+ return nil, err
}
records, err := s.store.GetByHash(hash)
if err != nil {
defer os.Remove(fileLocation) // attempt cleanup
- return types.Media{}, err
+ return nil, err
}
// If there's at least one record, then we have a duplicate hash - try and process it
if len(records) > 0 {
// See if we one of the duplicate records is a match for the host and media ID. We'll otherwise use
// the last duplicate (should only be 1 anyways) as our starting point for a new record.
- var media types.Media
+ var media *types.Media
for i := 0; i < len(records); i++ {
media = records[i]
@@ -165,9 +165,9 @@ func (s *MediaService) StoreMedia(contents io.Reader, contentType string, filena
media.ContentType = contentType
media.CreationTs = util.NowMillis()
- err = s.store.Insert(&media)
+ err = s.store.Insert(media)
if err != nil {
- return types.Media{}, err
+ return nil, err
}
overwriteExistingOrDeleteTempFile(fileLocation, media)
@@ -179,7 +179,7 @@ func (s *MediaService) StoreMedia(contents io.Reader, contentType string, filena
fileSize, err := util.FileSize(fileLocation)
if err != nil {
defer os.Remove(fileLocation) // attempt cleanup
- return types.Media{}, err
+ return nil, err
}
log.Info("Persisting unique media record")
@@ -199,10 +199,10 @@ func (s *MediaService) StoreMedia(contents io.Reader, contentType string, filena
err = s.store.Insert(media)
if err != nil {
defer os.Remove(fileLocation) // attempt cleanup
- return types.Media{}, err
+ return nil, err
}
- return *media, nil
+ return media, nil
}
func generateMediaId() string {
@@ -214,7 +214,7 @@ func generateMediaId() string {
return str
}
-func overwriteExistingOrDeleteTempFile(tempFileLocation string, media types.Media) {
+func overwriteExistingOrDeleteTempFile(tempFileLocation string, media *types.Media) {
// If the media's file exists, we'll delete the temp file
// If the media's file doesn't exist, we'll move the temp file to where the media expects it to be
exists, err := util.FileExists(media.Location)
diff --git a/src/github.com/turt2live/matrix-media-repo/services/thumbnail_service.go b/src/github.com/turt2live/matrix-media-repo/services/thumbnail_service.go
index a9cb589e..779fa781 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/thumbnail_service.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/thumbnail_service.go
@@ -1,36 +1,40 @@
package services
import (
+ "context"
"database/sql"
"errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services/handlers"
+ "github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/stores"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type ThumbnailService struct {
store *stores.ThumbnailStore
- i rcontext.RequestInfo
+ ctx context.Context
+ log *logrus.Entry
}
-func CreateThumbnailService(i rcontext.RequestInfo) (*ThumbnailService) {
- return &ThumbnailService{i.Db.GetThumbnailStore(i.Context, i.Log), i}
+func NewThumbnailService(ctx context.Context, log *logrus.Entry) (*ThumbnailService) {
+ store := storage.GetDatabase().GetThumbnailStore(ctx, log)
+ return &ThumbnailService{store, ctx, log}
}
-func (s *ThumbnailService) GetThumbnail(media types.Media, width int, height int, method string) (types.Thumbnail, error) {
+func (s *ThumbnailService) GetThumbnail(media *types.Media, width int, height int, method string) (*types.Thumbnail, error) {
if width <= 0 {
- return types.Thumbnail{}, errors.New("width must be positive")
+ return nil, errors.New("width must be positive")
}
if height <= 0 {
- return types.Thumbnail{}, errors.New("height must be positive")
+ return nil, errors.New("height must be positive")
}
if method != "crop" && method != "scale" {
- return types.Thumbnail{}, errors.New("method must be crop or scale")
+ return nil, errors.New("method must be crop or scale")
}
targetWidth := width
@@ -66,32 +70,29 @@ func (s *ThumbnailService) GetThumbnail(media types.Media, width int, height int
}
}
- s.i.Log = s.i.Log.WithFields(logrus.Fields{
+ s.log = s.log.WithFields(logrus.Fields{
"targetWidth": targetWidth,
"targetHeight": targetHeight,
})
- s.i.Log.Info("Looking up thumbnail")
+ s.log.Info("Looking up thumbnail")
thumb, err := s.store.Get(media.Origin, media.MediaId, targetWidth, targetHeight, method)
if err != nil && err != sql.ErrNoRows {
- s.i.Log.Error("Unexpected error processing thumbnail lookup: " + err.Error())
+ s.log.Error("Unexpected error processing thumbnail lookup: " + err.Error())
return thumb, err
}
if err != sql.ErrNoRows {
- s.i.Log.Info("Found existing thumbnail")
+ s.log.Info("Found existing thumbnail")
return thumb, nil
}
if media.SizeBytes > config.Get().Thumbnails.MaxSourceBytes {
- s.i.Log.Warn("Media too large to thumbnail")
- return thumb, util.ErrMediaTooLarge
+ s.log.Warn("Media too large to thumbnail")
+ return thumb, errs.ErrMediaTooLarge
}
- s.i.Log.Info("Generating new thumbnail")
- thumbnailer := &handlers.Thumbnailer{
- Info: s.i,
- ThumbnailStore: *s.store,
- }
+ s.log.Info("Generating new thumbnail")
+ thumbnailer := handlers.NewThumbnailer(s.ctx, s.log)
generated, err := thumbnailer.GenerateThumbnail(media, targetWidth, targetHeight, method)
if err != nil {
@@ -112,9 +113,9 @@ func (s *ThumbnailService) GetThumbnail(media types.Media, width int, height int
err = s.store.Insert(newThumb)
if err != nil {
- s.i.Log.Error("Unexpected error caching thumbnail: " + err.Error())
- return *newThumb, err
+ s.log.Error("Unexpected error caching thumbnail: " + err.Error())
+ return newThumb, err
}
- return *newThumb, nil
+ return newThumb, nil
}
diff --git a/src/github.com/turt2live/matrix-media-repo/services/url_service.go b/src/github.com/turt2live/matrix-media-repo/services/url_service.go
index dfe0b8d7..c173feed 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/url_service.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/url_service.go
@@ -1,6 +1,7 @@
package services
import (
+ "context"
"database/sql"
"fmt"
"net"
@@ -10,53 +11,55 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/rcontext"
"github.com/turt2live/matrix-media-repo/services/handlers"
+ "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/storage/stores"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/errcodes"
+ "github.com/turt2live/matrix-media-repo/util/errs"
)
type UrlService struct {
- //store *stores.UrlStore
- i rcontext.RequestInfo
+ store *stores.UrlStore
+ ctx context.Context
+ log *logrus.Entry
}
-func CreateUrlService(i rcontext.RequestInfo) (*UrlService) {
- return &UrlService{i}
+func NewUrlService(ctx context.Context, log *logrus.Entry) (*UrlService) {
+ store := storage.GetDatabase().GetUrlStore(ctx, log)
+ return &UrlService{store, ctx, log}
}
-func returnCachedPreview(cached *types.CachedUrlPreview) (types.UrlPreview, error) {
- if cached.ErrorCode == errcodes.ErrCodeInvalidHost {
- return types.UrlPreview{}, util.ErrInvalidHost
- } else if cached.ErrorCode == errcodes.ErrCodeHostNotFound {
- return types.UrlPreview{}, util.ErrHostNotFound
- } else if cached.ErrorCode == errcodes.ErrCodeHostBlacklisted {
- return types.UrlPreview{}, util.ErrHostBlacklisted
- } else if cached.ErrorCode == errcodes.ErrCodeNotFound {
- return types.UrlPreview{}, util.ErrMediaNotFound
- } else if cached.ErrorCode == errcodes.ErrCodeUnknown {
- return types.UrlPreview{}, errors.New("unknown error")
+func returnCachedPreview(cached *types.CachedUrlPreview) (*types.UrlPreview, error) {
+ if cached.ErrorCode == errs.ErrCodeInvalidHost {
+ return nil, errs.ErrInvalidHost
+ } else if cached.ErrorCode == errs.ErrCodeHostNotFound {
+ return nil, errs.ErrHostNotFound
+ } else if cached.ErrorCode == errs.ErrCodeHostBlacklisted {
+ return nil, errs.ErrHostBlacklisted
+ } else if cached.ErrorCode == errs.ErrCodeNotFound {
+ return nil, errs.ErrMediaNotFound
+ } else if cached.ErrorCode == errs.ErrCodeUnknown {
+ return nil, errors.New("unknown error")
}
- return *cached.Preview, nil
+ return cached.Preview, nil
}
-func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string, atTs int64) (types.UrlPreview, error) {
- s.i.Log = s.i.Log.WithFields(logrus.Fields{
+func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string, atTs int64) (*types.UrlPreview, error) {
+ s.log = s.log.WithFields(logrus.Fields{
"urlService_ts": atTs,
})
- urlStore := s.i.Db.GetUrlStore(s.i.Context, s.i.Log)
- cached, err := urlStore.GetPreview(urlStr, atTs)
+ cached, err := s.store.GetPreview(urlStr, atTs)
if err != nil {
- s.i.Log.Error("Error getting cached URL: " + err.Error())
+ s.log.Error("Error getting cached URL: " + err.Error())
}
if err != nil && err != sql.ErrNoRows {
- return types.UrlPreview{}, err
+ return nil, err
}
if err != sql.ErrNoRows {
- s.i.Log.Info("Returning cached URL preview")
+ s.log.Info("Returning cached URL preview")
return returnCachedPreview(cached)
}
@@ -66,24 +69,24 @@ func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string,
return s.GetPreview(urlStr, onHost, forUserId, now)
}
- s.i.Log.Info("URL preview not cached - fetching resource")
+ s.log.Info("URL preview not cached - fetching resource")
parsedUrl, err := url.ParseRequestURI(urlStr)
if err != nil {
- s.i.Log.Error("Error parsing url: " + err.Error())
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeInvalidHost)
- return types.UrlPreview{}, util.ErrInvalidHost
+ s.log.Error("Error parsing url: " + err.Error())
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeInvalidHost)
+ return nil, errs.ErrInvalidHost
}
addrs, err := net.LookupIP(parsedUrl.Host)
if err != nil {
- s.i.Log.Error("Error getting host info: " + err.Error())
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeInvalidHost)
- return types.UrlPreview{}, util.ErrInvalidHost
+ s.log.Error("Error getting host info: " + err.Error())
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeInvalidHost)
+ return nil, errs.ErrInvalidHost
}
if len(addrs) == 0 {
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeHostNotFound)
- return types.UrlPreview{}, util.ErrHostNotFound
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeHostNotFound)
+ return nil, errs.ErrHostNotFound
}
addr := addrs[0]
addrStr := fmt.Sprintf("%v", addr)[1:]
@@ -98,24 +101,24 @@ func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string,
if deniedCidrs == nil {
deniedCidrs = []string{}
}
- if !isAllowed(addr, allowedCidrs, deniedCidrs, s.i.Log) {
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeHostBlacklisted)
- return types.UrlPreview{}, util.ErrHostBlacklisted
+ if !isAllowed(addr, allowedCidrs, deniedCidrs, s.log) {
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeHostBlacklisted)
+ return nil, errs.ErrHostBlacklisted
}
- s.i.Log = s.i.Log.WithFields(logrus.Fields{
+ s.log = s.log.WithFields(logrus.Fields{
"previewer": "OpenGraph",
})
- previewer := &handlers.OpenGraphUrlPreviewer{Info: s.i}
+ previewer := handlers.NewOpenGraphPreviewer(s.ctx, s.log)
preview, err := previewer.GeneratePreview(urlStr)
if err != nil {
- if err == util.ErrMediaNotFound {
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeNotFound)
+ if err == errs.ErrMediaNotFound {
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeNotFound)
} else {
- urlStore.InsertPreviewError(urlStr, errcodes.ErrCodeUnknown)
+ s.store.InsertPreviewError(urlStr, errs.ErrCodeUnknown)
}
- return types.UrlPreview{}, err
+ return nil, err
}
result := &types.UrlPreview{
@@ -127,18 +130,18 @@ func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string,
}
// Store the thumbnail, if there is one
- mediaSvc := CreateMediaService(s.i)
- if preview.HasImage && !mediaSvc.IsTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader) {
+ mediaSvc := NewMediaService(s.ctx, s.log)
+ if preview.Image != nil && !mediaSvc.IsTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader) {
// UploadMedia will close the read stream for the thumbnail
media, err := mediaSvc.UploadMedia(preview.Image.Data, preview.Image.ContentType, preview.Image.Filename, forUserId, onHost)
if err != nil {
- s.i.Log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
+ s.log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
} else {
img, err := imaging.Open(media.Location)
if err != nil {
- s.i.Log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
+ s.log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
} else {
- result.ImageMxc = util.MediaToMxc(&media)
+ result.ImageMxc = media.MxcUri()
result.ImageType = media.ContentType
result.ImageSize = media.SizeBytes
result.ImageWidth = img.Bounds().Max.X
@@ -153,12 +156,12 @@ func (s *UrlService) GetPreview(urlStr string, onHost string, forUserId string,
ErrorCode: "",
FetchedTs: util.NowMillis(),
}
- err = urlStore.InsertPreview(dbRecord)
+ err = s.store.InsertPreview(dbRecord)
if err != nil {
- s.i.Log.Warn("Error caching URL preview: " + err.Error())
+ s.log.Warn("Error caching URL preview: " + err.Error())
}
- return *result, nil
+ return result, nil
}
func isAllowed(ip net.IP, allowed []string, disallowed []string, log *logrus.Entry) bool {
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/file_store.go b/src/github.com/turt2live/matrix-media-repo/storage/file_store.go
index 51c724f7..fc86dd41 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/file_store.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/file_store.go
@@ -13,12 +13,12 @@ import (
"github.com/turt2live/matrix-media-repo/util"
)
-func PersistFile(file io.Reader, ctx context.Context, db *Database) (string, error) {
+func PersistFile(file io.Reader, ctx context.Context) (string, error) {
var basePath string
var pathSize int64
for i := 0; i < len(config.Get().Uploads.StoragePaths); i++ {
currPath := config.Get().Uploads.StoragePaths[i]
- size, err := db.GetSizeOfFolderBytes(ctx, currPath)
+ size, err := GetDatabase().GetSizeOfFolderBytes(ctx, currPath)
if err != nil {
continue
}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/storage.go b/src/github.com/turt2live/matrix-media-repo/storage/storage.go
index 047cd760..e01c9c3b 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/storage.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/storage.go
@@ -3,9 +3,11 @@ package storage
import (
"context"
"database/sql"
+ "sync"
_ "github.com/lib/pq" // postgres driver
"github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/storage/schema"
"github.com/turt2live/matrix-media-repo/storage/stores"
)
@@ -32,12 +34,27 @@ type repos struct {
urlStore *stores.UrlStoreFactory
}
-func OpenDatabase(connectionString string) (*Database, error) {
- var d Database
+var dbInstance *Database
+
+func GetDatabase() (*Database) {
+ if dbInstance == nil {
+ var once sync.Once
+ once.Do(func() {
+ err := OpenDatabase(config.Get().Database.Postgres)
+ if err != nil {
+ panic(err)
+ }
+ })
+ }
+ return dbInstance
+}
+
+func OpenDatabase(connectionString string) (error) {
+ d := &Database{}
var err error
if d.db, err = sql.Open("postgres", connectionString); err != nil {
- return nil, err
+ return err
}
// Make sure the database is how we want it
@@ -45,23 +62,24 @@ func OpenDatabase(connectionString string) (*Database, error) {
schema.PrepareThumbnails(d.db)
schema.PrepareUrls(d.db)
- // Create the repo factories
+ // New the repo factories
if d.repos.mediaStore, err = stores.InitMediaStore(d.db); err != nil {
- return nil, err
+ return err
}
if d.repos.thumbnailStore, err = stores.InitThumbnailStore(d.db); err != nil {
- return nil, err
+ return err
}
if d.repos.urlStore, err = stores.InitUrlStore(d.db); err != nil {
- return nil, err
+ return err
}
// Prepare the general statements
if d.statements.selectSizeOfFolder, err = d.db.Prepare(selectSizeOfFolder); err != nil {
- return nil, err
+ return err
}
- return &d, nil
+ dbInstance = d
+ return nil
}
func (d *Database) GetMediaStore(ctx context.Context, log *logrus.Entry) (*stores.MediaStore) {
@@ -69,7 +87,7 @@ func (d *Database) GetMediaStore(ctx context.Context, log *logrus.Entry) (*store
}
func (d *Database) GetThumbnailStore(ctx context.Context, log *logrus.Entry) (*stores.ThumbnailStore) {
- return d.repos.thumbnailStore.Create(ctx, log)
+ return d.repos.thumbnailStore.New(ctx, log)
}
func (d *Database) GetUrlStore(ctx context.Context, log *logrus.Entry) (*stores.UrlStore) {
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/stores/media_store.go b/src/github.com/turt2live/matrix-media-repo/storage/stores/media_store.go
index 4db659fd..768c2550 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/stores/media_store.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/stores/media_store.go
@@ -74,15 +74,15 @@ func (s *MediaStore) Insert(media *types.Media) (error) {
return err
}
-func (s *MediaStore) GetByHash(hash string) ([]types.Media, error) {
+func (s *MediaStore) GetByHash(hash string) ([]*types.Media, error) {
rows, err := s.statements.selectMediaByHash.QueryContext(s.ctx, hash)
if err != nil {
return nil, err
}
- var results []types.Media
+ var results []*types.Media
for rows.Next() {
- obj := types.Media{}
+ obj := &types.Media{}
err = rows.Scan(
&obj.Origin,
&obj.MediaId,
@@ -103,7 +103,7 @@ func (s *MediaStore) GetByHash(hash string) ([]types.Media, error) {
return results, nil
}
-func (s *MediaStore) Get(origin string, mediaId string) (types.Media, error) {
+func (s *MediaStore) Get(origin string, mediaId string) (*types.Media, error) {
m := &types.Media{}
err := s.statements.selectMedia.QueryRowContext(s.ctx, origin, mediaId).Scan(
&m.Origin,
@@ -116,5 +116,5 @@ func (s *MediaStore) Get(origin string, mediaId string) (types.Media, error) {
&m.Location,
&m.CreationTs,
)
- return *m, err
+ return m, err
}
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/stores/thumbnail_store.go b/src/github.com/turt2live/matrix-media-repo/storage/stores/thumbnail_store.go
index 5e064126..e28651e3 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/stores/thumbnail_store.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/stores/thumbnail_store.go
@@ -44,7 +44,7 @@ func InitThumbnailStore(sqlDb *sql.DB) (*ThumbnailStoreFactory, error) {
return &store, nil
}
-func (f *ThumbnailStoreFactory) Create(ctx context.Context, entry *logrus.Entry) (*ThumbnailStore) {
+func (f *ThumbnailStoreFactory) New(ctx context.Context, entry *logrus.Entry) (*ThumbnailStore) {
return &ThumbnailStore{
factory: f,
ctx: ctx,
@@ -70,7 +70,7 @@ func (s *ThumbnailStore) Insert(thumbnail *types.Thumbnail) (error) {
return err
}
-func (s *ThumbnailStore) Get(origin string, mediaId string, width int, height int, method string) (types.Thumbnail, error) {
+func (s *ThumbnailStore) Get(origin string, mediaId string, width int, height int, method string) (*types.Thumbnail, error) {
t := &types.Thumbnail{}
err := s.statements.selectThumbnail.QueryRowContext(s.ctx, origin, mediaId, width, height, method).Scan(
&t.Origin,
@@ -83,5 +83,5 @@ func (s *ThumbnailStore) Get(origin string, mediaId string, width int, height in
&t.Location,
&t.CreationTs,
)
- return *t, err
+ return t, err
}
diff --git a/src/github.com/turt2live/matrix-media-repo/types/media.go b/src/github.com/turt2live/matrix-media-repo/types/media.go
index 99789bc4..d0e0baad 100644
--- a/src/github.com/turt2live/matrix-media-repo/types/media.go
+++ b/src/github.com/turt2live/matrix-media-repo/types/media.go
@@ -11,3 +11,7 @@ type Media struct {
Location string
CreationTs int64
}
+
+func (m *Media) MxcUri() string {
+ return "mxc://" + m.Origin + "/" + m.MediaId
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/util/errcodes/errorcodes.go b/src/github.com/turt2live/matrix-media-repo/util/errs/errorcodes.go
similarity index 92%
rename from src/github.com/turt2live/matrix-media-repo/util/errcodes/errorcodes.go
rename to src/github.com/turt2live/matrix-media-repo/util/errs/errorcodes.go
index b40b7a1a..8a2e76b2 100644
--- a/src/github.com/turt2live/matrix-media-repo/util/errcodes/errorcodes.go
+++ b/src/github.com/turt2live/matrix-media-repo/util/errs/errorcodes.go
@@ -1,4 +1,4 @@
-package errcodes
+package errs
const ErrCodeInvalidHost = "M_INVALID_HOST"
const ErrCodeHostNotFound = "M_HOST_NOT_FOUND"
diff --git a/src/github.com/turt2live/matrix-media-repo/util/errors.go b/src/github.com/turt2live/matrix-media-repo/util/errs/errors.go
similarity index 95%
rename from src/github.com/turt2live/matrix-media-repo/util/errors.go
rename to src/github.com/turt2live/matrix-media-repo/util/errs/errors.go
index c7c98eec..9857438a 100644
--- a/src/github.com/turt2live/matrix-media-repo/util/errors.go
+++ b/src/github.com/turt2live/matrix-media-repo/util/errs/errors.go
@@ -1,4 +1,4 @@
-package util
+package errs
import (
"errors"
diff --git a/src/github.com/turt2live/matrix-media-repo/util/mxc.go b/src/github.com/turt2live/matrix-media-repo/util/mxc.go
deleted file mode 100644
index 98c9bdea..00000000
--- a/src/github.com/turt2live/matrix-media-repo/util/mxc.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package util
-
-import "github.com/turt2live/matrix-media-repo/types"
-
-func MediaToMxc(media *types.Media) string {
- return "mxc://" + media.Origin + "/" + media.MediaId
-}
--
GitLab