diff --git a/.gitignore b/.gitignore
index 849fb6afa82a2afefe61f1fbbbee47d800941c7c..cdf3587bb50088283fbc05c8c363741c902e2be8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
/.idea
/bin
/pkg
+/logs
media-repo.yaml
diff --git a/config.sample.yaml b/config.sample.yaml
index 460952fb3c779568845c214baf8f8bb459554a1b..a6c7e46ece9488daa0b3b6f000914f9085d329a3 100644
--- a/config.sample.yaml
+++ b/config.sample.yaml
@@ -2,6 +2,7 @@
repo:
bindAddress: '127.0.0.1'
port: 8000
+ logDirectory: logs
# The configuration for the homeservers this media repository is known to control. Servers
# not listed here will not be able to upload media.
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/download.go b/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
index 7bf606ce83b1da7831bcc4a8d78ad40f1db0d4a4..928cc315b991e7450fb89bd974ea4b2880fa43f1 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/download.go
@@ -4,6 +4,7 @@ import (
"net/http"
"github.com/gorilla/mux"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/media_handler"
@@ -26,7 +27,7 @@ type DownloadMediaResponse struct {
Location string
}
-func DownloadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig) interface{} {
+func DownloadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig, log *logrus.Entry) interface{} {
if !ValidateUserCanDownload(r, db, c) {
return client.AuthFailed()
}
@@ -37,14 +38,21 @@ func DownloadMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
mediaId := params["mediaId"]
filename := params["filename"]
- media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db)
+ log = log.WithFields(logrus.Fields{
+ "mediaId": mediaId,
+ "server": server,
+ "filename": filename,
+ })
+
+ media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db, log)
if err != nil {
if err == media_handler.ErrMediaNotFound {
return client.NotFoundError()
} else if err == media_handler.ErrMediaTooLarge {
return client.RequestTooLarge()
}
- return client.InternalServerError(err.Error())
+ log.Error("Unexpected error locating media: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
}
if filename == "" {
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go b/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
index 8e57a1048dd89d17c5dcf5ad6fed5e7f786e531b..b1830d255af2d312f78a96b47183a90774ad567d 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go
@@ -5,6 +5,7 @@ import (
"strconv"
"github.com/gorilla/mux"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/media_handler"
@@ -22,7 +23,7 @@ import (
// Headers: Content-Type
// Body: <byte[]>
-func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig) interface{} {
+func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig, log *logrus.Entry) interface{} {
if !ValidateUserCanDownload(r, db, c) {
return client.AuthFailed()
}
@@ -32,6 +33,11 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
server := params["server"]
mediaId := params["mediaId"]
+ log = log.WithFields(logrus.Fields{
+ "mediaId": mediaId,
+ "server": server,
+ })
+
widthStr := r.URL.Query().Get("width")
heightStr := r.URL.Query().Get("height")
method := r.URL.Query().Get("method")
@@ -42,14 +48,14 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
if widthStr != "" {
parsedWidth, err := strconv.Atoi(widthStr)
if err != nil {
- return client.InternalServerError(err.Error())
+ return client.InternalServerError("Width does not appear to be an integer")
}
width = parsedWidth
}
if heightStr != "" {
parsedHeight, err := strconv.Atoi(heightStr)
if err != nil {
- return client.InternalServerError(err.Error())
+ return client.InternalServerError("Height does not appear to be an integer")
}
height = parsedHeight
}
@@ -57,17 +63,27 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
method = "crop"
}
- media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db)
+ log = log.WithFields(logrus.Fields{
+ "requestedWidth": width,
+ "requestedHeight": height,
+ "requestedMethod": method,
+ })
+
+ media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db, log)
if err != nil {
if err == media_handler.ErrMediaNotFound {
return client.NotFoundError()
+ } else if err == media_handler.ErrMediaTooLarge {
+ return client.RequestTooLarge()
}
- return client.InternalServerError(err.Error())
+ log.Error("Unexpected error locating media: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
}
- thumb, err := media_handler.GetThumbnail(r.Context(), media, width, height, method, c, db)
+ thumb, err := media_handler.GetThumbnail(r.Context(), media, width, height, method, c, db, log)
if err != nil {
- return client.InternalServerError(err.Error())
+ log.Error("Unexpected error getting thumbnail: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
}
return &DownloadMediaResponse{
diff --git a/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go b/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
index 6b8d09887b26c25fc4d37949881d4ff18caaa557..34904c2fc3fb30d3f0b8aafce385d7924928950d 100644
--- a/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
+++ b/src/github.com/turt2live/matrix-media-repo/client/r0/upload.go
@@ -4,6 +4,7 @@ import (
"io"
"net/http"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/media_handler"
@@ -23,7 +24,7 @@ type MediaUploadedResponse struct {
ContentUri string `json:"content_uri"`
}
-func UploadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig) interface{} {
+func UploadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c config.MediaRepoConfig, log *logrus.Entry) interface{} {
accessToken := util.GetAccessTokenFromRequest(r)
userId, err := util.GetUserIdFromToken(r.Context(), r.Host, accessToken, c)
if err != nil || userId == "" {
@@ -35,6 +36,11 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c
filename = "upload.bin"
}
+ log = log.WithFields(logrus.Fields{
+ "filename": filename,
+ "userId": userId,
+ })
+
contentType := r.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream" // binary
@@ -54,9 +60,10 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c
Contents: reader,
}
- mxc, err := request.StoreAndGetMxcUri(r.Context(), c, db)
+ mxc, err := request.StoreAndGetMxcUri(r.Context(), c, db, log)
if err != nil {
- return client.InternalServerError(err.Error())
+ log.Error("Unexpected error storing media: " + err.Error())
+ return client.InternalServerError("Unexpected Error")
}
return &MediaUploadedResponse{mxc}
diff --git a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
index 2f6ea627fd61ca4795c5b54cb90272e94dbe9595..f5d2fe4ce9b612f4184fb2dcedb4522353e2600b 100644
--- a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
+++ b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
@@ -1,31 +1,39 @@
package main
import (
- json "encoding/json"
+ "encoding/json"
"fmt"
"io"
"net/http"
"os"
+ "reflect"
"strconv"
"github.com/gorilla/mux"
+ log "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/client"
"github.com/turt2live/matrix-media-repo/client/r0"
"github.com/turt2live/matrix-media-repo/config"
+ "github.com/turt2live/matrix-media-repo/logging"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/util"
)
const UnkErrJson = `{"code":"M_UNKNOWN","message":"Unexpected error processing response"}`
+type requestCounter struct {
+ lastId int
+}
+
type Handler struct {
- h func(http.ResponseWriter, *http.Request, storage.Database, config.MediaRepoConfig) interface{}
+ h func(http.ResponseWriter, *http.Request, storage.Database, config.MediaRepoConfig, *log.Entry) interface{}
opts HandlerOpts
}
type HandlerOpts struct {
db storage.Database
config config.MediaRepoConfig
+ reqCounter *requestCounter
}
type EmptyResponse struct {}
@@ -38,24 +46,34 @@ func main() {
panic(err)
}
+ err = logging.Setup(c.General.LogDirectory)
+ if err != nil {
+ panic(err)
+ }
+
+ log.Info("Starting media repository...")
+
db, err := storage.OpenDatabase(c.Database.Postgres)
if err != nil {
panic(err)
}
- hOpts := HandlerOpts{*db, c}
+ counter := requestCounter{}
+ hOpts := HandlerOpts{*db, c, &counter}
uploadHandler := Handler{r0.UploadMedia, hOpts}
downloadHandler := Handler{r0.DownloadMedia, hOpts}
thumbnailHandler := Handler{r0.ThumbnailMedia, hOpts}
// r0 endpoints
+ log.Info("Registering r0 endpoints")
rtr.Handle("/_matrix/media/r0/upload", uploadHandler).Methods("POST")
rtr.Handle("/_matrix/media/r0/download/{server:[a-zA-Z0-9.:-_]+}/{mediaId:[a-zA-Z0-9]+}", downloadHandler).Methods("GET")
rtr.Handle("/_matrix/media/r0/download/{server:[a-zA-Z0-9.:-_]+}/{mediaId:[a-zA-Z0-9]+}/{filename:[a-zA-Z0-9._-]+}", downloadHandler).Methods("GET")
rtr.Handle("/_matrix/media/r0/thumbnail/{server:[a-zA-Z0-9.:-_]+}/{mediaId:[a-zA-Z0-9]+}", thumbnailHandler).Methods("GET")
// v1 endpoints (legacy)
+ log.Info("Registering v1 endpoints")
rtr.Handle("/_matrix/media/v1/upload", uploadHandler).Methods("POST")
rtr.Handle("/_matrix/media/v1/download/{server:[a-zA-Z0-9.:-_]+}/{mediaId:[a-zA-Z0-9]+}", downloadHandler).Methods("GET")
rtr.Handle("/_matrix/media/v1/download/{server:[a-zA-Z0-9.:-_]+}/{mediaId:[a-zA-Z0-9]+}/{filename:[a-zA-Z0-9._-]+}", downloadHandler).Methods("GET")
@@ -64,11 +82,25 @@ func main() {
// TODO: Intercept 404, 500, and 400 to respond with M_NOT_FOUND and M_UNKNOWN
// TODO: Rate limiting (429 M_LIMIT_EXCEEDED)
+ address:=c.General.BindAddress+":"+strconv.Itoa(c.General.Port)
http.Handle("/", rtr)
- http.ListenAndServe(c.General.BindAddress+":"+strconv.Itoa(c.General.Port), nil)
+
+ log.WithField("address", address).Info("Started up. Listening at http://" + address)
+ http.ListenAndServe(address, nil)
}
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ contextLog := log.WithFields(log.Fields{
+ "method": r.Method,
+ "host": r.Host,
+ "resource": r.URL.Path,
+ "contentType": r.Header.Get("Content-Type"),
+ "contentLength": r.Header.Get("Content-Length"),
+ "queryString": util.GetLogSafeQueryString(r),
+ "requestId": h.opts.reqCounter.GetNextId(),
+ })
+ contextLog.Info("Received request")
+
// Send CORS and other basic headers
w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
@@ -80,7 +112,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Process response
var res interface{} = client.AuthFailed()
if util.IsServerOurs(r.Host, h.opts.config) {
- res = h.h(w, r, h.opts.db, h.opts.config)
+ contextLog.Info("Server is owned by us, processing request")
+ res = h.h(w, r, h.opts.db, h.opts.config, contextLog)
if res == nil {
res = &EmptyResponse{}
}
@@ -94,6 +127,13 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
jsonStr := string(b)
+ _, isDownload := res.(r0.DownloadMediaResponse)
+ if isDownload {
+ contextLog.Info("Replying with result: " + reflect.TypeOf(res).Elem().Name())
+ } else {
+ contextLog.Info("Replying with result: " + jsonStr)
+ }
+
switch result := res.(type) {
case *client.ErrorResponse:
w.Header().Set("Content-Type", "application/json")
@@ -131,4 +171,11 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, jsonStr)
break
}
+}
+
+func (c *requestCounter) GetNextId() string {
+ strId := strconv.Itoa(c.lastId)
+ c.lastId = c.lastId + 1
+
+ return "REQ-" + strId
}
\ No newline at end of file
diff --git a/src/github.com/turt2live/matrix-media-repo/config/config.go b/src/github.com/turt2live/matrix-media-repo/config/config.go
index df902f2a632500da1d27392c6d1a03f9cd5dea71..9037d51d7334d3a67dbc21301b01627804324d79 100644
--- a/src/github.com/turt2live/matrix-media-repo/config/config.go
+++ b/src/github.com/turt2live/matrix-media-repo/config/config.go
@@ -17,6 +17,7 @@ type MediaRepoConfig struct {
General struct {
BindAddress string `yaml:"bindAddress"`
Port int `yaml:"port"`
+ LogDirectory string `yaml:"logDirectory"`
} `yaml:"repo"`
Homeservers []HomeserverConfig `yaml:"homeservers,flow"`
diff --git a/src/github.com/turt2live/matrix-media-repo/logging/logger.go b/src/github.com/turt2live/matrix-media-repo/logging/logger.go
new file mode 100644
index 0000000000000000000000000000000000000000..ca434f4e9517b4c756b6de851123d26074e8deae
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/logging/logger.go
@@ -0,0 +1,58 @@
+package logging
+
+import (
+ "os"
+ "path"
+ "time"
+
+ "github.com/lestrrat/go-file-rotatelogs"
+ "github.com/rifflock/lfshook"
+ "github.com/sirupsen/logrus"
+)
+
+type utcFormatter struct {
+ logrus.Formatter
+}
+
+func (f utcFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ entry.Time = entry.Time.UTC()
+ return f.Formatter.Format(entry)
+}
+
+func Setup(dir string) error {
+ logrus.SetFormatter(&utcFormatter{
+ &logrus.TextFormatter{
+ TimestampFormat: "2006-01-02 15:04:05.000 Z07:00",
+ FullTimestamp: true,
+ DisableColors: false,
+ DisableTimestamp: false,
+ QuoteEmptyFields: true,
+ },
+ })
+ logrus.SetOutput(os.Stdout)
+
+ if dir == "" { return nil }
+ _ = os.MkdirAll(dir, os.ModePerm)
+
+ logFile := path.Join(dir, "media_repo.log")
+ writer, err := rotatelogs.New(
+ logFile + ".%Y%m%d%H%M",
+ rotatelogs.WithLinkName(logFile),
+ rotatelogs.WithMaxAge((24 * time.Hour) * 14), // keep for 14 days
+ rotatelogs.WithRotationTime(24 * time.Hour), // rotate every 24 hours
+ )
+ if err != nil {
+ return err
+ }
+
+ logrus.AddHook(lfshook.NewHook(lfshook.WriterMap{
+ logrus.DebugLevel: writer,
+ logrus.InfoLevel: writer,
+ logrus.WarnLevel: writer,
+ logrus.ErrorLevel: writer,
+ logrus.FatalLevel: writer,
+ logrus.PanicLevel: writer,
+ }))
+
+ return nil
+}
\ No newline at end of file
diff --git a/src/github.com/turt2live/matrix-media-repo/media_handler/locator.go b/src/github.com/turt2live/matrix-media-repo/media_handler/locator.go
index bc447262158610cba72dffccbf55d1ff11e5c218..3bfff80c97bee85f819a73f5983e9c3a6c0a3267 100644
--- a/src/github.com/turt2live/matrix-media-repo/media_handler/locator.go
+++ b/src/github.com/turt2live/matrix-media-repo/media_handler/locator.go
@@ -8,6 +8,7 @@ import (
"strconv"
"github.com/matrix-org/gomatrixserverlib"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/types"
@@ -17,15 +18,18 @@ import (
var ErrMediaNotFound = errors.New("media not found")
var ErrMediaTooLarge = errors.New("media too large")
-func FindMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+func FindMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Media, error) {
+ log.Info("Looking up media")
media, err := db.GetMedia(ctx, server, mediaId)
if err != nil {
if err == sql.ErrNoRows {
if util.IsServerOurs(server, c) {
+ log.Warn("Media not found")
return media, ErrMediaNotFound
}
- media, err = DownloadMedia(ctx, server, mediaId, c, db)
+ log.Info("Attempting to download remote media")
+ media, err = DownloadMedia(ctx, server, mediaId, c, db, log)
return media, err
}
return media, err
@@ -34,7 +38,7 @@ func FindMedia(ctx context.Context, server string, mediaId string, c config.Medi
return media, nil
}
-func DownloadMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+func DownloadMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Media, error) {
request := &MediaUploadRequest{
UploadedBy: "",
Host: server,
@@ -51,8 +55,10 @@ func DownloadMedia(ctx context.Context, server string, mediaId string, c config.
}
if resp.StatusCode == 404 {
+ log.Info("Remote media not found")
return types.Media{}, ErrMediaNotFound
} else if resp.StatusCode != 200 {
+ log.Info("Unknown error fetching remote media; received status code " + strconv.Itoa(resp.StatusCode))
return types.Media{}, errors.New("could not fetch remote media")
}
@@ -62,6 +68,7 @@ func DownloadMedia(ctx context.Context, server string, mediaId string, c config.
return types.Media{}, err
}
if c.Downloads.MaxSizeBytes > 0 && contentLength > c.Downloads.MaxSizeBytes {
+ log.Warn("Attempted to download media that was too large")
return types.Media{}, ErrMediaTooLarge
}
@@ -73,5 +80,6 @@ func DownloadMedia(ctx context.Context, server string, mediaId string, c config.
request.DesiredFilename = params["filename"]
}
- return request.StoreMediaWithId(ctx, mediaId, c, db)
-}
+ log.Info("Persisting downloaded remote media")
+ return request.StoreMediaWithId(ctx, mediaId, c, db, log)
+}
\ No newline at end of file
diff --git a/src/github.com/turt2live/matrix-media-repo/media_handler/media_handler.go b/src/github.com/turt2live/matrix-media-repo/media_handler/media_handler.go
index 150d9b130f081f01a023bbd18abb7c38ae1d1983..baa97c0be6ed26b59bb79eeffc2dc30c45667b11 100644
--- a/src/github.com/turt2live/matrix-media-repo/media_handler/media_handler.go
+++ b/src/github.com/turt2live/matrix-media-repo/media_handler/media_handler.go
@@ -6,6 +6,7 @@ import (
"os"
"time"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/types"
@@ -20,8 +21,8 @@ type MediaUploadRequest struct {
ContentType string
}
-func (r MediaUploadRequest) StoreAndGetMxcUri(ctx context.Context, c config.MediaRepoConfig, db storage.Database) (string, error) {
- media, err := r.StoreMedia(ctx, c, db)
+func (r MediaUploadRequest) StoreAndGetMxcUri(ctx context.Context, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (string, error) {
+ media, err := r.StoreMedia(ctx, c, db, log)
if err != nil {
return "", err
}
@@ -29,7 +30,11 @@ func (r MediaUploadRequest) StoreAndGetMxcUri(ctx context.Context, c config.Medi
return util.MediaToMxc(&media), nil
}
-func (r MediaUploadRequest) StoreMediaWithId(ctx context.Context, mediaId string, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+func (r MediaUploadRequest) StoreMediaWithId(ctx context.Context, mediaId string, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Media, error) {
+ log = log.WithFields(logrus.Fields{
+ "handlerMediaId": mediaId,
+ })
+
destination, err := storage.PersistFile(ctx, r.Contents, c, db)
if err != nil {
return types.Media{}, err
@@ -58,16 +63,20 @@ func (r MediaUploadRequest) StoreMediaWithId(ctx context.Context, mediaId string
// If the media is exactly the same, just return it
if IsMediaSame(media, r) {
+ log.Info("Exact media duplicate found, returning unaltered media record")
return media, nil
}
if media.Origin == r.Host {
+ log.Info("Media duplicate found, assigning a new media ID for new origin")
// Generate a new ID for this upload
media.MediaId = GenerateMediaId()
break
}
}
+ log.Info("Duplicate media found, generating new record using existing file")
+
media.Origin = r.Host
media.UserId = r.UploadedBy
media.UploadName = r.DesiredFilename
@@ -82,6 +91,8 @@ func (r MediaUploadRequest) StoreMediaWithId(ctx context.Context, mediaId string
return media, nil
}
+ log.Info("Persisting unique media record")
+
fileSize, err := util.FileSize(destination)
if err != nil {
return types.Media{}, err
@@ -108,8 +119,8 @@ func (r MediaUploadRequest) StoreMediaWithId(ctx context.Context, mediaId string
return *media, nil
}
-func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
- return r.StoreMediaWithId(ctx, GenerateMediaId(), c, db)
+func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Media, error) {
+ return r.StoreMediaWithId(ctx, GenerateMediaId(), c, db, log)
}
func GenerateMediaId() string {
diff --git a/src/github.com/turt2live/matrix-media-repo/media_handler/thumbnailer.go b/src/github.com/turt2live/matrix-media-repo/media_handler/thumbnailer.go
index f000e8c7b20153bb0a96dd23ea937646b21bbb6b..a93a6a1652644675c6a0f2d1f2380f4f3178bd92 100644
--- a/src/github.com/turt2live/matrix-media-repo/media_handler/thumbnailer.go
+++ b/src/github.com/turt2live/matrix-media-repo/media_handler/thumbnailer.go
@@ -8,13 +8,14 @@ import (
"time"
"github.com/disintegration/imaging"
+ "github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
)
-func GetThumbnail(ctx context.Context, media types.Media, width int, height int, method string, c config.MediaRepoConfig, db storage.Database) (types.Thumbnail, error) {
+func GetThumbnail(ctx context.Context, media types.Media, width int, height int, method string, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Thumbnail, error) {
if width <= 0 {
return types.Thumbnail{}, errors.New("width must be positive")
}
@@ -58,22 +59,32 @@ func GetThumbnail(ctx context.Context, media types.Media, width int, height int,
}
}
+ log = log.WithFields(logrus.Fields{
+ "targetWidth": targetWidth,
+ "targetHeight": targetHeight,
+ })
+ log.Info("Looking up thumbnail")
+
thumb, err := db.GetThumbnail(ctx, media.Origin, media.MediaId, targetWidth, targetHeight, method)
if err != nil && err != sql.ErrNoRows {
+ log.Error("Unexpected error processing thumbnail lookup: " + err.Error())
return thumb, err
}
if err != sql.ErrNoRows {
- return thumb, err
+ log.Info("Found existing thumbnail")
+ return thumb, nil
}
if media.SizeBytes > c.Thumbnails.MaxSourceBytes {
+ log.Warn("Media too large to thumbnail")
return thumb, errors.New("cannot thumbnail, image too large")
}
- return generateThumbnail(ctx, media, targetWidth, targetHeight, method, c, db)
+ log.Info("Generating new thumbnail")
+ return generateThumbnail(ctx, media, targetWidth, targetHeight, method, c, db, log)
}
-func generateThumbnail(ctx context.Context, media types.Media, width int, height int, method string, c config.MediaRepoConfig, db storage.Database) (types.Thumbnail, error) {
+func generateThumbnail(ctx context.Context, media types.Media, width int, height int, method string, c config.MediaRepoConfig, db storage.Database, log *logrus.Entry) (types.Thumbnail, error) {
thumb := &types.Thumbnail{
Origin: media.Origin,
MediaId: media.MediaId,
@@ -99,6 +110,7 @@ func generateThumbnail(ctx context.Context, media types.Media, width int, height
if aspectRatio == targetAspectRatio {
// Highly unlikely, but if the aspect ratios match then just resize
method = "scale"
+ log.Info("Aspect ratio is the same, converting method to 'scale'")
}
if srcWidth <= width && srcHeight <= height {
@@ -106,6 +118,7 @@ func generateThumbnail(ctx context.Context, media types.Media, width int, height
thumb.ContentType = media.ContentType
thumb.Location = media.Location
thumb.SizeBytes = media.SizeBytes
+ log.Warn("Image too small, returning raw image")
return *thumb, nil
}
@@ -114,6 +127,7 @@ func generateThumbnail(ctx context.Context, media types.Media, width int, height
} else if method == "crop" {
src = imaging.Fill(src, width, height, imaging.Center, imaging.Lanczos)
} else {
+ log.Error("Unrecognized thumbnail method: " + method)
return *thumb, errors.New("unrecognized method: " + method)
}
@@ -121,17 +135,20 @@ func generateThumbnail(ctx context.Context, media types.Media, width int, height
imgData := &bytes.Buffer{}
err = imaging.Encode(imgData, src, imaging.PNG)
if err != nil {
+ log.Error("Unexpected error encoding thumbnail: " + err.Error())
return *thumb, err
}
// Reset the buffer pointer and store the file
location, err := storage.PersistFile(ctx, imgData, c, db)
if err != nil {
+ log.Error("Unexpected error saving thumbnail: " + err.Error())
return *thumb, err
}
fileSize, err := util.FileSize(location)
if err != nil {
+ log.Error("Unexpected error getting the size of the thumbnail: " + err.Error())
return *thumb, err
}
@@ -141,6 +158,7 @@ func generateThumbnail(ctx context.Context, media types.Media, width int, height
err = db.InsertThumbnail(ctx, thumb)
if err != nil {
+ log.Error("Unexpected error caching thumbnail: " + err.Error())
return *thumb, err
}
diff --git a/src/github.com/turt2live/matrix-media-repo/util/http.go b/src/github.com/turt2live/matrix-media-repo/util/http.go
index 1161672b26884631db863ccee773760f562a0457..1dbedeee61df1abc72a9bb16eca298ba51f9b292 100644
--- a/src/github.com/turt2live/matrix-media-repo/util/http.go
+++ b/src/github.com/turt2live/matrix-media-repo/util/http.go
@@ -12,4 +12,14 @@ func GetAccessTokenFromRequest(request *http.Request) (string) {
}
return request.URL.Query().Get("access_token")
-}
\ No newline at end of file
+}
+
+func GetLogSafeQueryString(r *http.Request) (string) {
+ qs := r.URL.Query()
+
+ if qs.Get("access_token") != "" {
+ qs.Set("access_token", "redacted")
+ }
+
+ return qs.Encode()
+}
diff --git a/vendor/manifest b/vendor/manifest
index 22c63c01cbf61f9142a7eb1de6f075f66d89aec6..29ba77e3d7b3dd2faa8c4fe921dca5b0ddae5ee8 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -13,6 +13,18 @@
"revision": "2d5fef06b891c971b14aa6f71ca5ab6c03a36e0e",
"branch": "master"
},
+ {
+ "importpath": "github.com/lestrrat/go-file-rotatelogs",
+ "repository": "https://github.com/lestrrat/go-file-rotatelogs",
+ "revision": "9df8b44f21785240553882138c5df2e9cc1db910",
+ "branch": "master"
+ },
+ {
+ "importpath": "github.com/lestrrat/go-strftime",
+ "repository": "https://github.com/lestrrat/go-strftime",
+ "revision": "04ef93e285313c8978cbc7cad26d2aa7a9927451",
+ "branch": "master"
+ },
{
"importpath": "github.com/lib/pq",
"repository": "https://github.com/lib/pq",
@@ -37,6 +49,18 @@
"revision": "8b1c8ab81986c1ce7f06a52fce48f4a1156b66ee",
"branch": "master"
},
+ {
+ "importpath": "github.com/pkg/errors",
+ "repository": "https://github.com/pkg/errors",
+ "revision": "f15c970de5b76fac0b59abb32d62c17cc7bed265",
+ "branch": "master"
+ },
+ {
+ "importpath": "github.com/rifflock/lfshook",
+ "repository": "https://github.com/rifflock/lfshook",
+ "revision": "3bcf86f879c771238f8a67832a1af71308801a47",
+ "branch": "master"
+ },
{
"importpath": "github.com/sirupsen/logrus",
"repository": "https://github.com/sirupsen/logrus",