Download remote media

config.sample.yaml

@@ -23,6 +23,10 @@ uploads:
   storagePaths:
   - /var/matrix/media
 
+# The file download settings when retrieving media from other servers
+downloads:
+  maxBytes: 10485760 # 10MB default, 0 to disable
+
 # The thumbnail configuration for the media repository.
 thumbnails:
   # The maximum number of bytes an image can be before the thumbnailer refuses.

src/github.com/turt2live/matrix-media-repo/client/r0/download.go

 package r0
 
 import (
-	"database/sql"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/turt2live/matrix-media-repo/client"
 	"github.com/turt2live/matrix-media-repo/config"
+	"github.com/turt2live/matrix-media-repo/media_handler"
 	"github.com/turt2live/matrix-media-repo/storage"
 )
 
@@ -32,11 +32,12 @@ func DownloadMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
 	mediaId := params["mediaId"]
 	filename := params["filename"]
 
-	media, err := db.GetMedia(r.Context(), server, mediaId)
+	media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db)
 	if err != nil {
-		if err == sql.ErrNoRows {
-			// TODO: Try remote fetch
+		if err == media_handler.ErrMediaNotFound {
 			return client.NotFoundError()
+		} else if err == media_handler.ErrMediaTooLarge {
+			return client.RequestTooLarge()
 		}
 		return client.InternalServerError(err.Error())
 	}

src/github.com/turt2live/matrix-media-repo/client/r0/thumbnail.go

 package r0
 
 import (
-	"database/sql"
 	"net/http"
 	"strconv"
 
@@ -54,10 +53,9 @@ func ThumbnailMedia(w http.ResponseWriter, r *http.Request, db storage.Database,
 		method = "crop"
 	}
 
-	media, err := db.GetMedia(r.Context(), server, mediaId)
+	media, err := media_handler.FindMedia(r.Context(), server, mediaId, c, db)
 	if err != nil {
-		if err == sql.ErrNoRows {
-			// TODO: Try remote fetch
+		if err == media_handler.ErrMediaNotFound {
 			return client.NotFoundError()
 		}
 		return client.InternalServerError(err.Error())

src/github.com/turt2live/matrix-media-repo/client/r0/upload.go

@@ -49,7 +49,7 @@ func UploadMedia(w http.ResponseWriter, r *http.Request, db storage.Database, c
 		Contents:        reader,
 	}
 
-	mxc, err := request.StoreMedia(r.Context(), c, db)
+	mxc, err := request.StoreAndGetMxcUri(r.Context(), c, db)
 	if err != nil {
 		return client.InternalServerError(err.Error())
 	}

src/github.com/turt2live/matrix-media-repo/client/responses.go

@@ -3,12 +3,17 @@ package client
 type ErrorResponse struct {
 	Code string `json:"errcode"`
 	Message string `json:"error"`
+	InternalCode string `json:"mr_errcode"`
 }
 
 func InternalServerError(message string) *ErrorResponse {
-	return &ErrorResponse{"M_UNKNOWN", message}
+	return &ErrorResponse{"M_UNKNOWN", message, "M_UNKNOWN"}
 }
 
 func NotFoundError() *ErrorResponse {
-	return &ErrorResponse{"M_NOT_FOUND", "Not found"}
+	return &ErrorResponse{"M_NOT_FOUND", "Not found", "M_NOT_FOUND"}
+}
+
+func RequestTooLarge() *ErrorResponse {
+	return &ErrorResponse{"M_UNKNOWN", "Too Large", "M_MEDIA_TOO_LARGE"}
 }
\ No newline at end of file

src/github.com/turt2live/matrix-media-repo/config/config.go

@@ -23,6 +23,10 @@ type MediaRepoConfig struct {
 		MaxSizeBytes int64 `yaml:"maxBytes"`
 	} `yaml:"uploads"`
 
+	Downloads struct {
+		MaxSizeBytes int64 `yaml:"maxBytes"`
+	} `yaml:"downloads"`
+
 	Thumbnails struct {
 		MaxSourceBytes int64 `yaml:"maxSourceBytes"`
 		Sizes []struct {

src/github.com/turt2live/matrix-media-repo/media_handler/locator.go

+package media_handler
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"mime"
+	"strconv"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/turt2live/matrix-media-repo/config"
+	"github.com/turt2live/matrix-media-repo/storage"
+	"github.com/turt2live/matrix-media-repo/types"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+var ErrMediaNotFound = errors.New("media not found")
+var ErrMediaTooLarge = errors.New("media too large")
+
+func FindMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+	media, err := db.GetMedia(ctx, server, mediaId)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			if util.IsServerOurs(server, c) {
+				return media, ErrMediaNotFound
+			}
+
+			media, err = DownloadMedia(ctx, server, mediaId, c, db)
+			return media, err
+		}
+		return media, err
+	}
+
+	return media, nil
+}
+
+func DownloadMedia(ctx context.Context, server string, mediaId string, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+	request := &MediaUploadRequest{
+		UploadedBy: "",
+		Host:       server,
+		//ContentType:
+		//DesiredFilename:
+		//Contents:
+	}
+
+	mtxClient := gomatrixserverlib.NewClient()
+	mtxServer := gomatrixserverlib.ServerName(server)
+	resp, err := mtxClient.CreateMediaDownloadRequest(ctx, mtxServer, mediaId)
+	if err != nil {
+		return types.Media{}, err
+	}
+
+	if resp.StatusCode == 404 {
+		return types.Media{}, ErrMediaNotFound
+	} else if resp.StatusCode != 200 {
+		return types.Media{}, errors.New("could not fetch remote media")
+	}
+
+	defer resp.Body.Close()
+	contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return types.Media{}, err
+	}
+	if c.Downloads.MaxSizeBytes > 0 && contentLength > c.Downloads.MaxSizeBytes {
+		return types.Media{}, ErrMediaTooLarge
+	}
+
+	request.ContentType = resp.Header.Get("Content-Type")
+	request.Contents = resp.Body
+
+	_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
+	if err == nil && params["filename"] != "" {
+		request.DesiredFilename = params["filename"]
+	}
+
+	return request.StoreMedia(ctx, c, db)
+}

src/github.com/turt2live/matrix-media-repo/media_handler/media_handler.go

@@ -20,22 +20,33 @@ type MediaUploadRequest struct {
 	ContentType string
 }
 
-func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoConfig, db storage.Database) (string, error) {
-	destination, err := storage.PersistFile(ctx, r.Contents, c, db)
+func (r MediaUploadRequest) StoreAndGetMxcUri(ctx context.Context, c config.MediaRepoConfig, db storage.Database) (string, error) {
+	media, err := r.StoreMedia(ctx, c, db)
 	if err != nil {
 		return "", err
 	}
 
+	return util.MediaToMxc(&media), nil
+}
+
+func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoConfig, db storage.Database) (types.Media, error) {
+	placeholder := types.Media{}
+
+	destination, err := storage.PersistFile(ctx, r.Contents, c, db)
+	if err != nil {
+		return placeholder, err
+	}
+
 	hash, err := storage.GetFileHash(destination)
 	if err != nil {
 		os.Remove(destination)
-		return "", err
+		return placeholder, err
 	}
 
 	records, err := db.GetMediaByHash(ctx, hash)
 	if err != nil {
 		os.Remove(destination)
-		return "", err
+		return placeholder, err
 	}
 	if len(records) > 0 {
 		// We can delete the media: It's already duplicated at this point
@@ -49,7 +60,7 @@ func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoCo
 
 			// If the media is exactly the same, just return it
 			if IsMediaSame(media, r) {
-				return util.MediaToMxc(&media), nil
+				return media, nil
 			}
 
 			if media.Origin == r.Host {
@@ -67,15 +78,15 @@ func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoCo
 
 		err = db.InsertMedia(ctx, &media)
 		if err != nil {
-			return "", err
+			return placeholder, err
 		}
 
-		return util.MediaToMxc(&media), nil
+		return media, nil
 	}
 
 	fileSize, err := util.FileSize(destination)
 	if err != nil {
-		return "", err
+		return placeholder, err
 	}
 
 	media := &types.Media{
@@ -93,10 +104,10 @@ func (r MediaUploadRequest) StoreMedia(ctx context.Context, c config.MediaRepoCo
 	err = db.InsertMedia(ctx, media)
 	if err != nil {
 		os.Remove(destination)
-		return "", err
+		return placeholder, err
 	}
 
-	return util.MediaToMxc(media), nil
+	return *media, nil
 }
 
 func GenerateMediaId() string {

src/github.com/turt2live/matrix-media-repo/media_repo.go

@@ -72,11 +72,15 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch result := res.(type) {
 	case *client.ErrorResponse:
-		switch result.Code {
+		switch result.InternalCode {
 		case "M_NOT_FOUND":
 			w.Header().Set("Content-Type", "application/json")
 			http.Error(w, jsonStr, http.StatusNotFound)
 			break
+		case "M_MEDIA_TOO_LARGE":
+			w.Header().Set("Content-Type", "application/json")
+			http.Error(w, jsonStr, http.StatusRequestEntityTooLarge)
+			break
 		//case "M_UNKNOWN":
 		default:
 			w.Header().Set("Content-Type", "application/json")

src/github.com/turt2live/matrix-media-repo/util/config.go

+package util
+
+import "github.com/turt2live/matrix-media-repo/config"
+
+func IsServerOurs(server string, c config.MediaRepoConfig) (bool) {
+	for i := 0; i < len(c.Homeservers); i++ {
+		hs := c.Homeservers[i]
+		if hs.Name == server {
+			return true
+		}
+	}
+
+	return false
+}

vendor/manifest

@@ -19,6 +19,44 @@
 			"revision": "b609790bd85edf8e9ab7e0f8912750a786177bcf",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/matrix-org/gomatrix",
+			"repository": "https://github.com/matrix-org/gomatrix",
+			"revision": "a7fc80c8060c2544fe5d4dae465b584f8e9b4e27",
+			"branch": "master"
+		},
+		{
+			"importpath": "github.com/matrix-org/gomatrixserverlib",
+			"repository": "https://github.com/matrix-org/gomatrixserverlib",
+			"revision": "e46695aa9a7cd517095c9e2c518e2701eae291d5",
+			"branch": "master"
+		},
+		{
+			"importpath": "github.com/matrix-org/util",
+			"repository": "https://github.com/matrix-org/util",
+			"revision": "8b1c8ab81986c1ce7f06a52fce48f4a1156b66ee",
+			"branch": "master"
+		},
+		{
+			"importpath": "github.com/sirupsen/logrus",
+			"repository": "https://github.com/sirupsen/logrus",
+			"revision": "89742aefa4b206dcf400792f3bd35b542998eb3b",
+			"branch": "master"
+		},
+		{
+			"importpath": "golang.org/x/crypto/ed25519",
+			"repository": "https://go.googlesource.com/crypto",
+			"revision": "6a293f2d4b14b8e6d3f0539e383f6d0d30fce3fd",
+			"branch": "master",
+			"path": "/ed25519"
+		},
+		{
+			"importpath": "golang.org/x/crypto/ssh/terminal",
+			"repository": "https://go.googlesource.com/crypto",
+			"revision": "6a293f2d4b14b8e6d3f0539e383f6d0d30fce3fd",
+			"branch": "master",
+			"path": "/ssh/terminal"
+		},
 		{
 			"importpath": "golang.org/x/image/bmp",
 			"repository": "https://go.googlesource.com/image",
@@ -33,6 +71,13 @@
 			"branch": "master",
 			"path": "/tiff"
 		},
+		{
+			"importpath": "golang.org/x/sys/windows",
+			"repository": "https://go.googlesource.com/sys",
+			"revision": "1e2299c37cc91a509f1b12369872d27be0ce98a6",
+			"branch": "master",
+			"path": "/windows"
+		},
 		{
 			"importpath": "gopkg.in/yaml.v2",
 			"repository": "https://gopkg.in/yaml.v2",