diff --git a/config.sample.yaml b/config.sample.yaml
index dffa03aaa78c8a7e2aa18bdefe7a500c9f8247a2..f4be43712b2ca35ce0ebe256c096d29bc7bba807 100644
--- a/config.sample.yaml
+++ b/config.sample.yaml
@@ -43,8 +43,25 @@ uploads:
# This is intended for larger deployments where media should be distributed among other
# directories, drives, servers, etc. For smaller deployments, a single entry in this list
# is recommended.
- storagePaths:
- - /var/matrix/media
+ # DEPRECATED.
+ #storagePaths:
+ # - /var/matrix/media
+
+ datastores:
+ - type: file
+ enabled: false # Enable this to set up data storage.
+ priority: 1
+ opts:
+ path: /var/matrix/media
+ - type: s3
+ enabled: false
+ priority: 1
+ opts:
+ endpoint: sfo2.digitaloceanspaces.com
+ accessKeyId: ""
+ accessSecret: ""
+ ssl: true
+ bucketName: "your-media-bucket"
# An optional list of file types that are allowed to be uploaded. If */* or nothing is
# supplied here, then all file types are allowed. Asterisks (*) are wildcards and can be
diff --git a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
index f64f51d77956ebda79347f63a79028c373fa2ec8..b3d0a79bfcdb99ef25ef88ea61dc5c71f646d4c1 100644
--- a/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
+++ b/src/github.com/turt2live/matrix-media-repo/cmd/media_repo/main.go
@@ -26,10 +26,43 @@ func main() {
panic(err)
}
- logrus.Info("Starting media repository...")
+ mediaStore := storage.GetDatabase().GetMediaStore(context.TODO(), &logrus.Entry{})
+
+ logrus.Info("Initializing datastores...")
+ enabledDatastores := 0
+ for _, ds:=range config.Get().Uploads.DataStores {
+ if !ds.Enabled {
+ continue
+ }
+
+ enabledDatastores++
+
+ uri := ""
+ if ds.Type == "file" {
+ path, pathFound := ds.Options["path"]
+ if !pathFound {
+ logrus.Fatal("Missing 'path' on file datastore")
+ }
+ uri = path
+ } else if ds.Type == "s3" {
+ endpoint, epFound := ds.Options["endpoint"]
+ bucket, bucketFound := ds.Options["bucketName"]
+ if !epFound || !bucketFound {
+ logrus.Fatal("Missing 'endpoint' or 'bucketName' on s3 datastore")
+ }
+ uri = fmt.Sprintf("s3://%s/%s", endpoint, bucket)
+ } else {
+ logrus.Fatal("Unknown datastore type: ", ds.Type)
+ }
+
+ _, err := storage.GetOrCreateDatastoreOfType(context.TODO(), &logrus.Entry{}, ds.Type, uri)
+ if err != nil {
+ logrus.Fatal(err)
+ }
+ }
// Print all the known datastores at startup. Doubles as a way to initialize the database.
- datastores, err := storage.GetDatabase().GetMediaStore(context.TODO(), &logrus.Entry{}).GetAllDatastores()
+ datastores, err := mediaStore.GetAllDatastores()
if err != nil {
logrus.Fatal(err)
}
@@ -38,6 +71,9 @@ func main() {
logrus.Info(fmt.Sprintf("\t%s (%s): %s", ds.Type, ds.DatastoreId, ds.Uri))
}
+ // TODO: https://github.com/minio/minio-go support
+
+ logrus.Info("Starting media repository...")
metrics.Init()
webserver.Init() // blocks to listen for requests
}
diff --git a/src/github.com/turt2live/matrix-media-repo/common/config/config.go b/src/github.com/turt2live/matrix-media-repo/common/config/config.go
index ac3d8f37656db9c171ea32a4a0b208f9141afa06..446b6b3e6c427e18bb68411b1eaea1fa69718159 100644
--- a/src/github.com/turt2live/matrix-media-repo/common/config/config.go
+++ b/src/github.com/turt2live/matrix-media-repo/common/config/config.go
@@ -33,6 +33,7 @@ type DatabaseConfig struct {
type UploadsConfig struct {
StoragePaths []string `yaml:"storagePaths,flow"`
+ DataStores []DatastoreConfig `yaml:"datastores"`
MaxSizeBytes int64 `yaml:"maxBytes"`
MinSizeBytes int64 `yaml:"minBytes"`
AllowedTypes []string `yaml:"allowedTypes,flow"`
@@ -40,6 +41,13 @@ type UploadsConfig struct {
ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"`
}
+type DatastoreConfig struct {
+ Type string `yaml:"type"`
+ Enabled bool `yaml:"enabled"`
+ Priority int `yaml:"priority"`
+ Options map[string]string `yaml:"opts,flow"`
+}
+
type DownloadsConfig struct {
MaxSizeBytes int64 `yaml:"maxBytes"`
NumWorkers int `yaml:"numWorkers"`
@@ -209,6 +217,7 @@ func NewDefaultConfig() *MediaRepoConfig {
MinSizeBytes: 100,
ReportedMaxSizeBytes: 0,
StoragePaths: []string{},
+ DataStores: []DatastoreConfig{},
AllowedTypes: []string{"*/*"},
},
Downloads: &DownloadsConfig{
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go
new file mode 100644
index 0000000000000000000000000000000000000000..f141945857ad6683ca6401460d1d8aad10cbd4ab
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/storage/datastore/datastore.go
@@ -0,0 +1,4 @@
+package datastore
+
+// TODO: Upload to DS
+// TODO: Download (get stream) from DS
diff --git a/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go b/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
index eddbe348c4b6cd1ceed0f26793e925b168e02a39..e0f78d0221e75b6dfc068552e3a390fcd2c0c462 100644
--- a/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
+++ b/src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go
@@ -16,6 +16,29 @@ func GetOrCreateDatastore(ctx context.Context, log *logrus.Entry, basePath strin
return getOrCreateDatastoreWithMediaService(mediaService, basePath)
}
+func GetOrCreateDatastoreOfType(ctx context.Context, log *logrus.Entry, dsType string, dsUri string) (*types.Datastore, error) {
+ mediaService := GetDatabase().GetMediaStore(ctx, log)
+ datastore, err := mediaService.GetDatastoreByUri(dsUri)
+ if err != nil && err == sql.ErrNoRows {
+ id, err2 := util.GenerateRandomString(32)
+ if err2 != nil {
+ logrus.Error("Error generating datastore ID for URI ", dsUri, ": ", err)
+ return nil, err2
+ }
+ datastore = &types.Datastore{
+ DatastoreId: id,
+ Type: dsType,
+ Uri: dsUri,
+ }
+ err2 = mediaService.InsertDatastore(datastore)
+ if err2 != nil {
+ logrus.Error("Error creating datastore for URI ", dsUri, ": ", err)
+ return nil, err2
+ }
+ }
+ return datastore, nil
+}
+
func getOrCreateDatastoreWithMediaService(mediaService *stores.MediaStore, basePath string) (*types.Datastore, error) {
datastore, err := mediaService.GetDatastoreByUri(basePath)
if err != nil && err == sql.ErrNoRows {
diff --git a/vendor/manifest b/vendor/manifest
index 0fb7d376742b08725729091f97e5adcb180c10b2..7730ad31b1f6e64a55bc343ba50c6ec2c333ac26 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -163,6 +163,18 @@
"branch": "master",
"path": "/pbutil"
},
+ {
+ "importpath": "github.com/minio/minio-go",
+ "repository": "https://github.com/minio/minio-go",
+ "revision": "c8a261de75c1a9a9ece4dcc0c81ff6db525bcf27",
+ "branch": "master"
+ },
+ {
+ "importpath": "github.com/mitchellh/go-homedir",
+ "repository": "https://github.com/mitchellh/go-homedir",
+ "revision": "af06845cf3004701891bf4fdb884bfe4920b3727",
+ "branch": "master"
+ },
{
"importpath": "github.com/olebedev/emitter",
"repository": "https://github.com/olebedev/emitter",
@@ -257,6 +269,20 @@
"revision": "89742aefa4b206dcf400792f3bd35b542998eb3b",
"branch": "master"
},
+ {
+ "importpath": "golang.org/x/crypto/argon2",
+ "repository": "https://go.googlesource.com/crypto",
+ "revision": "b8fe1690c61389d7d2a8074a507d1d40c5d30448",
+ "branch": "master",
+ "path": "/argon2"
+ },
+ {
+ "importpath": "golang.org/x/crypto/blake2b",
+ "repository": "https://go.googlesource.com/crypto",
+ "revision": "b8fe1690c61389d7d2a8074a507d1d40c5d30448",
+ "branch": "master",
+ "path": "/blake2b"
+ },
{
"importpath": "golang.org/x/crypto/ed25519",
"repository": "https://go.googlesource.com/crypto",
@@ -327,6 +353,34 @@
"branch": "master",
"path": "/html"
},
+ {
+ "importpath": "golang.org/x/net/http/httpguts",
+ "repository": "https://go.googlesource.com/net",
+ "revision": "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf",
+ "branch": "master",
+ "path": "/http/httpguts"
+ },
+ {
+ "importpath": "golang.org/x/net/idna",
+ "repository": "https://go.googlesource.com/net",
+ "revision": "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf",
+ "branch": "master",
+ "path": "/idna"
+ },
+ {
+ "importpath": "golang.org/x/net/publicsuffix",
+ "repository": "https://go.googlesource.com/net",
+ "revision": "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf",
+ "branch": "master",
+ "path": "/publicsuffix"
+ },
+ {
+ "importpath": "golang.org/x/sys/cpu",
+ "repository": "https://go.googlesource.com/sys",
+ "revision": "7ae0202eb74c2b534255c71b5a15fa4115aabbcc",
+ "branch": "master",
+ "path": "/cpu"
+ },
{
"importpath": "golang.org/x/sys/unix",
"repository": "https://go.googlesource.com/sys",
@@ -341,6 +395,34 @@
"branch": "master",
"path": "/windows"
},
+ {
+ "importpath": "golang.org/x/text/secure/bidirule",
+ "repository": "https://go.googlesource.com/text",
+ "revision": "e6919f6577db79269a6443b9dc46d18f2238fb5d",
+ "branch": "master",
+ "path": "/secure/bidirule"
+ },
+ {
+ "importpath": "golang.org/x/text/transform",
+ "repository": "https://go.googlesource.com/text",
+ "revision": "e6919f6577db79269a6443b9dc46d18f2238fb5d",
+ "branch": "master",
+ "path": "/transform"
+ },
+ {
+ "importpath": "golang.org/x/text/unicode/bidi",
+ "repository": "https://go.googlesource.com/text",
+ "revision": "e6919f6577db79269a6443b9dc46d18f2238fb5d",
+ "branch": "master",
+ "path": "/unicode/bidi"
+ },
+ {
+ "importpath": "golang.org/x/text/unicode/norm",
+ "repository": "https://go.googlesource.com/text",
+ "revision": "e6919f6577db79269a6443b9dc46d18f2238fb5d",
+ "branch": "master",
+ "path": "/unicode/norm"
+ },
{
"importpath": "golang.org/x/time/rate",
"repository": "https://go.googlesource.com/time",
@@ -362,6 +444,12 @@
"branch": "master",
"path": "/types"
},
+ {
+ "importpath": "gopkg.in/ini.v1",
+ "repository": "https://gopkg.in/ini.v1",
+ "revision": "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe",
+ "branch": "master"
+ },
{
"importpath": "gopkg.in/yaml.v2",
"repository": "https://gopkg.in/yaml.v2",