Synapse import

This is done over http to ensure that synapse is left with a backup of the media repository. 

Adds #4

README.md

@@ -107,3 +107,31 @@ listeners:
       - names: [federation]
         compress: false
 ```
+
+# Importing media from synapse
+
+Media is imported by connecting to your synapse database and downloading all the content from the homeserver. This is so you have a backup of the media repository still with synapse. **Do not point traffic at the media repo until after the import is complete.**
+
+1. Build the media repo
+2. Configure the `media-repo.yaml`
+3. Run `bin/import_synapse`. The usage is below. 
+    ```
+    Usage of ./bin/import_synapse:
+      -baseUrl string
+            The base URL to access your homeserver with (default "http://localhost:8008")
+      -dbHost string
+            The IP or hostname of the postgresql server with the synapse database (default "localhost")
+      -dbName string
+            The name of the synapse database (default "synapse")
+      -dbPassword string
+            The password to authorize the postgres user. Can be omitted to be prompted when run
+      -dbPort int
+            The port to access postgres on (default 5432)
+      -dbUsername string
+            The username to access postgres with (default "synapse")
+      -serverName string
+            The name of your homeserver (eg: matrix.org) (default "localhost")
+    ```
+    Assuming the media repository, postgres database, and synapse are all on the same host, the command to run would look something like: `bin/import_synapse -serverName myserver.com -dbUsername my_database_user -dbName synapse`
+4. Wait for the import to complete. The script will automatically deduplicate media.
+5. Point traffic to the media repository

src/github.com/turt2live/matrix-media-repo/cmd/import_synapse/main.go

 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"net/http"
+	"strconv"
 
+	"github.com/howeyc/gopass"
+	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/config"
+	"github.com/turt2live/matrix-media-repo/logging"
+	"github.com/turt2live/matrix-media-repo/rcontext"
+	"github.com/turt2live/matrix-media-repo/services"
 	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/synapse"
 )
 
 func main() {
+	postgresHost := flag.String("dbHost", "localhost", "The IP or hostname of the postgresql server with the synapse database")
+	postgresPort := flag.Int("dbPort", 5432, "The port to access postgres on")
+	postgresUsername := flag.String("dbUsername", "synapse", "The username to access postgres with")
+	postgresPassword := flag.String("dbPassword", "", "The password to authorize the postgres user. Can be omitted to be prompted when run")
+	postgresDatabase := flag.String("dbName", "synapse", "The name of the synapse database")
+	baseUrl := flag.String("baseUrl", "http://localhost:8008", "The base URL to access your homeserver with")
+	serverName := flag.String("serverName", "localhost", "The name of your homeserver (eg: matrix.org)")
+	flag.Parse()
+
+	var realPsqlPassword string
+	if *postgresPassword == "" {
+		fmt.Printf("Postgres password: ")
+		pass, err := gopass.GetPasswd()
+		if err != nil {
+			panic(err)
+		}
+		realPsqlPassword = string(pass[:])
+	} else {
+		realPsqlPassword = *postgresPassword
+	}
+
 	c, err := config.ReadConfig()
 	if err != nil {
 		panic(err)
 	}
 
+	err = logging.Setup(c.General.LogDirectory)
+	if err != nil {
+		panic(err)
+	}
+
+	logrus.Info("Setting up for importing...")
+
 	db, err := storage.OpenDatabase(c.Database.Postgres)
 	if err != nil {
 		panic(err)
 	}
 
-	homeserverYamlPath := flag.String("homeserver", "homeserver.yaml", "Path to your homeserver.yaml")
-	moveFiles := flag.Bool("move", false, "If set, files will be moved instead of copied")
-	importRemote := flag.Bool("remote", false, "If set, remote media will also be imported")
-	flag.Parse()
+	connectionString := "postgres://" + *postgresUsername + ":" + realPsqlPassword + "@" + *postgresHost + ":" + strconv.Itoa(*postgresPort) + "/" + *postgresDatabase + "?sslmode=disable"
+	csApiUrl := *baseUrl
+	if csApiUrl[len(csApiUrl)-1:] == "/" {
+		csApiUrl = csApiUrl[:len(csApiUrl)-1]
+	}
+
+	logrus.Info("Connecting to synapse database...")
+	synDb, err := synapse.OpenDatabase(connectionString)
+	if err != nil {
+		panic(err)
+	}
 
-	hsConfig, err := synapse.ReadConfig(*homeserverYamlPath)
+	logrus.Info("Fetching all local media records from synapse...")
+	records, err := synDb.GetAllMedia()
 	if err != nil {
 		panic(err)
 	}
 
-	fmt.Println(*moveFiles)
-	fmt.Println(*importRemote)
-	fmt.Println(*db)
-	fmt.Println(hsConfig.GetConnectionString())
-}
\ No newline at end of file
+	logrus.Info(fmt.Sprintf("Downloading %d media records", len(records)))
+	ctx := context.TODO()
+	for i := 0; i < len(records); i++ {
+		percent := int((float32(i+1) / float32(len(records))) * 100)
+		record := records[i]
+
+		info := rcontext.RequestInfo{
+			Log: logrus.WithFields(logrus.Fields{
+				"mediaId": record.MediaId,
+			}),
+			Context: ctx,
+			Db:      *db,
+			Config:  c,
+		}
+
+		info.Log.Info(fmt.Sprintf("Downloading %s (%d/%d %d%%)", record.MediaId, i+1, len(records), percent))
+
+		body, err := downloadMedia(csApiUrl, *serverName, record.MediaId)
+		if err != nil {
+			info.Log.Error(err.Error())
+			continue
+		}
+
+		svc := services.CreateMediaService(info)
+
+		_, err = svc.StoreMedia(body, record.ContentType, record.UploadName, record.UserId, *serverName, record.MediaId)
+		if err != nil {
+			info.Log.Error(err.Error())
+			continue
+		}
+
+		body.Close()
+	}
+
+	logrus.Info("Import completed")
+}
+
+func downloadMedia(baseUrl string, serverName string, mediaId string) (io.ReadCloser, error) {
+	downloadUrl := baseUrl + "/_matrix/media/r0/download/" + serverName + "/" + mediaId
+	resp, err := http.Get(downloadUrl)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("received status code " + strconv.Itoa(resp.StatusCode))
+	}
+
+	return resp.Body, nil
+}

src/github.com/turt2live/matrix-media-repo/synapse/db.go

+package synapse
+
+import (
+	"database/sql"
+
+	_ "github.com/lib/pq" // postgres driver
+)
+
+const selectLocalMedia = "SELECT media_id, media_type, media_length, created_ts, upload_name, user_id FROM local_media_repository;"
+
+type SynapseDatabase struct {
+	db         *sql.DB
+	statements statements
+}
+
+type statements struct {
+	selectLocalMedia *sql.Stmt
+}
+
+func OpenDatabase(connectionString string) (*SynapseDatabase, error) {
+	var d SynapseDatabase
+	var err error
+
+	if d.db, err = sql.Open("postgres", connectionString); err != nil {
+		return nil, err
+	}
+
+	if d.statements.selectLocalMedia, err = d.db.Prepare(selectLocalMedia); err != nil {
+		return nil, err
+	}
+
+	return &d, nil
+}
+
+func (d *SynapseDatabase) GetAllMedia() ([]LocalMedia, error) {
+	rows, err := d.statements.selectLocalMedia.Query()
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return []LocalMedia{}, nil // no records
+		}
+		return nil, err
+	}
+
+	var results []LocalMedia
+	for rows.Next() {
+		var mediaId sql.NullString
+		var contentType sql.NullString
+		var sizeBytes sql.NullInt64
+		var createdTs sql.NullInt64
+		var uploadName sql.NullString
+		var userId sql.NullString
+		err = rows.Scan(
+			&mediaId,
+			&contentType,
+			&sizeBytes,
+			&createdTs,
+			&uploadName,
+			&userId,
+		)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, LocalMedia{
+			MediaId:     mediaId.String,
+			ContentType: contentType.String,
+			SizeBytes:   sizeBytes.Int64,
+			CreatedTs:   createdTs.Int64,
+			UploadName:  uploadName.String,
+			UserId:      userId.String,
+		})
+	}
+
+	return results, nil
+}

src/github.com/turt2live/matrix-media-repo/synapse/homeserver_yaml.go

-package synapse
-
-import (
-	"errors"
-	"io/ioutil"
-	"os"
-
-	"gopkg.in/yaml.v2"
-)
-
-type HomeserverYaml struct {
-	Database struct {
-		Name string `yaml:"name"`
-		Arguments struct {
-			Username string `yaml:"user"`
-			Password string `yaml:"password"`
-			Database string `yaml:"database"`
-			Hostname string `yaml:"host"`
-		} `yaml:"args"`
-	} `yaml:"database"`
-}
-
-func ReadConfig(yamlPath string) (HomeserverYaml, error) {
-	c := &HomeserverYaml{}
-
-	f, err := os.Open(yamlPath)
-	if err != nil {
-		return *c, err
-	}
-
-	defer f.Close()
-
-	buffer, err := ioutil.ReadAll(f)
-	err = yaml.Unmarshal(buffer, &c)
-	if err != nil {
-		return *c, err
-	}
-
-	return *c, nil
-}
-
-func (c *HomeserverYaml) GetConnectionString() (string) {
-	if c.Database.Name != "psycopg2" {
-		panic(errors.New("homeserver database must be postgres"))
-	}
-
-	a := c.Database.Arguments
-
-	return "postgres://" + a.Username + ":" + a.Password + "@" + a.Hostname + "/" + a.Database + "?sslmode=disable"
-}
\ No newline at end of file

src/github.com/turt2live/matrix-media-repo/synapse/local_media.go

+package synapse
+
+type LocalMedia struct {
+	MediaId string
+	ContentType string
+	SizeBytes int64
+	CreatedTs int64
+	UploadName string
+	UserId string
+}

vendor/manifest

@@ -38,6 +38,12 @@
 			"revision": "2d5fef06b891c971b14aa6f71ca5ab6c03a36e0e",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/howeyc/gopass",
+			"repository": "https://github.com/howeyc/gopass",
+			"revision": "bf9dde6d0d2c004a008c27aaee91170c786f6db8",
+			"branch": "master"
+		},
 		{
 			"importpath": "github.com/lestrrat/go-file-rotatelogs",
 			"repository": "https://github.com/lestrrat/go-file-rotatelogs",