From a6a2d59ed96cd36e60eb4748e00de66c45b45698 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sat, 4 Jan 2020 16:31:19 -0700
Subject: [PATCH] Compile assets into the binary
Fixes https://github.com/turt2live/matrix-media-repo/issues/214
---
.gitignore | 3 ++
CHANGELOG.md | 4 ++
README.md | 8 ++--
build.ps1 | 2 +
build.sh | 2 +
cmd/compile_assets/main.go | 78 +++++++++++++++++++++++++++++++++++++
cmd/import_synapse/main.go | 6 ++-
cmd/media_repo/main.go | 11 ++++--
common/assets/process.go | 79 ++++++++++++++++++++++++++++++++++++++
common/config/access.go | 3 ++
10 files changed, 188 insertions(+), 8 deletions(-)
create mode 100644 cmd/compile_assets/main.go
create mode 100644 common/assets/process.go
diff --git a/.gitignore b/.gitignore
index ecdfdf7b..17100f70 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@
/vendor
/config
+# Generated files
+assets.bin.go
+
media-repo*.yaml
homeserver.yaml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf0e4d40..bc2c80af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
## [Unreleased]
+### Added
+
+* Compile assets (templates and migrations) into the binary for ease of deployment.
+
### Fixed
* Fix error message when an invalid access token is provided.
diff --git a/README.md b/README.md
index 551482b4..c7d209dc 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,9 @@ for your environment.
For help and support, visit [#mediarepo:t2bot.io](https://matrix.to/#/#mediarepo:t2bot.io).
-# Installing / building
+## Installing / building
+
+**Note**: developing on matrix-media-repo requires you to follow these steps at least once.
Assuming Go 1.12+ is already installed on your PATH:
```bash
@@ -46,7 +48,7 @@ docker run --rm -it -p 8000:8000 -v /etc/matrix-media-repo:/data turt2live/matri
Note that using `latest` is dangerous - it is effectively the development branch for this project. Instead,
prefer to use one of the tagged versions and update regularly.
-# Deployment
+## Deployment
This is intended to run behind a load balancer and beside your homeserver deployments. Assuming your load balancer handles SSL termination, a sample nginx config would be:
@@ -97,7 +99,7 @@ listeners:
After importing your media, setting `enable_media_repo: false` in your Synapse configuration will disable the media repository.
-# Importing media from synapse
+## Importing media from synapse
Media is imported by connecting to your synapse database and downloading all the content from the homeserver. This is so
you have a backup of the media repository still with synapse. **Do not point traffic at the media repo until after the
diff --git a/build.ps1 b/build.ps1
index be245b52..911ffc0a 100644
--- a/build.ps1
+++ b/build.ps1
@@ -1,3 +1,5 @@
$GitCommit = (git rev-list -1 HEAD)
$Version = (git describe --tags)
+go install -v ./cmd/compile_assets
+compile_assets
go install -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$GitCommit -X github.com/turt2live/matrix-media-repo/common/version.Version=$Version" -v ./cmd/...
diff --git a/build.sh b/build.sh
index 15d83fd7..bfb787f4 100755
--- a/build.sh
+++ b/build.sh
@@ -1,3 +1,5 @@
#!/bin/sh
+GOBIN=$PWD/bin go install -v ./cmd/compile_assets
+compile_assets
GOBIN=$PWD/bin go install -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(git rev-list -1 HEAD) -X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
diff --git a/cmd/compile_assets/main.go b/cmd/compile_assets/main.go
new file mode 100644
index 00000000..bd654b0f
--- /dev/null
+++ b/cmd/compile_assets/main.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "bytes"
+ "compress/gzip"
+ "encoding/hex"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "path"
+
+ "github.com/turt2live/matrix-media-repo/common/config"
+)
+
+func main() {
+ migrationsPath := flag.String("migrations", config.DefaultMigrationsPath, "The absolute path for the migrations folder")
+ templatesPath := flag.String("templates", config.DefaultTemplatesPath, "The absolute path for the templates folder")
+ outputFile := flag.String("output", "./common/assets/assets.bin.go", "The output Go file to dump the files into")
+ flag.Parse()
+
+ fmt.Println("Reading migrations into memory...")
+ migrations := readDir(*migrationsPath, "migrations")
+ templates := readDir(*templatesPath, "templates")
+
+ fileMap := make(map[string][]byte)
+ for k, v := range migrations {
+ fileMap[k] = v
+ }
+ for k, v := range templates {
+ fileMap[k] = v
+ }
+
+ fmt.Println("Writing assets go file")
+ str := "package " + path.Base(path.Dir(*outputFile)) + "\n\n"
+ str += "// !! THIS FILE IS AUTOMATICALLY GENERATED. You can edit it, but it will be overwritten over time.\n\n"
+ str += "var compressedFiles = map[string]string{\n"
+ for f, b := range fileMap {
+ b64 := hex.EncodeToString(b)
+ str += fmt.Sprintf("\t\"%s\": \"%s\",\n", f, b64)
+ }
+ str += "}\n"
+ err := ioutil.WriteFile(*outputFile, []byte(str), 644)
+ if err != nil {
+ panic(err)
+ }
+
+ fmt.Println("Done")
+}
+
+func readDir(dir string, pathName string) map[string][]byte {
+ fileMap := make(map[string][]byte)
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ panic(err)
+ }
+ for _, f := range files {
+ fname := path.Join(dir, f.Name())
+ fmt.Println("Reading ", fname)
+
+ b, err := ioutil.ReadFile(fname)
+ if err != nil {
+ panic(err)
+ }
+
+ // Compress the file
+ fmt.Println("Compressing ", fname)
+ out := &bytes.Buffer{}
+ gw, err := gzip.NewWriterLevel(out, gzip.BestCompression)
+ if err != nil {
+ panic(err)
+ }
+ gw.Write(b)
+ gw.Close()
+
+ fileMap[path.Join(pathName, f.Name())] = out.Bytes()
+ }
+ return fileMap
+}
diff --git a/cmd/import_synapse/main.go b/cmd/import_synapse/main.go
index 651640cc..b1f6b77d 100644
--- a/cmd/import_synapse/main.go
+++ b/cmd/import_synapse/main.go
@@ -14,6 +14,7 @@ import (
"github.com/jeffail/tunny"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/common/assets"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/logging"
"github.com/turt2live/matrix-media-repo/common/rcontext"
@@ -43,7 +44,7 @@ func main() {
flag.Parse()
config.Path = *configPath
- config.Runtime.MigrationsPath = *migrationsPath
+ assets.SetupTemplatesAndMigrations(*migrationsPath, "")
var realPsqlPassword string
if *postgresPassword == "" {
@@ -115,6 +116,9 @@ func main() {
time.Sleep(1 * time.Second)
}
+ // Clean up
+ assets.Cleanup()
+
logrus.Info("Import completed")
}
diff --git a/cmd/media_repo/main.go b/cmd/media_repo/main.go
index 43207c35..d26fc9b6 100644
--- a/cmd/media_repo/main.go
+++ b/cmd/media_repo/main.go
@@ -7,6 +7,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api/webserver"
+ "github.com/turt2live/matrix-media-repo/common/assets"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/logging"
"github.com/turt2live/matrix-media-repo/common/runtime"
@@ -17,8 +18,8 @@ import (
func main() {
configPath := flag.String("config", "media-repo.yaml", "The path to the configuration")
- migrationsPath := flag.String("migrations", "./migrations", "The absolute path for the migrations folder")
- templatesPath := flag.String("templates", "./templates", "The absolute path for the templates folder")
+ migrationsPath := flag.String("migrations", config.DefaultMigrationsPath, "The absolute path for the migrations folder")
+ templatesPath := flag.String("templates", config.DefaultTemplatesPath, "The absolute path for the templates folder")
versionFlag := flag.Bool("version", false, "Prints the version and exits")
flag.Parse()
@@ -34,8 +35,7 @@ func main() {
}
config.Path = *configPath
- config.Runtime.MigrationsPath = *migrationsPath
- config.Runtime.TemplatesPath = *templatesPath
+ assets.SetupTemplatesAndMigrations(*migrationsPath, *templatesPath)
err := logging.Setup(config.Get().General.LogDirectory)
if err != nil {
@@ -99,6 +99,9 @@ func main() {
stopAllButWeb()
}
+ // Clean up
+ assets.Cleanup()
+
// For debugging
logrus.Info("Goodbye!")
}
diff --git a/common/assets/process.go b/common/assets/process.go
new file mode 100644
index 00000000..93a473f7
--- /dev/null
+++ b/common/assets/process.go
@@ -0,0 +1,79 @@
+package assets
+
+import (
+ "encoding/hex"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/common/config"
+)
+
+var tempMigrations string
+var tempTemplates string
+
+func SetupTemplatesAndMigrations(givenMigrationsPath string, givenTemplatesPath string) {
+ _, err := os.Stat(givenMigrationsPath)
+ exists := err == nil || !os.IsNotExist(err)
+ if !exists {
+ tempMigrations, err = ioutil.TempDir(os.TempDir(), "media-repo-migrations")
+ if err != nil {
+ panic(err)
+ }
+ logrus.Info("Migrations path doesn't exist - attempting to unpack from compiled data")
+ extractPrefixTo("migrations", tempMigrations)
+ givenMigrationsPath = tempMigrations
+ }
+
+ if givenTemplatesPath != "" {
+ _, err = os.Stat(givenTemplatesPath)
+ exists = err == nil || !os.IsNotExist(err)
+ if !exists {
+ tempTemplates, err = ioutil.TempDir(os.TempDir(), "media-repo-templates")
+ if err != nil {
+ panic(err)
+ }
+ logrus.Info("Templates path doesn't exist - attempting to unpack from compiled data")
+ extractPrefixTo("templates", tempTemplates)
+ givenTemplatesPath = tempTemplates
+ }
+ }
+
+ config.Runtime.MigrationsPath = givenMigrationsPath
+ config.Runtime.TemplatesPath = givenTemplatesPath
+}
+
+func Cleanup() {
+ if tempMigrations != "" {
+ logrus.Info("Cleaning up temporary assets directory: ", tempMigrations)
+ os.Remove(tempMigrations)
+ }
+ if tempTemplates != "" {
+ logrus.Info("Cleaning up temporary assets directory: ", tempTemplates)
+ os.Remove(tempTemplates)
+ }
+}
+
+func extractPrefixTo(pathName string, destination string) {
+ for f, h := range compressedFiles {
+ if !strings.HasPrefix(f, pathName) {
+ continue
+ }
+
+ logrus.Infof("Decoding %s", f)
+ b, err := hex.DecodeString(h)
+ if err != nil {
+ panic(err)
+ }
+
+ dest := path.Join(destination, filepath.Base(f))
+ logrus.Infof("Writing %s to %s", f, dest)
+ err = ioutil.WriteFile(dest, b, 644)
+ if err != nil {
+ panic(err)
+ }
+ }
+}
diff --git a/common/config/access.go b/common/config/access.go
index 7f28be2c..69b6cbf2 100644
--- a/common/config/access.go
+++ b/common/config/access.go
@@ -17,6 +17,9 @@ type runtimeConfig struct {
TemplatesPath string
}
+const DefaultMigrationsPath = "./migrations"
+const DefaultTemplatesPath = "./templates"
+
var Runtime = &runtimeConfig{}
var Path = "media-repo.yaml"
--
GitLab