From bf4fe206a677a064fff1ad9c478862a02b825869 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sat, 29 Feb 2020 18:40:55 -0700
Subject: [PATCH] Add blurhash calculations
---
api/custom/version.go | 5 +-
api/r0/upload.go | 12 +++-
api/webserver/webserver.go | 3 +
.../info_controller/info_controller.go | 68 +++++++++++++++++++
go.mod | 1 +
go.sum | 2 +
migrations/14_add_blurhash_tables_down.sql | 1 +
migrations/14_add_blurhash_tables_up.sql | 4 ++
storage/stores/metadata_store.go | 32 +++++++++
9 files changed, 126 insertions(+), 2 deletions(-)
create mode 100644 controllers/info_controller/info_controller.go
create mode 100644 migrations/14_add_blurhash_tables_down.sql
create mode 100644 migrations/14_add_blurhash_tables_up.sql
diff --git a/api/custom/version.go b/api/custom/version.go
index 5715f78e..85b79695 100644
--- a/api/custom/version.go
+++ b/api/custom/version.go
@@ -10,9 +10,12 @@ import (
func GetVersion(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
return &api.DoNotCacheResponse{
- Payload: map[string]string{
+ Payload: map[string]interface{}{
"Version": version.Version,
"GitCommit": version.GitCommit,
+ "unstable_features": []string{
+ "xyz.amorgan.blurhash",
+ },
},
}
}
diff --git a/api/r0/upload.go b/api/r0/upload.go
index 9b1fd744..4b669271 100644
--- a/api/r0/upload.go
+++ b/api/r0/upload.go
@@ -10,11 +10,13 @@ import (
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/controllers/info_controller"
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
)
type MediaUploadedResponse struct {
ContentUri string `json:"content_uri"`
+ Blurhash string `json:"blurhash"`
}
func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
@@ -56,5 +58,13 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf
return api.InternalServerError("Unexpected Error")
}
- return &MediaUploadedResponse{media.MxcUri()}
+ hash, err := info_controller.GetOrCalculateBlurhash(media, rctx)
+ if err != nil {
+ rctx.Log.Warn("Failed to calculate blurhash: " + err.Error())
+ }
+
+ return &MediaUploadedResponse{
+ ContentUri: media.MxcUri(),
+ Blurhash: hash,
+ }
}
diff --git a/api/webserver/webserver.go b/api/webserver/webserver.go
index c22aeb72..86b5c3d3 100644
--- a/api/webserver/webserver.go
+++ b/api/webserver/webserver.go
@@ -134,6 +134,9 @@ func Init() *sync.WaitGroup {
routes["/_matrix/media/"+version+"/local_copy/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"GET", localCopyHandler}
routes["/_matrix/media/"+version+"/info/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"GET", infoHandler}
routes["/_matrix/media/"+version+"/download/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"DELETE", purgeOneHandler}
+
+ // MSC2448: Blurhash
+ routes["/_matrix/media/"+version+"/xyz.amorgan/upload"] = route{"POST", uploadHandler}
}
}
diff --git a/controllers/info_controller/info_controller.go b/controllers/info_controller/info_controller.go
new file mode 100644
index 00000000..223fe325
--- /dev/null
+++ b/controllers/info_controller/info_controller.go
@@ -0,0 +1,68 @@
+package info_controller
+
+import (
+ "bytes"
+ "image/png"
+
+ "github.com/buckket/go-blurhash"
+ "github.com/disintegration/imaging"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/controllers/download_controller"
+ "github.com/turt2live/matrix-media-repo/storage"
+ "github.com/turt2live/matrix-media-repo/types"
+)
+
+func GetOrCalculateBlurhash(media *types.Media, rctx rcontext.RequestContext) (string, error) {
+ rctx.Log.Info("Attempting fetch of blurhash for sha256 of " + media.Sha256Hash)
+ db := storage.GetDatabase().GetMetadataStore(rctx)
+ cached, err := db.GetBlurhash(media.Sha256Hash)
+ if err != nil {
+ return "", err
+ }
+
+ if cached != "" {
+ rctx.Log.Info("Returning cached blurhash: " + cached)
+ return cached, nil
+ }
+
+ rctx.Log.Info("Getting minimal media record to calculate blurhash")
+ minMedia, err := download_controller.FindMinimalMediaRecord(media.Origin, media.MediaId, true, rctx)
+ if err != nil {
+ return "", err
+ }
+
+ // No cached blurhash: calculate one
+ rctx.Log.Info("Decoding image for blurhash calculation")
+ imgSrc, err := imaging.Decode(minMedia.Stream)
+ if err != nil {
+ return "", err
+ }
+
+ // Resize the image to make the blurhash a bit more reasonable to calculate
+ rctx.Log.Info("Resizing image for blurhash (faster calculation)")
+ smallImg := imaging.Fill(imgSrc, 128, 128, imaging.Center, imaging.Lanczos)
+ imgBuf := &bytes.Buffer{}
+ err = imaging.Encode(imgBuf, smallImg, imaging.PNG)
+ if err != nil {
+ return "", err
+ }
+ decoded, err := png.Decode(imgBuf)
+ if err != nil {
+ return "", err
+ }
+
+ rctx.Log.Info("Calculating blurhash")
+ encoded, err := blurhash.Encode(4, 3, &decoded)
+ if err != nil {
+ return "", err
+ }
+
+ // Save the blurhash for next time
+ rctx.Log.Infof("Saving blurhash %s and returning", encoded)
+ err = db.InsertBlurhash(media.Sha256Hash, encoded)
+ if err != nil {
+ return "", err
+ }
+
+ return encoded, nil
+}
diff --git a/go.mod b/go.mod
index 34aa48f1..636957d2 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
github.com/alioygur/is v0.0.0-20170213121024-204f48747743
github.com/andybalholm/cascadia v0.0.0-20161224141413-349dd0209470 // indirect
github.com/bep/debounce v1.2.0
+ github.com/buckket/go-blurhash v1.0.3
github.com/cenk/backoff v2.0.0+incompatible // indirect
github.com/cupcake/sigil v0.0.0-20131127230922-6bf9722f2ae8
github.com/didip/tollbooth v4.0.0+incompatible
diff --git a/go.sum b/go.sum
index cd8ded22..9fc25f48 100644
--- a/go.sum
+++ b/go.sum
@@ -16,6 +16,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
+github.com/buckket/go-blurhash v1.0.3 h1:zCSPYlKYWxF+3I/JJT2GrF4ut6wRaifz89JdsdZClpw=
+github.com/buckket/go-blurhash v1.0.3/go.mod h1:BUt9nlD6V+23blJqm6Vn/423xpTnP1OLA9yv+y4l44U=
github.com/cenk/backoff v2.0.0+incompatible h1:7vXVw3g7XE+Vnj0A9TmFGtMeP4oZQ5ZzpPvKhLFa80E=
github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE=
github.com/cupcake/sigil v0.0.0-20131127230922-6bf9722f2ae8 h1:OPuOoDEMJx86BQOPt4rfZvOjquI3Ym3XUE1Dy+gQoVs=
diff --git a/migrations/14_add_blurhash_tables_down.sql b/migrations/14_add_blurhash_tables_down.sql
new file mode 100644
index 00000000..e0eac9d3
--- /dev/null
+++ b/migrations/14_add_blurhash_tables_down.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS blurhashes;
diff --git a/migrations/14_add_blurhash_tables_up.sql b/migrations/14_add_blurhash_tables_up.sql
new file mode 100644
index 00000000..1ff24109
--- /dev/null
+++ b/migrations/14_add_blurhash_tables_up.sql
@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS blurhashes (
+ sha256_hash TEXT PRIMARY KEY NOT NULL,
+ blurhash TEXT NOT NULL
+);
diff --git a/storage/stores/metadata_store.go b/storage/stores/metadata_store.go
index a47d30dc..28c5f712 100644
--- a/storage/stores/metadata_store.go
+++ b/storage/stores/metadata_store.go
@@ -29,6 +29,8 @@ const selectAllBackgroundTasks = "SELECT id, task, params, start_ts, end_ts FROM
const insertReservation = "INSERT INTO reserved_media (origin, media_id, reason) VALUES ($1, $2, $3);"
const selectReservation = "SELECT origin, media_id, reason FROM reserved_media WHERE origin = $1 AND media_id = $2;"
const selectMediaLastAccessed = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM media AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1;"
+const insertBlurhash = "INSERT INTO blurhashes (sha256_hash, blurhash) VALUES ($1, $2);"
+const selectBlurhash = "SELECT blurhash FROM blurhashes WHERE sha256_hash = $1;"
type metadataStoreStatements struct {
upsertLastAccessed *sql.Stmt
@@ -47,6 +49,8 @@ type metadataStoreStatements struct {
insertReservation *sql.Stmt
selectReservation *sql.Stmt
selectMediaLastAccessed *sql.Stmt
+ insertBlurhash *sql.Stmt
+ selectBlurhash *sql.Stmt
}
type MetadataStoreFactory struct {
@@ -114,6 +118,12 @@ func InitMetadataStore(sqlDb *sql.DB) (*MetadataStoreFactory, error) {
if store.stmts.selectMediaLastAccessed, err = store.sqlDb.Prepare(selectMediaLastAccessed); err != nil {
return nil, err
}
+ if store.stmts.insertBlurhash, err = store.sqlDb.Prepare(insertBlurhash); err != nil {
+ return nil, err
+ }
+ if store.stmts.selectBlurhash, err = store.sqlDb.Prepare(selectBlurhash); err != nil {
+ return nil, err
+ }
return &store, nil
}
@@ -376,3 +386,25 @@ func (s *MetadataStore) IsReserved(origin string, mediaId string) (bool, error)
}
return true, nil
}
+
+func (s *MetadataStore) InsertBlurhash(sha256Hash string, blurhash string) error {
+ _, err := s.statements.insertBlurhash.ExecContext(s.ctx, sha256Hash, blurhash)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *MetadataStore) GetBlurhash(sha256Hash string) (string, error) {
+ r := s.statements.selectBlurhash.QueryRowContext(s.ctx, sha256Hash)
+ var blurhash string
+
+ err := r.Scan(&blurhash)
+ if err == sql.ErrNoRows {
+ return "", nil
+ }
+ if err != nil {
+ return "", err
+ }
+ return blurhash, nil
+}
--
GitLab