From f1986162160f631618f97b207ec83b5f45c27bf4 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sat, 13 Jan 2018 19:58:35 -0700
Subject: [PATCH] Add a resource handler for url previews
Adds #13
---
.../services/url_service/resource_handler.go | 130 ++++++++++++++++++
.../services/url_service/url_service.go | 60 +-------
2 files changed, 132 insertions(+), 58 deletions(-)
create mode 100644 src/github.com/turt2live/matrix-media-repo/services/url_service/resource_handler.go
diff --git a/src/github.com/turt2live/matrix-media-repo/services/url_service/resource_handler.go b/src/github.com/turt2live/matrix-media-repo/services/url_service/resource_handler.go
new file mode 100644
index 00000000..c907a9b8
--- /dev/null
+++ b/src/github.com/turt2live/matrix-media-repo/services/url_service/resource_handler.go
@@ -0,0 +1,130 @@
+package url_service
+
+import (
+ "context"
+ "fmt"
+ "sync"
+
+ "github.com/disintegration/imaging"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/config"
+ "github.com/turt2live/matrix-media-repo/resource_handler"
+ "github.com/turt2live/matrix-media-repo/services/media_service"
+ "github.com/turt2live/matrix-media-repo/types"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/errs"
+)
+
+type urlResourceHandler struct {
+ resourceHandler *resource_handler.ResourceHandler
+}
+
+type urlPreviewRequest struct {
+ urlStr string
+ forUserId string
+ onHost string
+}
+
+type urlPreviewResponse struct {
+ preview *types.UrlPreview
+ err error
+}
+
+var resHandlerInstance *urlResourceHandler
+var resHandlerSingletonLock = &sync.Once{}
+
+func getResourceHandler() (*urlResourceHandler) {
+ if resHandlerInstance == nil {
+ resHandlerSingletonLock.Do(func() {
+ handler, err := resource_handler.New(config.Get().UrlPreviews.NumWorkers, urlPreviewWorkFn)
+ if err != nil {
+ panic(err)
+ }
+
+ resHandlerInstance = &urlResourceHandler{handler}
+ })
+ }
+
+ return resHandlerInstance
+}
+
+func urlPreviewWorkFn(request *resource_handler.WorkRequest) interface{} {
+ info := request.Metadata.(*urlPreviewRequest)
+ log := logrus.WithFields(logrus.Fields{
+ "worker_requestId": request.Id,
+ "worker_url": info.urlStr,
+ "worker_previewer": "OpenGraph",
+ })
+ log.Info("Processing url preview request")
+
+ ctx := context.TODO() // TODO: Should we use a real context?
+
+ svc := New(ctx, log) // url_service (us)
+ previewer := NewOpenGraphPreviewer(ctx, log)
+ preview, err := previewer.GeneratePreview(info.urlStr)
+ if err != nil {
+ if err == errs.ErrMediaNotFound {
+ svc.store.InsertPreviewError(info.urlStr, errs.ErrCodeNotFound)
+ } else {
+ svc.store.InsertPreviewError(info.urlStr, errs.ErrCodeUnknown)
+ }
+ return &urlPreviewResponse{err:err}
+ }
+
+ result := &types.UrlPreview{
+ Url: preview.Url,
+ SiteName: preview.SiteName,
+ Type: preview.Type,
+ Description: preview.Description,
+ Title: preview.Title,
+ }
+
+ // Store the thumbnail, if there is one
+ mediaSvc := media_service.New(ctx, log)
+ if preview.Image != nil && !mediaSvc.IsTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader) {
+ // UploadMedia will close the read stream for the thumbnail
+ media, err := mediaSvc.UploadMedia(preview.Image.Data, preview.Image.ContentType, preview.Image.Filename, info.forUserId, info.onHost)
+ if err != nil {
+ log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
+ } else {
+ img, err := imaging.Open(media.Location)
+ if err != nil {
+ log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
+ } else {
+ result.ImageMxc = media.MxcUri()
+ result.ImageType = media.ContentType
+ result.ImageSize = media.SizeBytes
+ result.ImageWidth = img.Bounds().Max.X
+ result.ImageHeight = img.Bounds().Max.Y
+ }
+ }
+ }
+
+ dbRecord := &types.CachedUrlPreview{
+ Preview: result,
+ SearchUrl: info.urlStr,
+ ErrorCode: "",
+ FetchedTs: util.NowMillis(),
+ }
+ err = svc.store.InsertPreview(dbRecord)
+ if err != nil {
+ log.Warn("Error caching URL preview: " + err.Error())
+ // Non-fatal: Just report it and move on. The worst that happens is we re-cache it.
+ }
+
+ return &urlPreviewResponse{preview:result}
+}
+
+func (h *urlResourceHandler) GeneratePreview(urlStr string, forUserId string, onHost string) chan *urlPreviewResponse {
+ resultChan := make(chan *urlPreviewResponse)
+ go func() {
+ reqId := fmt.Sprintf("preview_%s", urlStr) // don't put the user id or host in the ID string
+ result := <-h.resourceHandler.GetResource(reqId, &urlPreviewRequest{
+ urlStr:urlStr,
+ forUserId:forUserId,
+ onHost:onHost,
+ })
+ resultChan <- result.(*urlPreviewResponse)
+ }()
+ return resultChan
+}
diff --git a/src/github.com/turt2live/matrix-media-repo/services/url_service/url_service.go b/src/github.com/turt2live/matrix-media-repo/services/url_service/url_service.go
index 40844aca..889dde98 100644
--- a/src/github.com/turt2live/matrix-media-repo/services/url_service/url_service.go
+++ b/src/github.com/turt2live/matrix-media-repo/services/url_service/url_service.go
@@ -7,11 +7,9 @@ import (
"net"
"net/url"
- "github.com/disintegration/imaging"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/config"
- "github.com/turt2live/matrix-media-repo/services/media_service"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/stores"
"github.com/turt2live/matrix-media-repo/types"
@@ -106,62 +104,8 @@ func (s *urlService) GetPreview(urlStr string, onHost string, forUserId string,
return nil, errs.ErrHostBlacklisted
}
- s.log = s.log.WithFields(logrus.Fields{
- "previewer": "OpenGraph",
- })
-
- previewer := NewOpenGraphPreviewer(s.ctx, s.log)
- preview, err := previewer.GeneratePreview(urlStr)
- if err != nil {
- if err == errs.ErrMediaNotFound {
- s.store.InsertPreviewError(urlStr, errs.ErrCodeNotFound)
- } else {
- s.store.InsertPreviewError(urlStr, errs.ErrCodeUnknown)
- }
- return nil, err
- }
-
- result := &types.UrlPreview{
- Url: preview.Url,
- SiteName: preview.SiteName,
- Type: preview.Type,
- Description: preview.Description,
- Title: preview.Title,
- }
-
- // Store the thumbnail, if there is one
- mediaSvc := media_service.New(s.ctx, s.log)
- if preview.Image != nil && !mediaSvc.IsTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader) {
- // UploadMedia will close the read stream for the thumbnail
- media, err := mediaSvc.UploadMedia(preview.Image.Data, preview.Image.ContentType, preview.Image.Filename, forUserId, onHost)
- if err != nil {
- s.log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
- } else {
- img, err := imaging.Open(media.Location)
- if err != nil {
- s.log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
- } else {
- result.ImageMxc = media.MxcUri()
- result.ImageType = media.ContentType
- result.ImageSize = media.SizeBytes
- result.ImageWidth = img.Bounds().Max.X
- result.ImageHeight = img.Bounds().Max.Y
- }
- }
- }
-
- dbRecord := &types.CachedUrlPreview{
- Preview: result,
- SearchUrl: urlStr,
- ErrorCode: "",
- FetchedTs: util.NowMillis(),
- }
- err = s.store.InsertPreview(dbRecord)
- if err != nil {
- s.log.Warn("Error caching URL preview: " + err.Error())
- }
-
- return result, nil
+ result := <-getResourceHandler().GeneratePreview(urlStr, forUserId, onHost)
+ return result.preview, result.err
}
func isAllowed(ip net.IP, allowed []string, disallowed []string, log *logrus.Entry) bool {
--
GitLab