Ensure we join the worker queue for URL previews

pipelines/_steps/url_preview/preview.go

@@ -2,31 +2,50 @@ package url_preview
 
 import (
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/pool"
 	"github.com/turt2live/matrix-media-repo/url_previewing/m"
 	"github.com/turt2live/matrix-media-repo/url_previewing/p"
 )
 
+type generateResult struct {
+	preview m.PreviewResult
+	err     error
+}
+
 func Preview(ctx rcontext.RequestContext, targetUrl *m.UrlPayload, languageHeader string) (m.PreviewResult, error) {
-	var preview m.PreviewResult
-	err := m.ErrPreviewUnsupported
+	ch := make(chan generateResult)
+	defer close(ch)
+	fn := func() {
+		var preview m.PreviewResult
+		err := m.ErrPreviewUnsupported
 
-	// Try oEmbed first
-	if ctx.Config.UrlPreviews.OEmbed {
-		ctx.Log.Debug("Trying oEmbed previewer")
-		preview, err = p.GenerateOEmbedPreview(targetUrl, languageHeader, ctx)
-	}
+		// Try oEmbed first
+		if ctx.Config.UrlPreviews.OEmbed {
+			ctx.Log.Debug("Trying oEmbed previewer")
+			preview, err = p.GenerateOEmbedPreview(targetUrl, languageHeader, ctx)
+		}
 
-	// Try OpenGraph if that failed
-	if err == m.ErrPreviewUnsupported {
-		ctx.Log.Debug("Trying OpenGraph previewer")
-		preview, err = p.GenerateOpenGraphPreview(targetUrl, languageHeader, ctx)
-	}
+		// Try OpenGraph if that failed
+		if err == m.ErrPreviewUnsupported {
+			ctx.Log.Debug("Trying OpenGraph previewer")
+			preview, err = p.GenerateOpenGraphPreview(targetUrl, languageHeader, ctx)
+		}
+
+		// Try scraping if that failed
+		if err == m.ErrPreviewUnsupported {
+			ctx.Log.Debug("Trying built-in previewer")
+			preview, err = p.GenerateCalculatedPreview(targetUrl, languageHeader, ctx)
+		}
 
-	// Try scraping if that failed
-	if err == m.ErrPreviewUnsupported {
-		ctx.Log.Debug("Trying built-in previewer")
-		preview, err = p.GenerateCalculatedPreview(targetUrl, languageHeader, ctx)
+		ch <- generateResult{
+			preview: preview,
+			err:     err,
+		}
 	}
 
-	return preview, err
+	if err := pool.UrlPreviewQueue.Schedule(fn); err != nil {
+		return m.PreviewResult{}, err
+	}
+	res := <-ch
+	return res.preview, res.err
 }

pool/init.go

@@ -8,6 +8,7 @@ import (
 
 var DownloadQueue *Queue
 var ThumbnailQueue *Queue
+var UrlPreviewQueue *Queue
 
 func Init() {
 	var err error
@@ -21,14 +22,21 @@ func Init() {
 		logrus.Error("Error setting up thumbnails queue")
 		logrus.Fatal(err)
 	}
+	if UrlPreviewQueue, err = NewQueue(config.Get().UrlPreviews.NumWorkers, "url_previews"); err != nil {
+		sentry.CaptureException(err)
+		logrus.Error("Error setting up url previews queue")
+		logrus.Fatal(err)
+	}
 }
 
 func AdjustSize() {
 	DownloadQueue.pool.Tune(config.Get().Downloads.NumWorkers)
 	ThumbnailQueue.pool.Tune(config.Get().Thumbnails.NumWorkers)
+	UrlPreviewQueue.pool.Tune(config.Get().UrlPreviews.NumWorkers)
 }
 
 func Drain() {
 	DownloadQueue.pool.Release()
 	ThumbnailQueue.pool.Release()
+	UrlPreviewQueue.pool.Release()
 }