Support a config flag for URL previews on hosts with invalid certificates

Required for sytests to work

config.sample.yaml

@@ -97,6 +97,10 @@ urlPreviews:
   enabled: true # If enabled, the preview_url routes will be accessible
   maxPageSizeBytes: 10485760 # 10MB default, 0 to disable
 
+  # If true, the media repository will try to provide previews for URLs with invalid or unsafe
+  # certificates. If false (the default), the media repo will fail requests to said URLs.
+  previewUnsafeCertificates: false
+
   # Note: URL previews are limited to a given number of words, which are then limited to a number
   # of characters, taking off the last word if it needs to. This also applies for the title.
 

src/github.com/turt2live/matrix-media-repo/common/config/config.go

@@ -16,9 +16,9 @@ type runtimeConfig struct {
 var Runtime = &runtimeConfig{}
 
 type HomeserverConfig struct {
-	Name                 string `yaml:"name"`
-	ClientServerApi      string `yaml:"csApi"`
-	BackoffAt            int    `yaml:"backoffAt"`
+	Name            string `yaml:"name"`
+	ClientServerApi string `yaml:"csApi"`
+	BackoffAt       int    `yaml:"backoffAt"`
 }
 
 type GeneralConfig struct {
@@ -71,6 +71,7 @@ type UrlPreviewsConfig struct {
 	FilePreviewTypes   []string `yaml:"filePreviewTypes,flow"`
 	DisallowedNetworks []string `yaml:"disallowedNetworks,flow"`
 	AllowedNetworks    []string `yaml:"allowedNetworks,flow"`
+	UnsafeCertificates bool     `yaml:"previewUnsafeCertificates"`
 }
 
 type RateLimitConfig struct {

src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/previewers/opengraph_previewer.go

 package previewers
 
 import (
+	"crypto/tls"
 	"errors"
 	"io"
 	"io/ioutil"
 	"mime"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -89,9 +91,43 @@ func GenerateOpenGraphPreview(urlStr string, log *logrus.Entry) (PreviewResult,
 	return *graph, nil
 }
 
+func doHttpGet(urlStr string, log *logrus.Entry) (*http.Response, error) {
+	var resp *http.Response
+	var err error
+	if config.Get().UrlPreviews.UnsafeCertificates {
+		log.Warn("Ignoring any certificate errors while making request")
+		tr := &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			// Based on https://github.com/matrix-org/gomatrixserverlib/blob/51152a681e69a832efcd934b60080b92bc98b286/client.go#L74-L90
+			DialTLS: func(network, addr string) (net.Conn, error) {
+				rawconn, err := net.Dial(network, addr)
+				if err != nil {
+					return nil, err
+				}
+				// Wrap a raw connection ourselves since tls.Dial defaults the SNI
+				conn := tls.Client(rawconn, &tls.Config{
+					ServerName: "",
+					// TODO: We should be checking that the TLS certificate we see here matches one of the allowed SHA-256 fingerprints for the server.
+					InsecureSkipVerify: true,
+				})
+				if err := conn.Handshake(); err != nil {
+					return nil, err
+				}
+				return conn, nil
+			},
+		}
+		client := &http.Client{Transport: tr}
+		resp, err = client.Get(urlStr)
+	} else {
+		resp, err = http.Get(urlStr)
+	}
+
+	return resp, err
+}
+
 func downloadHtmlContent(urlStr string, log *logrus.Entry) (string, error) {
 	log.Info("Fetching remote content...")
-	resp, err := http.Get(urlStr)
+	resp, err := doHttpGet(urlStr, log)
 	if err != nil {
 		return "", err
 	}
@@ -130,7 +166,7 @@ func downloadHtmlContent(urlStr string, log *logrus.Entry) (string, error) {
 
 func downloadImage(imageUrl string, log *logrus.Entry) (*PreviewImage, error) {
 	log.Info("Getting image from " + imageUrl)
-	resp, err := http.Get(imageUrl)
+	resp, err := doHttpGet(imageUrl, log)
 	if err != nil {
 		return nil, err
 	}