diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index f4ab6bba362dda60afe13945b610db1026943694..0000000000000000000000000000000000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-version: 2
-jobs:
- build:
- docker:
- - image: circleci/golang:1.14
- environment:
- GO111MODULE: "on"
- working_directory: /go/src/github.com/turt2live/matrix-media-repo
- steps:
- - checkout
- - run:
- name: build binaries
- command: './build.sh'
- - store_artifacts:
- path: bin/media_repo
- destination: media_repo
- - store_artifacts:
- path: bin/import_synapse
- destination: import_synapse
- - store_artifacts:
- path: bin/gdpr_export
- destination: gdpr_export
- - store_artifacts:
- path: bin/gdpr_import
- destination: gdpr_import
-workflows:
- version: 2
- build_and_test:
- jobs:
- - build
diff --git a/Complement.Dockerfile b/Complement.Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..e525d8c80d9191ceae9bfcf7ade90a678b0b8b9c
--- /dev/null
+++ b/Complement.Dockerfile
@@ -0,0 +1,41 @@
+# ---- Stage 0 ----
+# Builds media repo binaries
+FROM golang:1.14-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git musl-dev dos2unix build-base
+
+WORKDIR /opt
+COPY . /opt
+RUN dos2unix ./build.sh
+RUN ./build.sh
+
+# ---- Stage 1 ----
+# Final runtime stage.
+FROM alpine
+
+COPY --from=builder /opt/bin/media_repo /opt/bin/complement_hs /usr/local/bin/
+
+RUN apk add --no-cache ca-certificates postgresql openssl dos2unix
+
+RUN mkdir -p /data/media
+COPY ./docker/complement.yaml /data/media-repo.yaml
+ENV REPO_CONFIG=/data/media-repo.yaml
+ENV SERVER_NAME=localhost
+ENV PGDATA=/data/pgdata
+ENV MEDIA_REPO_UNSAFE_FEDERATION=true
+
+COPY ./docker/complement.sh ./docker/complement-run.sh /usr/local/bin/
+RUN dos2unix /usr/local/bin/complement.sh /usr/local/bin/complement-run.sh
+
+EXPOSE 8008
+EXPOSE 8448
+
+RUN mkdir -p /data/pgdata
+RUN mkdir -p /run/postgresql
+RUN chown postgres:postgres /data/pgdata
+RUN chown postgres:postgres /run/postgresql
+RUN su postgres -c initdb
+RUN sh /usr/local/bin/complement.sh
+
+CMD /usr/local/bin/complement-run.sh
\ No newline at end of file
diff --git a/README.md b/README.md
index 1648dcd19e43c829b7016b223e09e235c20f3676..da217c7c25e61ba4622725c3a23057a79050eaca 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
# matrix-media-repo
-[](https://ci.appveyor.com/project/turt2live/matrix-media-repo)
-[](https://circleci.com/gh/turt2live/matrix-media-repo/tree/master)
+[](https://buildkite.com/t2bot/matrix-media-repo)
matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments
consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant
diff --git a/api/webserver/route_handler.go b/api/webserver/route_handler.go
index 88daf8e2cbe419fbd05f342118ba1fd0b5f04ae7..2d35369c5fd7d66dc4bea589926ae233f1a005ec 100644
--- a/api/webserver/route_handler.go
+++ b/api/webserver/route_handler.go
@@ -166,29 +166,8 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
"statusCode": strconv.Itoa(http.StatusOK),
}).Inc()
- textTypes := []string{
- "text/css",
- "text/csv",
- "text/html",
- "text/calendar",
- "text/plain",
- "text/javascript",
- "application/json",
- "application/ld+json",
- "application/rtf",
- "image/svg+xml",
- "text/xml",
- }
- contentType := strings.ToLower(result.ContentType)
- for _, v := range textTypes {
- if contentType == v {
- contentType += "; charset=UTF-8"
- break
- }
- }
-
w.Header().Set("Cache-Control", "private, max-age=259200") // 3 days
- w.Header().Set("Content-Type", contentType)
+ w.Header().Set("Content-Type", result.ContentType)
if result.SizeBytes > 0 {
w.Header().Set("Content-Length", fmt.Sprint(result.SizeBytes))
}
@@ -236,7 +215,7 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}).Inc()
// Order is important: Set headers before sending responses
- w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
encoder := json.NewEncoder(w)
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index 6414a746b9ec23d58a48ccd89e71c304842a6432..0000000000000000000000000000000000000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-version: "{build}"
-
-clone_folder: c:\gopath\src\github.com\turt2live\matrix-media-repo
-
-environment:
- GOPATH: c:\gopath
- GOVERSION: 1.14
-
-init:
- - git config --global core.autocrlf input
-
-install:
- # Install the specific Go version.
- - rmdir c:\go /s /q
- - appveyor DownloadFile https://storage.googleapis.com/golang/go%GOVERSION%.windows-amd64.msi
- - msiexec /i go%GOVERSION%.windows-amd64.msi /q
- - set PATH=c:\go\bin;c:\gopath\bin;%PATH%
- - go version
- - go env
-
-build_script:
- - set GOBIN=%CD%/bin
- - set GO111MODULE=on
- - ps: .\build.ps1
-
-artifacts:
- - path: bin/media_repo.exe
- name: media_repo.exe
- - path: bin/import_synapse.exe
- name: import_synapse.exe
- - path: bin/gdpr_export.exe
- name: gdpr_export.exe
- - path: bin/gdpr_import.exe
- name: gdpr_import.exe
-
-test_script: [] # https://github.com/turt2live/matrix-media-repo/issues/40
diff --git a/build.ps1 b/build.ps1
deleted file mode 100644
index d225b9e8f9a126effd2b8fbe481fe5cdfe56ac9e..0000000000000000000000000000000000000000
--- a/build.ps1
+++ /dev/null
@@ -1,5 +0,0 @@
-$GitCommit = (git rev-list -1 HEAD)
-$Version = (git describe --tags)
-go install -v ./cmd/compile_assets
-bin\compile_assets.exe
-go install -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$GitCommit -X github.com/turt2live/matrix-media-repo/common/version.Version=$Version" -v ./cmd/...
diff --git a/ci-complement.sh b/ci-complement.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5d4ca09b8ba393237aed3d855ba47353150c5f94
--- /dev/null
+++ b/ci-complement.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+git clone --depth=1 https://github.com/matrix-org/complement.git CI_COMPLEMENT
+cd CI_COMPLEMENT
+git fetch origin pull/14/head:complement-with-timeout
+git checkout complement-with-timeout
+go test -run '^(TestMediaWithoutFileName)$' -v ./tests
diff --git a/cmd/complement_hs/main.go b/cmd/complement_hs/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..3a926dfb889813d7a0f14cac98899fa1eb7e7fec
--- /dev/null
+++ b/cmd/complement_hs/main.go
@@ -0,0 +1,153 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "os/signal"
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/gorilla/mux"
+ "github.com/turt2live/matrix-media-repo/util/cleanup"
+)
+
+type VersionsResponse struct {
+ CSAPIVersions []string `json:"versions,flow"`
+}
+
+type RegisterRequest struct {
+ DesiredUsername string `json:"username"`
+}
+
+type RegisterResponse struct {
+ UserID string `json:"user_id"`
+ AccessToken string `json:"access_token"`
+}
+
+type WhoamiResponse struct {
+ UserID string `json:"user_id"`
+}
+
+func requestJson(r *http.Request, i interface{}) error {
+ b, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(b, &i)
+}
+
+func respondJson(w http.ResponseWriter, i interface{}) error {
+ resp, err := json.Marshal(i)
+ if err != nil {
+ return err
+ }
+ w.Header().Set("Content-Length",strconv.Itoa(len(resp)))
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(200)
+ _, err = w.Write(resp)
+ return err
+}
+
+func main() {
+ // Prepare local server
+ log.Println("Preparing local server...")
+ rtr := mux.NewRouter()
+ rtr.HandleFunc("/_matrix/client/versions", func(w http.ResponseWriter, r *http.Request) {
+ defer cleanup.DumpAndCloseStream(r.Body)
+ err := respondJson(w, &VersionsResponse{CSAPIVersions: []string{"r0.6.0"}})
+ if err != nil {
+ log.Fatal(err)
+ }
+ })
+ rtr.HandleFunc("/_matrix/client/r0/register", func(w http.ResponseWriter, r *http.Request) {
+ rr := &RegisterRequest{}
+ err := requestJson(r, &rr)
+ if err != nil {
+ log.Fatal(err)
+ }
+ userId := fmt.Sprintf("@%s:%s", rr.DesiredUsername, os.Getenv("SERVER_NAME"))
+ err = respondJson(w, &RegisterResponse{
+ AccessToken: userId,
+ UserID: userId,
+ })
+ if err != nil {
+ log.Fatal(err)
+ }
+ })
+ rtr.HandleFunc("/_matrix/client/r0/account/whoami", func(w http.ResponseWriter, r *http.Request) {
+ defer cleanup.DumpAndCloseStream(r.Body)
+ userId := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") // including space after Bearer.
+ err := respondJson(w, &WhoamiResponse{UserID: userId})
+ if err != nil {
+ log.Fatal(err)
+ }
+ })
+ rtr.PathPrefix("/_matrix/media/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // Proxy to the media repo running within the container
+ defer cleanup.DumpAndCloseStream(r.Body)
+ r2, err := http.NewRequest(r.Method, "http://127.0.0.1:8228" + r.RequestURI, r.Body)
+ if err != nil {
+ log.Fatal(err)
+ }
+ for k, v := range r.Header {
+ r2.Header.Set(k, v[0])
+ }
+ r2.Host = os.Getenv("SERVER_NAME")
+ resp, err := http.DefaultClient.Do(r2)
+ if err != nil {
+ log.Fatal(err)
+ }
+ for k, v := range resp.Header {
+ w.Header().Set(k, v[0])
+ }
+ defer cleanup.DumpAndCloseStream(resp.Body)
+ _, err = io.Copy(w, resp.Body)
+ if err != nil {
+ log.Fatal(err)
+ }
+ })
+
+ srv1 := &http.Server{Addr: "0.0.0.0:8008", Handler: rtr}
+ srv2 := &http.Server{Addr: "0.0.0.0:8448", Handler: rtr}
+
+ log.Println("Starting local server...")
+ waitGroup1 := &sync.WaitGroup{}
+ waitGroup2 := &sync.WaitGroup{}
+ go func() {
+ if err := srv1.ListenAndServe(); err != http.ErrServerClosed {
+ log.Fatal(err)
+ }
+ srv1 = nil
+ waitGroup1.Done()
+ }()
+ go func() {
+ if err := srv2.ListenAndServeTLS("/data/server.crt", "/data/server.key"); err != http.ErrServerClosed {
+ log.Fatal(err)
+ }
+ srv2 = nil
+ waitGroup2.Done()
+ }()
+
+ stop := make(chan os.Signal)
+ signal.Notify(stop, os.Interrupt, os.Kill)
+ go func() {
+ defer close(stop)
+ <-stop
+ log.Println("Stopping local server...")
+ _ = srv1.Close()
+ _ = srv2.Close()
+ }()
+
+ waitGroup1.Add(1)
+ waitGroup2.Add(1)
+ waitGroup1.Wait()
+ waitGroup2.Wait()
+
+ log.Println("Goodbye!")
+}
diff --git a/docker/complement-run.sh b/docker/complement-run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9dbef054d412295c606bb64226f5508374c6e0d1
--- /dev/null
+++ b/docker/complement-run.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+openssl req -new -newkey rsa:1024 -days 365 -nodes -x509 -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=${SERVER_NAME}" -keyout /data/server.key -out /data/server.crt
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /data/media-repo.yaml
+su postgres -c "postgres -h 0.0.0.0" &
+sleep 12
+/usr/local/bin/media_repo &
+/usr/local/bin/complement_hs
diff --git a/docker/complement.sh b/docker/complement.sh
new file mode 100644
index 0000000000000000000000000000000000000000..13128b69f5a472a294f84509ac0ea003a6bbcce3
--- /dev/null
+++ b/docker/complement.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+su postgres -c "postgres -h 0.0.0.0" &
+sleep 1
+su postgres -c "psql -c \"create user mediarepo with password 'mediarepo';\""
+su postgres -c "psql -c \"create database mediarepo;\""
+su postgres -c "psql -c \"grant all privileges on database mediarepo to mediarepo;\""
diff --git a/docker/complement.yaml b/docker/complement.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e73f26a3d9c8654df90ef6209b7c0e1063450c98
--- /dev/null
+++ b/docker/complement.yaml
@@ -0,0 +1,21 @@
+repo:
+ bindAddress: '127.0.0.1'
+ port: 8228
+database:
+ postgres: "postgres://mediarepo:mediarepo@127.0.0.1/mediarepo?sslmode=disable"
+homeservers:
+ - name: SERVER_NAME
+ csApi: "http://127.0.0.1:8008/"
+datastores:
+ - type: file
+ enabled: true
+ forKinds: ["all"]
+ opts:
+ path: /data/media
+urlPreviews:
+ disallowedNetworks:
+ - "192.168.0.0/16" # Don't limit localhost
+ allowedNetworks:
+ - "0.0.0.0/0"
+rateLimit:
+ enabled: false
diff --git a/go.mod b/go.mod
index 30246d0ea7b242204dd64cf0cc92a70125fa30b6..75ee76f8cda429284e8a5f37960a66becd029a82 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,6 @@ require (
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
github.com/golang/snappy v0.0.1 // indirect
github.com/gorilla/mux v1.7.4
- github.com/h2non/filetype v1.0.12
github.com/ipfs/go-cid v0.0.4
github.com/ipfs/go-ipfs v0.4.22-0.20191119151441-b8ec598d5801
github.com/ipfs/go-ipfs-config v0.0.11
diff --git a/go.sum b/go.sum
index eface946d1aeafd3e737a5e81541214c0db69d8f..8f2eb8d0856fa479f4b771225c205d0802a48774 100644
--- a/go.sum
+++ b/go.sum
@@ -195,8 +195,6 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
-github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao=
-github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
diff --git a/matrix/federation.go b/matrix/federation.go
index ff10a4fee2e5bbcc7cac58efefa6c1d43194c44b..ea92bb6318d3c48103ee5d9964e49ca4b3692a19 100644
--- a/matrix/federation.go
+++ b/matrix/federation.go
@@ -8,6 +8,7 @@ import (
"io/ioutil"
"net"
"net/http"
+ "os"
"strconv"
"strings"
"sync"
@@ -211,26 +212,55 @@ func FederatedGet(url string, realHost string, ctx rcontext.RequestContext) (*ht
req.Header.Set("User-Agent", "matrix-media-repo")
req.Host = realHost
- // This is how we verify the certificate is valid for the host we expect.
- // Previously using `req.URL.Host` we'd end up changing which server we were
- // connecting to (ie: matrix.org instead of matrix.org.cdn.cloudflare.net),
- // which obviously doesn't help us. We needed to do that though because the
- // HTTP client doesn't verify against the req.Host certificate, but it does
- // handle it off the req.URL.Host. So, we need to tell it which certificate
- // to verify.
-
- h, _, err := net.SplitHostPort(realHost)
- if err == nil {
- // Strip the port first, certs are port-insensitive
- realHost = h
- }
- client := http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{
- ServerName: realHost,
+ var client *http.Client
+ if os.Getenv("MEDIA_REPO_UNSAFE_FEDERATION") != "true" {
+ // This is how we verify the certificate is valid for the host we expect.
+ // Previously using `req.URL.Host` we'd end up changing which server we were
+ // connecting to (ie: matrix.org instead of matrix.org.cdn.cloudflare.net),
+ // which obviously doesn't help us. We needed to do that though because the
+ // HTTP client doesn't verify against the req.Host certificate, but it does
+ // handle it off the req.URL.Host. So, we need to tell it which certificate
+ // to verify.
+
+ h, _, err := net.SplitHostPort(realHost)
+ if err == nil {
+ // Strip the port first, certs are port-insensitive
+ realHost = h
+ }
+ client = &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ ServerName: realHost,
+ },
},
- },
- Timeout: time.Duration(ctx.Config.TimeoutSeconds.Federation) * time.Second,
+ Timeout: time.Duration(ctx.Config.TimeoutSeconds.Federation) * time.Second,
+ }
+ } else {
+ ctx.Log.Warn("Ignoring any certificate errors while making request")
+ tr := &http.Transport{
+ DisableKeepAlives: true,
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ // Based on https://github.com/matrix-org/gomatrixserverlib/blob/51152a681e69a832efcd934b60080b92bc98b286/client.go#L74-L90
+ DialTLS: func(network, addr string) (net.Conn, error) {
+ rawconn, err := net.Dial(network, addr)
+ if err != nil {
+ return nil, err
+ }
+ // Wrap a raw connection ourselves since tls.Dial defaults the SNI
+ conn := tls.Client(rawconn, &tls.Config{
+ ServerName: "",
+ InsecureSkipVerify: true,
+ })
+ if err := conn.Handshake(); err != nil {
+ return nil, err
+ }
+ return conn, nil
+ },
+ }
+ client = &http.Client{
+ Transport: tr,
+ Timeout: time.Duration(ctx.Config.TimeoutSeconds.UrlPreviews) * time.Second,
+ }
}
resp, err = client.Do(req)