diff --git a/CHANGELOG.md b/CHANGELOG.md
index 255999468a3813fb9226dcd0f9cf3f5deadc0725..af5c4acfbe0bab95847fb72694568dc5a50c6b2a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
* IPFS support has been removed due to maintenance burden.
+### Changed
+
+* Some admin endpoints for purging media, quarantining media, and background task information now require additional path components. See [docs/admin.md](./docs/admin.md) for more information.
+
## [1.2.13] - February 12, 2023
### Deprecations
diff --git a/api/_apimeta/auth.go b/api/_apimeta/auth.go
new file mode 100644
index 0000000000000000000000000000000000000000..943bbb18ed1059389eefd8b9f30a26cb02536a9b
--- /dev/null
+++ b/api/_apimeta/auth.go
@@ -0,0 +1,29 @@
+package _apimeta
+
+import (
+ "net/http"
+
+ "github.com/getsentry/sentry-go"
+
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/matrix"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+type UserInfo struct {
+ UserId string
+ AccessToken string
+ IsShared bool
+}
+
+func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
+ isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
+ isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)
+ if err != nil {
+ sentry.CaptureException(err)
+ rctx.Log.Error("Error verifying local admin: " + err.Error())
+ return isGlobalAdmin, false
+ }
+
+ return isGlobalAdmin, isLocalAdmin
+}
diff --git a/api/auth_cache/auth_cache.go b/api/_auth_cache/auth_cache.go
similarity index 99%
rename from api/auth_cache/auth_cache.go
rename to api/_auth_cache/auth_cache.go
index 69c0875db464742c76066fd3c38922eec38bba36..41756fb8eb1865c9b50dacbc33d93ef164665177 100644
--- a/api/auth_cache/auth_cache.go
+++ b/api/_auth_cache/auth_cache.go
@@ -1,4 +1,4 @@
-package auth_cache
+package _auth_cache
import (
"errors"
diff --git a/api/_responses/content.go b/api/_responses/content.go
new file mode 100644
index 0000000000000000000000000000000000000000..1f697ff00adf1509f670eea2717173095c51f74d
--- /dev/null
+++ b/api/_responses/content.go
@@ -0,0 +1,21 @@
+package _responses
+
+import "io"
+
+type EmptyResponse struct{}
+
+type HtmlResponse struct {
+ HTML string
+}
+
+type DownloadResponse struct {
+ ContentType string
+ Filename string
+ SizeBytes int64
+ Data io.ReadCloser
+ TargetDisposition string
+}
+
+type StreamDataResponse struct {
+ Stream io.Reader
+}
diff --git a/api/responses.go b/api/_responses/errors.go
similarity index 91%
rename from api/responses.go
rename to api/_responses/errors.go
index 93d2c39cbf59adcfa907400da85f55247be35794..deb93b29af320d492c13bdeb4234769c9c02e938 100644
--- a/api/responses.go
+++ b/api/_responses/errors.go
@@ -1,17 +1,7 @@
-package api
+package _responses
import "github.com/turt2live/matrix-media-repo/common"
-type EmptyResponse struct{}
-
-type DoNotCacheResponse struct {
- Payload interface{}
-}
-
-type HtmlResponse struct {
- HTML string
-}
-
type ErrorResponse struct {
Code string `json:"errcode"`
Message string `json:"error"`
@@ -22,6 +12,10 @@ func InternalServerError(message string) *ErrorResponse {
return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown}
}
+func BadGatewayError(message string) *ErrorResponse {
+ return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown}
+}
+
func MethodNotAllowed() *ErrorResponse {
return &ErrorResponse{common.ErrCodeUnknown, "Method Not Allowed", common.ErrCodeMethodNotAllowed}
}
diff --git a/api/_responses/meta.go b/api/_responses/meta.go
new file mode 100644
index 0000000000000000000000000000000000000000..0971bd72c889af94ba46ccea03bd239e17fde1c6
--- /dev/null
+++ b/api/_responses/meta.go
@@ -0,0 +1,5 @@
+package _responses
+
+type DoNotCacheResponse struct {
+ Payload interface{}
+}
diff --git a/api/_routers/00-install-params.go b/api/_routers/00-install-params.go
new file mode 100644
index 0000000000000000000000000000000000000000..47c48d9a2473913564e23c202b3b141a9ebe980f
--- /dev/null
+++ b/api/_routers/00-install-params.go
@@ -0,0 +1,29 @@
+package _routers
+
+import (
+ "errors"
+ "net/http"
+ "regexp"
+
+ "github.com/julienschmidt/httprouter"
+)
+
+func localCompile(expr string) *regexp.Regexp {
+ r, err := regexp.Compile(expr)
+ if err != nil {
+ panic(errors.New("error compiling expression: " + expr + " | " + err.Error()))
+ }
+ return r
+}
+
+var ServerNameRegex = localCompile("[a-zA-Z0-9.:\\-_]+")
+
+//var NumericIdRegex = localCompile("[0-9]+")
+
+func GetParam(name string, r *http.Request) string {
+ p := httprouter.ParamsFromContext(r.Context())
+ if p == nil {
+ return ""
+ }
+ return p.ByName(name)
+}
diff --git a/api/_routers/01-install_metadata.go b/api/_routers/01-install_metadata.go
new file mode 100644
index 0000000000000000000000000000000000000000..2ab5b29b9d347ddebc2740399966cef5cc0615c1
--- /dev/null
+++ b/api/_routers/01-install_metadata.go
@@ -0,0 +1,100 @@
+package _routers
+
+import (
+ "context"
+ "net/http"
+ "strconv"
+
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+const requestIdCtxKey = "mmr.request_id"
+const actionNameCtxKey = "mmr.action"
+const shouldIgnoreHostCtxKey = "mmr.ignore_host"
+const loggerCtxKey = "mmr.logger"
+
+type RequestCounter struct {
+ lastId uint64
+}
+
+func (c *RequestCounter) NextId() string {
+ strId := strconv.FormatUint(c.lastId, 10)
+ c.lastId = c.lastId + 1
+
+ return "REQ-" + strId
+}
+
+type InstallMetadataRouter struct {
+ next http.Handler
+ ignoreHost bool
+ actionName string
+ counter *RequestCounter
+}
+
+func NewInstallMetadataRouter(ignoreHost bool, actionName string, counter *RequestCounter, next http.Handler) *InstallMetadataRouter {
+ return &InstallMetadataRouter{
+ next: next,
+ ignoreHost: ignoreHost,
+ actionName: actionName,
+ counter: counter,
+ }
+}
+
+func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ requestId := i.counter.NextId()
+ logger := logrus.WithFields(logrus.Fields{
+ "method": r.Method,
+ "host": r.Host,
+ "resource": r.URL.Path,
+ "contentType": r.Header.Get("Content-Type"),
+ "contentLength": r.ContentLength,
+ "queryString": util.GetLogSafeQueryString(r),
+ "requestId": requestId,
+ "remoteAddr": r.RemoteAddr,
+ "userAgent": r.UserAgent(),
+ })
+
+ ctx := r.Context()
+ ctx = context.WithValue(ctx, requestIdCtxKey, requestId)
+ ctx = context.WithValue(ctx, actionNameCtxKey, i.actionName)
+ ctx = context.WithValue(ctx, shouldIgnoreHostCtxKey, i.ignoreHost)
+ ctx = context.WithValue(ctx, loggerCtxKey, logger)
+ r = r.WithContext(ctx)
+
+ if i.next != nil {
+ i.next.ServeHTTP(w, r)
+ }
+}
+
+func GetRequestId(r *http.Request) string {
+ x, ok := r.Context().Value(requestIdCtxKey).(string)
+ if !ok {
+ return "REQ-ID-UNKNOWN"
+ }
+ return x
+}
+
+func GetActionName(r *http.Request) string {
+ x, ok := r.Context().Value(actionNameCtxKey).(string)
+ if !ok {
+ return "<UNKNOWN>"
+ }
+ return x
+}
+
+func ShouldIgnoreHost(r *http.Request) bool {
+ x, ok := r.Context().Value(shouldIgnoreHostCtxKey).(bool)
+ if !ok {
+ return false
+ }
+ return x
+}
+
+func GetLogger(r *http.Request) *logrus.Entry {
+ x, ok := r.Context().Value(loggerCtxKey).(*logrus.Entry)
+ if !ok {
+ return nil
+ }
+ return x
+}
diff --git a/api/_routers/02-install-headers.go b/api/_routers/02-install-headers.go
new file mode 100644
index 0000000000000000000000000000000000000000..e0b13f3a71baa1821eafa0a8bd77596cc82b3852
--- /dev/null
+++ b/api/_routers/02-install-headers.go
@@ -0,0 +1,28 @@
+package _routers
+
+import (
+ "net/http"
+)
+
+type InstallHeadersRouter struct {
+ next http.Handler
+}
+
+func NewInstallHeadersRouter(next http.Handler) *InstallHeadersRouter {
+ return &InstallHeadersRouter{next: next}
+}
+
+func (i *InstallHeadersRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ headers := w.Header()
+ headers.Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+ headers.Set("Access-Control-Allow-Origin", "*")
+ headers.Set("Content-Security-Policy", "sandbox; default-src 'none'; script-src 'none'; plugin-types application/pdf; style-src 'unsafe-inline'; media-src 'self'; object-src 'self';")
+ headers.Set("Cross-Origin-Resource-Policy", "cross-origin")
+ headers.Set("X-Content-Security-Policy", "sandbox;")
+ headers.Set("X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex")
+ headers.Set("Server", "matrix-media-repo")
+
+ if i.next != nil {
+ i.next.ServeHTTP(w, r)
+ }
+}
diff --git a/api/_routers/03-host_detection.go b/api/_routers/03-host_detection.go
new file mode 100644
index 0000000000000000000000000000000000000000..a5e2ffb5ee2c8c746619f7837b92760735bf9574
--- /dev/null
+++ b/api/_routers/03-host_detection.go
@@ -0,0 +1,95 @@
+package _routers
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net"
+ "net/http"
+ "strings"
+
+ "github.com/getsentry/sentry-go"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/sebest/xff"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/metrics"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+const domainConfigCtxKey = "mmr.domain_config"
+
+type HostRouter struct {
+ next http.Handler
+}
+
+func NewHostRouter(next http.Handler) *HostRouter {
+ return &HostRouter{next: next}
+}
+
+func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("X-Forwarded-Host") != "" && config.Get().General.UseForwardedHost {
+ r.Host = r.Header.Get("X-Forwarded-Host")
+ }
+ r.Host = strings.Split(r.Host, ":")[0]
+
+ var raddr string
+ if config.Get().General.TrustAnyForward {
+ raddr = r.Header.Get("X-Forwarded-For")
+ } else {
+ raddr = xff.GetRemoteAddr(r)
+ }
+ if raddr == "" {
+ raddr = r.RemoteAddr
+ }
+ host, _, err := net.SplitHostPort(raddr)
+ if err != nil {
+ logrus.Error(err)
+ sentry.CaptureException(err)
+ host = raddr
+ }
+ r.RemoteAddr = host
+
+ ignoreHost := ShouldIgnoreHost(r)
+ isOurs := ignoreHost || util.IsServerOurs(r.Host)
+ if !isOurs {
+ logger := GetLogger(r)
+ metrics.InvalidHttpRequests.With(prometheus.Labels{
+ "action": GetActionName(r),
+ "method": r.Method,
+ }).Inc()
+ logger.Warn("The server name provided in the Host header is not configured, or the request was made directly to the media repo. Please specify a Host header and check your reverse proxy configuration. The request is being rejected.")
+ w.WriteHeader(http.StatusBadGateway)
+ if b, err := json.Marshal(_responses.BadGatewayError("Review server logs to continue")); err != nil {
+ panic(errors.New("error preparing BadGatewayError: " + err.Error()))
+ } else {
+ if _, err = w.Write(b); err != nil {
+ panic(errors.New("error sending BadGatewayError: " + err.Error()))
+ }
+ }
+ return // don't call next handler
+ }
+
+ cfg := config.GetDomain(r.Host)
+ if ignoreHost {
+ dc := config.DomainConfigFrom(*config.Get())
+ cfg = &dc
+ }
+
+ ctx := r.Context()
+ ctx = context.WithValue(ctx, domainConfigCtxKey, cfg)
+ r = r.WithContext(ctx)
+
+ if h.next != nil {
+ h.next.ServeHTTP(w, r)
+ }
+}
+
+func GetDomainConfig(r *http.Request) *config.DomainRepoConfig {
+ x, ok := r.Context().Value(domainConfigCtxKey).(*config.DomainRepoConfig)
+ if !ok {
+ return nil
+ }
+ return x
+}
diff --git a/api/_routers/04-request-metrics.go b/api/_routers/04-request-metrics.go
new file mode 100644
index 0000000000000000000000000000000000000000..911e84d630a0648d23eb3d8827ef021b0e9268a7
--- /dev/null
+++ b/api/_routers/04-request-metrics.go
@@ -0,0 +1,28 @@
+package _routers
+
+import (
+ "net/http"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/turt2live/matrix-media-repo/metrics"
+)
+
+type MetricsRequestRouter struct {
+ next http.Handler
+}
+
+func NewMetricsRequestRouter(next http.Handler) *MetricsRequestRouter {
+ return &MetricsRequestRouter{next: next}
+}
+
+func (m *MetricsRequestRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ metrics.HttpRequests.With(prometheus.Labels{
+ "host": r.Host,
+ "action": GetActionName(r),
+ "method": r.Method,
+ }).Inc()
+
+ if m.next != nil {
+ m.next.ServeHTTP(w, r)
+ }
+}
diff --git a/api/_routers/97-optional-access-token.go b/api/_routers/97-optional-access-token.go
new file mode 100644
index 0000000000000000000000000000000000000000..517ab346cd6e1d3901e34296f1786c0c12661be8
--- /dev/null
+++ b/api/_routers/97-optional-access-token.go
@@ -0,0 +1,55 @@
+package _routers
+
+import (
+ "net/http"
+
+ "github.com/getsentry/sentry-go"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_auth_cache"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/matrix"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn {
+ return func(r *http.Request, ctx rcontext.RequestContext) interface{} {
+ accessToken := util.GetAccessTokenFromRequest(r)
+ if accessToken == "" {
+ return generator(r, ctx, _apimeta.UserInfo{
+ UserId: "",
+ AccessToken: "",
+ IsShared: false,
+ })
+ }
+ if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token {
+ ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true})
+ return generator(r, ctx, _apimeta.UserInfo{
+ UserId: "@sharedsecret",
+ AccessToken: accessToken,
+ IsShared: true,
+ })
+ }
+ appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
+ userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId)
+ if err != nil {
+ if err != matrix.ErrInvalidToken {
+ sentry.CaptureException(err)
+ ctx.Log.Error("Error verifying token: ", err)
+ return _responses.InternalServerError("unexpected error validating access token")
+ }
+
+ ctx.Log.Warn("Failed to verify token (non-fatal): ", err)
+ userId = ""
+ }
+
+ ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId})
+ return generator(r, ctx, _apimeta.UserInfo{
+ UserId: userId,
+ AccessToken: accessToken,
+ IsShared: false,
+ })
+ }
+}
diff --git a/api/_routers/97-require-access-token.go b/api/_routers/97-require-access-token.go
new file mode 100644
index 0000000000000000000000000000000000000000..8d4b1fecf3c1baa89603896c41734c3e15e7dd81
--- /dev/null
+++ b/api/_routers/97-require-access-token.go
@@ -0,0 +1,59 @@
+package _routers
+
+import (
+ "net/http"
+
+ "github.com/getsentry/sentry-go"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_auth_cache"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/matrix"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+type GeneratorWithUserFn = func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{}
+
+func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn {
+ return func(r *http.Request, ctx rcontext.RequestContext) interface{} {
+ accessToken := util.GetAccessTokenFromRequest(r)
+ if accessToken == "" {
+ return &_responses.ErrorResponse{
+ Code: common.ErrCodeMissingToken,
+ Message: "no token provided (required)",
+ InternalCode: common.ErrCodeMissingToken,
+ }
+ }
+ if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token {
+ ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true})
+ return generator(r, ctx, _apimeta.UserInfo{
+ UserId: "@sharedsecret",
+ AccessToken: accessToken,
+ IsShared: true,
+ })
+ }
+ appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
+ userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId)
+ if err != nil || userId == "" {
+ if err == matrix.ErrGuestToken {
+ return _responses.GuestAuthFailed()
+ }
+ if err != nil && err != matrix.ErrInvalidToken {
+ sentry.CaptureException(err)
+ ctx.Log.Error("Error verifying token: ", err)
+ return _responses.InternalServerError("unexpected error validating access token")
+ }
+ return _responses.AuthFailed()
+ }
+
+ ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId})
+ return generator(r, ctx, _apimeta.UserInfo{
+ UserId: userId,
+ AccessToken: accessToken,
+ IsShared: false,
+ })
+ }
+}
diff --git a/api/_routers/97-require-repo-admin.go b/api/_routers/97-require-repo-admin.go
new file mode 100644
index 0000000000000000000000000000000000000000..a298c5a6690c6f3d1103fd3630460502c292801f
--- /dev/null
+++ b/api/_routers/97-require-repo-admin.go
@@ -0,0 +1,27 @@
+package _routers
+
+import (
+ "errors"
+ "net/http"
+
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn {
+ return func(r *http.Request, ctx rcontext.RequestContext) interface{} {
+ return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ if user.UserId == "" {
+ panic(errors.New("safety check failed: Repo admin access check received empty user ID"))
+ }
+
+ if !util.IsGlobalAdmin(user.UserId) {
+ return _responses.AuthFailed()
+ }
+
+ return generator(r, ctx, user)
+ })
+ }
+}
diff --git a/api/_routers/98-use-rcontext.go b/api/_routers/98-use-rcontext.go
new file mode 100644
index 0000000000000000000000000000000000000000..63fc7d06a5881826b3eda974d3579dad9d4040bc
--- /dev/null
+++ b/api/_routers/98-use-rcontext.go
@@ -0,0 +1,305 @@
+package _routers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "mime"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "github.com/alioygur/is"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+)
+
+const statusCodeCtxKey = "mmr.status_code"
+
+type GeneratorFn = func(r *http.Request, ctx rcontext.RequestContext) interface{}
+
+type RContextRouter struct {
+ generatorFn GeneratorFn
+ next http.Handler
+}
+
+func NewRContextRouter(generatorFn GeneratorFn, next http.Handler) *RContextRouter {
+ return &RContextRouter{generatorFn: generatorFn, next: next}
+}
+
+func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ log := GetLogger(r)
+ rctx := rcontext.RequestContext{
+ Context: r.Context(),
+ Log: log,
+ Config: *GetDomainConfig(r),
+ Request: r,
+ }
+
+ var res interface{}
+ res = c.generatorFn(r, rctx)
+ if res == nil {
+ res = &_responses.EmptyResponse{}
+ }
+
+ shouldCache := true
+ wrappedRes, isNoCache := res.(*_responses.DoNotCacheResponse)
+ if isNoCache {
+ shouldCache = false
+ res = wrappedRes.Payload
+ }
+
+ headers := w.Header()
+
+ // Check for HTML response and reply accordingly
+ if htmlRes, isHtml := res.(*_responses.HtmlResponse); isHtml {
+ log.Infof("Replying with result: %T <%d chars of html>", res, len(htmlRes.HTML))
+
+ // Write out HTML here, now that we know it's happening
+ if shouldCache {
+ headers.Set("Cache-Control", "private, max-age=259200") // 3 days
+ }
+ headers.Set("Content-Type", "text/html; charset=UTF-8")
+
+ // Clear the CSP because we're serving HTML
+ headers.Set("Content-Security-Policy", "")
+ headers.Set("X-Content-Security-Policy", "")
+
+ r = writeStatusCode(w, r, http.StatusOK)
+ if _, err := w.Write([]byte(htmlRes.HTML)); err != nil {
+ panic(errors.New("error sending HtmlResponse: " + err.Error()))
+ }
+ return // don't continue
+ }
+
+ // Next try handling the response as a download, which might turn into an error
+ proposedStatusCode := http.StatusOK
+ var stream io.ReadCloser
+ expectedBytes := int64(0)
+ var contentType string
+beforeParseDownload:
+ log.Infof("Replying with result: %T %+v", res, res)
+ if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload {
+ doRange, rangeStart, rangeEnd, grabBytes, rangeErrMsg := parseRange(r, downloadRes)
+ if doRange && rangeErrMsg != "" {
+ proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+ res = _responses.BadRequest(rangeErrMsg)
+ doRange = false
+ goto beforeParseDownload // reprocess `res`
+ }
+
+ contentType = downloadRes.ContentType
+ expectedBytes = downloadRes.SizeBytes
+
+ if shouldCache {
+ headers.Set("Cache-Control", "private, max-age=259200") // 3 days
+ }
+
+ if downloadRes.SizeBytes > 0 {
+ if config.Get().Redis.Enabled {
+ headers.Set("Accept-Ranges", "bytes")
+ }
+ }
+
+ disposition := downloadRes.TargetDisposition
+ if disposition == "" {
+ disposition = "inline"
+ } else if disposition == "infer" {
+ if contentType == "" {
+ disposition = "attachment"
+ } else {
+ if util.HasAnyPrefix(contentType, []string{"image/", "audio/", "video/", "text/plain"}) {
+ disposition = "inline"
+ } else {
+ disposition = "attachment"
+ }
+ }
+ }
+ fname := downloadRes.Filename
+ if fname == "" {
+ exts, err := mime.ExtensionsByType(contentType)
+ if err != nil {
+ exts = nil
+ sentry.CaptureException(err)
+ log.Warn("Unexpected error inferring file extension: ", err)
+ }
+ ext := ""
+ if exts != nil && len(exts) > 0 {
+ ext = exts[0]
+ }
+ fname = "file" + ext
+ }
+ if is.ASCII(fname) {
+ headers.Set("Content-Disposition", disposition+"; filename="+url.QueryEscape(fname))
+ } else {
+ headers.Set("Content-Disposition", disposition+"; filename*=utf-8''"+url.QueryEscape(fname))
+ }
+
+ if doRange {
+ defer stream_util.DumpAndCloseStream(downloadRes.Data)
+ seekStream, err := stream_util.ManualSeekStream(downloadRes.Data, rangeStart, grabBytes)
+ if err != nil {
+ panic(err) // blow up the request
+ }
+ stream = io.NopCloser(seekStream)
+ expectedBytes = grabBytes
+ headers.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", rangeStart, rangeEnd, downloadRes.SizeBytes))
+ proposedStatusCode = http.StatusPartialContent
+ } else {
+ stream = downloadRes.Data
+ }
+ }
+
+ // Try to find a suitable error code, if one is needed
+ if errRes, isError := res.(*_responses.ErrorResponse); isError && proposedStatusCode == http.StatusOK {
+ switch errRes.InternalCode {
+ case common.ErrCodeUnknownToken:
+ proposedStatusCode = http.StatusUnauthorized
+ break
+ case common.ErrCodeNotFound:
+ proposedStatusCode = http.StatusNotFound
+ break
+ case common.ErrCodeMediaTooLarge:
+ proposedStatusCode = http.StatusRequestEntityTooLarge
+ break
+ case common.ErrCodeBadRequest:
+ proposedStatusCode = http.StatusBadRequest
+ break
+ case common.ErrCodeMethodNotAllowed:
+ proposedStatusCode = http.StatusMethodNotAllowed
+ break
+ case common.ErrCodeForbidden:
+ proposedStatusCode = http.StatusForbidden
+ break
+ default: // Treat as unknown (a generic server error)
+ proposedStatusCode = http.StatusInternalServerError
+ break
+ }
+ }
+
+ // Prepare a stream if one isn't set, and assume JSON
+ if stream == nil {
+ contentType = "application/json"
+ b, err := json.Marshal(res)
+ if err != nil {
+ panic(err) // blow up this request
+ }
+ stream = io.NopCloser(bytes.NewReader(b))
+ expectedBytes = int64(len(b))
+ }
+
+ mediaType, params, err := mime.ParseMediaType(contentType)
+ if err != nil {
+ sentry.CaptureException(err)
+ log.Warn("Failed to parse content type header for media on reply: ", err)
+ } else {
+ // TODO: Maybe we only strip the charset from images? Is it valid to have the param on other types?
+ if !strings.HasPrefix(mediaType, "text/") && mediaType != "application/json" {
+ delete(params, "charset")
+ }
+ contentType = mime.FormatMediaType(mediaType, params)
+ }
+ headers.Set("Content-Type", contentType)
+
+ if expectedBytes > 0 {
+ headers.Set("Content-Length", strconv.FormatInt(expectedBytes, 10))
+ }
+
+ r = writeStatusCode(w, r, proposedStatusCode)
+
+ defer stream_util.DumpAndCloseStream(stream)
+ written, err := io.Copy(w, stream)
+ if err != nil {
+ panic(err) // blow up this request
+ }
+ if expectedBytes > 0 && written != expectedBytes {
+ panic(errors.New(fmt.Sprintf("mismatch transfer size: %d expected, %d sent", expectedBytes, written)))
+ }
+
+ if c.next != nil {
+ c.next.ServeHTTP(w, r)
+ }
+}
+
+func GetStatusCode(r *http.Request) int {
+ x, ok := r.Context().Value(statusCodeCtxKey).(int)
+ if !ok {
+ return http.StatusOK
+ }
+ return x
+}
+
+func writeStatusCode(w http.ResponseWriter, r *http.Request, statusCode int) *http.Request {
+ w.WriteHeader(statusCode)
+ return r.WithContext(context.WithValue(r.Context(), statusCodeCtxKey, statusCode))
+}
+
+func parseRange(r *http.Request, res *_responses.DownloadResponse) (bool, int64, int64, int64, string) {
+ rangeHeader := r.Header.Get("Range")
+ if rangeHeader == "" || res.SizeBytes <= 0 || !config.Get().Redis.Enabled {
+ return false, 0, 0, 0, ""
+ }
+
+ if !strings.HasPrefix(rangeHeader, "bytes=") {
+ return true, 0, 0, 0, "Improper range units"
+ }
+ if !strings.Contains(rangeHeader, ",") && !strings.HasPrefix(rangeHeader, "bytes=-") {
+ parts := strings.Split(rangeHeader[len("bytes="):], "-")
+ if len(parts) <= 2 {
+ grabBytes := int64(0)
+ rstart, err := strconv.ParseInt(parts[0], 10, 64)
+ if err != nil {
+ return true, 0, 0, 0, "Improper start of range"
+ }
+ if rstart < 0 {
+ return true, 0, 0, 0, "Improper start of range: negative"
+ }
+
+ rend := int64(-1)
+ if len(parts) > 1 && parts[1] != "" {
+ rend, err = strconv.ParseInt(parts[1], 10, 64)
+ if err != nil {
+ return true, 0, 0, 0, "Improper end of range"
+ }
+ if rend < 1 {
+ return true, 0, 0, 0, "Improper end of range: negative"
+ }
+ if rend >= res.SizeBytes {
+ return true, 0, 0, 0, "Improper end of range: out of bounds"
+ }
+ if rend <= rstart {
+ return true, 0, 0, 0, "Start must be before end"
+ }
+ if (rstart + rend) >= res.SizeBytes {
+ return true, 0, 0, 0, "Range too large"
+ }
+
+ grabBytes = rend - rstart
+ } else {
+ add := int64(10485760) // 10mb default
+ conf := GetDomainConfig(r)
+ if conf.Downloads.DefaultRangeChunkSizeBytes > 0 {
+ add = conf.Downloads.DefaultRangeChunkSizeBytes
+ }
+ rend = int64(math.Min(float64(rstart+add), float64(res.SizeBytes-1)))
+ grabBytes = (rend - rstart) + 1
+ }
+
+ if (rend-rstart) <= 0 || grabBytes <= 0 {
+ return true, 0, 0, 0, "Range invalid at last pass"
+ }
+ return true, rstart, rend, grabBytes, ""
+ }
+ }
+ return false, 0, 0, 0, ""
+}
diff --git a/api/_routers/99-response-metrics.go b/api/_routers/99-response-metrics.go
new file mode 100644
index 0000000000000000000000000000000000000000..8dd6eae2ea944553121328015188a215b4a0d870
--- /dev/null
+++ b/api/_routers/99-response-metrics.go
@@ -0,0 +1,30 @@
+package _routers
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/turt2live/matrix-media-repo/metrics"
+)
+
+type MetricsResponseRouter struct {
+ next http.Handler
+}
+
+func NewMetricsResponseRouter(next http.Handler) *MetricsResponseRouter {
+ return &MetricsResponseRouter{next: next}
+}
+
+func (m *MetricsResponseRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ metrics.HttpResponses.With(prometheus.Labels{
+ "host": r.Host,
+ "action": GetActionName(r),
+ "method": r.Method,
+ "statusCode": strconv.Itoa(GetStatusCode(r)),
+ }).Inc()
+
+ if m.next != nil {
+ m.next.ServeHTTP(w, r)
+ }
+}
diff --git a/api/auth.go b/api/auth.go
deleted file mode 100644
index 3cb51ef9d4a276f98a6e326a5a19722eb65f89cf..0000000000000000000000000000000000000000
--- a/api/auth.go
+++ /dev/null
@@ -1,127 +0,0 @@
-package api
-
-import (
- "github.com/getsentry/sentry-go"
- "net/http"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api/auth_cache"
- "github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/matrix"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type UserInfo struct {
- UserId string
- AccessToken string
- IsShared bool
-}
-
-func callUserNext(next func(r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{}, r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{} {
- r.WithContext(rctx)
- return next(r, rctx, user)
-}
-
-func AccessTokenRequiredRoute(next func(r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{}) func(*http.Request, rcontext.RequestContext) interface{} {
- return func(r *http.Request, rctx rcontext.RequestContext) interface{} {
- accessToken := util.GetAccessTokenFromRequest(r)
- if accessToken == "" {
- rctx.Log.Error("Error: no token provided (required)")
- return &ErrorResponse{common.ErrCodeMissingToken, "no token provided (required)", common.ErrCodeUnknownToken}
- }
- if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token {
- log := rctx.Log.WithFields(logrus.Fields{"isRepoAdmin": true})
- log.Info("User authed using shared secret")
- return callUserNext(next, r, rctx, UserInfo{UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true})
- }
- appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
- userId, err := auth_cache.GetUserId(rctx, accessToken, appserviceUserId)
- if err != nil || userId == "" {
- if err == matrix.ErrGuestToken {
- return GuestAuthFailed()
- }
- if err != nil && err != matrix.ErrInvalidToken {
- sentry.CaptureException(err)
- rctx.Log.Error("Error verifying token (fatal): ", err)
- return InternalServerError("Unexpected Error")
- }
-
- rctx.Log.Warn("Failed to verify token (fatal): ", err)
- return AuthFailed()
- }
-
- rctx = rctx.LogWithFields(logrus.Fields{"authUserId": userId})
- return callUserNext(next, r, rctx, UserInfo{userId, accessToken, false})
- }
-}
-
-func AccessTokenOptionalRoute(next func(r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{}) func(*http.Request, rcontext.RequestContext) interface{} {
- return func(r *http.Request, rctx rcontext.RequestContext) interface{} {
- accessToken := util.GetAccessTokenFromRequest(r)
- if accessToken == "" {
- return callUserNext(next, r, rctx, UserInfo{"", "", false})
- }
- if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token {
- rctx = rctx.LogWithFields(logrus.Fields{"isRepoAdmin": true})
- rctx.Log.Info("User authed using shared secret")
- return callUserNext(next, r, rctx, UserInfo{UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true})
- }
- appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
- userId, err := auth_cache.GetUserId(rctx, accessToken, appserviceUserId)
- if err != nil {
- if err != matrix.ErrInvalidToken {
- rctx.Log.Error("Error verifying token: ", err)
- return InternalServerError("Unexpected Error")
- }
-
- rctx.Log.Warn("Failed to verify token (non-fatal): ", err)
- userId = ""
- }
-
- rctx = rctx.LogWithFields(logrus.Fields{"authUserId": userId})
- return callUserNext(next, r, rctx, UserInfo{userId, accessToken, false})
- }
-}
-
-func RepoAdminRoute(next func(r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{}) func(*http.Request, rcontext.RequestContext) interface{} {
- regularFunc := AccessTokenRequiredRoute(func(r *http.Request, rctx rcontext.RequestContext, user UserInfo) interface{} {
- if user.UserId == "" {
- rctx.Log.Warn("Could not identify user for this admin route")
- return AuthFailed()
- }
- if !util.IsGlobalAdmin(user.UserId) {
- rctx.Log.Warn("User " + user.UserId + " is not a repository administrator")
- return AuthFailed()
- }
-
- rctx = rctx.LogWithFields(logrus.Fields{"isRepoAdmin": true})
- return callUserNext(next, r, rctx, user)
- })
-
- return func(r *http.Request, rctx rcontext.RequestContext) interface{} {
- if config.Get().SharedSecret.Enabled {
- accessToken := util.GetAccessTokenFromRequest(r)
- if accessToken == config.Get().SharedSecret.Token {
- rctx = rctx.LogWithFields(logrus.Fields{"isRepoAdmin": true})
- rctx.Log.Info("User authed using shared secret")
- return callUserNext(next, r, rctx, UserInfo{UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true})
- }
- }
-
- return regularFunc(r, rctx)
- }
-}
-
-func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
- isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
- isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)
- if err != nil {
- sentry.CaptureException(err)
- rctx.Log.Error("Error verifying local admin: " + err.Error())
- return isGlobalAdmin, false
- }
-
- return isGlobalAdmin, isLocalAdmin
-}
diff --git a/api/custom/datastores.go b/api/custom/datastores.go
index 75f60c0b3a686d2a0b59378527a1991436f9a2e9..93f3d23b9ecc738a742d3d217a65be481f7e9079 100644
--- a/api/custom/datastores.go
+++ b/api/custom/datastores.go
@@ -2,12 +2,14 @@ package custom
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"net/http"
"strconv"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
"github.com/turt2live/matrix-media-repo/storage"
@@ -21,12 +23,12 @@ type DatastoreMigration struct {
TaskID int `json:"task_id"`
}
-func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
datastores, err := storage.GetDatabase().GetMediaStore(rctx).GetAllDatastores()
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Error getting datastores")
+ return _responses.InternalServerError("Error getting datastores")
}
response := make(map[string]interface{})
@@ -38,24 +40,22 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user api.UserI
response[ds.DatastoreId] = dsMap
}
- return &api.DoNotCacheResponse{Payload: response}
+ return &_responses.DoNotCacheResponse{Payload: response}
}
-func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
beforeTsStr := r.URL.Query().Get("before_ts")
beforeTs := util.NowMillis()
var err error
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
- params := mux.Vars(r)
-
- sourceDsId := params["sourceDsId"]
- targetDsId := params["targetDsId"]
+ sourceDsId := _routers.GetParam("sourceDsId", r)
+ targetDsId := _routers.GetParam("targetDsId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"beforeTs": beforeTs,
@@ -64,19 +64,19 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use
})
if sourceDsId == targetDsId {
- return api.BadRequest("Source and target datastore cannot be the same")
+ return _responses.BadRequest("Source and target datastore cannot be the same")
}
sourceDatastore, err := datastore.LocateDatastore(rctx, sourceDsId)
if err != nil {
rctx.Log.Error(err)
- return api.BadRequest("Error getting source datastore. Does it exist?")
+ return _responses.BadRequest("Error getting source datastore. Does it exist?")
}
targetDatastore, err := datastore.LocateDatastore(rctx, targetDsId)
if err != nil {
rctx.Log.Error(err)
- return api.BadRequest("Error getting target datastore. Does it exist?")
+ return _responses.BadRequest("Error getting target datastore. Does it exist?")
}
rctx.Log.Info("User ", user.UserId, " has started a datastore media transfer")
@@ -84,14 +84,14 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected error starting migration")
+ return _responses.InternalServerError("Unexpected error starting migration")
}
estimate, err := maintenance_controller.EstimateDatastoreSizeWithAge(beforeTs, sourceDsId, rctx)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected error getting storage estimate")
+ return _responses.InternalServerError("Unexpected error getting storage estimate")
}
migration := &DatastoreMigration{
@@ -99,23 +99,21 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use
TaskID: task.ID,
}
- return &api.DoNotCacheResponse{Payload: migration}
+ return &_responses.DoNotCacheResponse{Payload: migration}
}
-func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
beforeTsStr := r.URL.Query().Get("before_ts")
beforeTs := util.NowMillis()
var err error
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
- params := mux.Vars(r)
-
- datastoreId := params["datastoreId"]
+ datastoreId := _routers.GetParam("datastoreId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"beforeTs": beforeTs,
@@ -126,7 +124,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext,
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected error getting storage estimate")
+ return _responses.InternalServerError("Unexpected error getting storage estimate")
}
- return &api.DoNotCacheResponse{Payload: result}
+ return &_responses.DoNotCacheResponse{Payload: result}
}
diff --git a/api/custom/exports.go b/api/custom/exports.go
index dd79da759732deeafd62622ff620def4c945d070..f12439e94d433807676e23b53e31e5648e8eeb1c 100644
--- a/api/custom/exports.go
+++ b/api/custom/exports.go
@@ -2,14 +2,16 @@ package custom
import (
"bytes"
- "github.com/getsentry/sentry-go"
"net/http"
"strconv"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/dustin/go-humanize"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/api/r0"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/data_controller"
@@ -36,25 +38,23 @@ type ExportMetadata struct {
Parts []*ExportPartMetadata `json:"parts"`
}
-func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
if !rctx.Config.Archiving.SelfService && !isAdmin {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
includeData := r.URL.Query().Get("include_data") != "false"
s3urls := r.URL.Query().Get("s3_urls") != "false"
- params := mux.Vars(r)
-
- userId := params["userId"]
+ userId := _routers.GetParam("userId", r)
if !isAdmin && user.UserId != userId {
- return api.BadRequest("cannot export data for another user")
+ return _responses.BadRequest("cannot export data for another user")
}
rctx = rctx.LogWithFields(logrus.Fields{
@@ -66,38 +66,36 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("fatal error starting export")
+ return _responses.InternalServerError("fatal error starting export")
}
- return &api.DoNotCacheResponse{Payload: &ExportStarted{
+ return &_responses.DoNotCacheResponse{Payload: &ExportStarted{
TaskID: task.ID,
ExportID: exportId,
}}
}
-func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
if !rctx.Config.Archiving.SelfService && !isAdmin {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
includeData := r.URL.Query().Get("include_data") != "false"
s3urls := r.URL.Query().Get("s3_urls") != "false"
- params := mux.Vars(r)
-
- serverName := params["serverName"]
+ serverName := _routers.GetParam("serverName", r)
if !isAdmin {
// They might be a local admin, so check that.
// We won't be able to check unless we know about the homeserver though
if !util.IsServerOurs(serverName) {
- return api.BadRequest("cannot export data for another server")
+ return _responses.BadRequest("cannot export data for another server")
}
isLocalAdmin, err := matrix.IsUserAdmin(rctx, serverName, user.AccessToken, r.RemoteAddr)
@@ -106,7 +104,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user api.Us
isLocalAdmin = false
}
if !isLocalAdmin {
- return api.BadRequest("cannot export data for another server")
+ return _responses.BadRequest("cannot export data for another server")
}
}
@@ -119,23 +117,26 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user api.Us
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("fatal error starting export")
+ return _responses.InternalServerError("fatal error starting export")
}
- return &api.DoNotCacheResponse{Payload: &ExportStarted{
+ return &_responses.DoNotCacheResponse{Payload: &ExportStarted{
TaskID: task.ID,
ExportID: exportId,
}}
}
-func ViewExport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ exportId := _routers.GetParam("exportId", r)
+
+ if !_routers.ServerNameRegex.MatchString(exportId) {
+ _responses.BadRequest("invalid export ID")
+ }
- exportId := params["exportId"]
rctx = rctx.LogWithFields(logrus.Fields{
"exportId": exportId,
})
@@ -146,21 +147,21 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get metadata")
+ return _responses.InternalServerError("failed to get metadata")
}
parts, err := exportDb.GetExportParts(exportId)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get export parts")
+ return _responses.InternalServerError("failed to get export parts")
}
template, err := templating.GetTemplate("view_export")
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get template")
+ return _responses.InternalServerError("failed to get template")
}
model := &templating.ViewExportModel{
@@ -183,20 +184,23 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to render template")
+ return _responses.InternalServerError("failed to render template")
}
- return &api.HtmlResponse{HTML: string(html.Bytes())}
+ return &_responses.HtmlResponse{HTML: string(html.Bytes())}
}
-func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ exportId := _routers.GetParam("exportId", r)
+
+ if !_routers.ServerNameRegex.MatchString(exportId) {
+ _responses.BadRequest("invalid export ID")
+ }
- exportId := params["exportId"]
rctx = rctx.LogWithFields(logrus.Fields{
"exportId": exportId,
})
@@ -207,14 +211,14 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user api.U
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get metadata")
+ return _responses.InternalServerError("failed to get metadata")
}
parts, err := exportDb.GetExportParts(exportId)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get export parts")
+ return _responses.InternalServerError("failed to get export parts")
}
metadata := &ExportMetadata{
@@ -229,21 +233,25 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user api.U
})
}
- return &api.DoNotCacheResponse{Payload: metadata}
+ return &_responses.DoNotCacheResponse{Payload: metadata}
}
-func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ exportId := _routers.GetParam("exportId", r)
+ pid := _routers.GetParam("partId", r)
- exportId := params["exportId"]
- partId, err := strconv.ParseInt(params["partId"], 10, 64)
+ if !_routers.ServerNameRegex.MatchString(exportId) {
+ _responses.BadRequest("invalid export ID")
+ }
+
+ partId, err := strconv.ParseInt(pid, 10, 64)
if err != nil {
rctx.Log.Error(err)
- return api.BadRequest("invalid part index")
+ return _responses.BadRequest("invalid part index")
}
rctx = rctx.LogWithFields(logrus.Fields{
@@ -256,33 +264,35 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user api.
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get part")
+ return _responses.InternalServerError("failed to get part")
}
s, err := datastore.DownloadStream(rctx, part.DatastoreID, part.Location)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to start download")
+ return _responses.InternalServerError("failed to start download")
}
return &r0.DownloadMediaResponse{
- ContentType: "application/gzip",
- SizeBytes: part.SizeBytes,
- Data: s,
- Filename: part.FileName,
+ ContentType: "application/gzip",
+ SizeBytes: part.SizeBytes,
+ Data: s,
+ Filename: part.FileName,
TargetDisposition: "attachment",
}
}
-func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ exportId := _routers.GetParam("exportId", r)
- exportId := params["exportId"]
+ if !_routers.ServerNameRegex.MatchString(exportId) {
+ _responses.BadRequest("invalid export ID")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"exportId": exportId,
@@ -295,7 +305,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to delete export")
+ return _responses.InternalServerError("failed to delete export")
}
for _, part := range parts {
@@ -304,7 +314,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to delete export")
+ return _responses.InternalServerError("failed to delete export")
}
rctx.Log.Info("Deleting object: " + part.Location)
@@ -320,8 +330,8 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to delete export")
+ return _responses.InternalServerError("failed to delete export")
}
- return api.EmptyResponse{}
+ return _responses.EmptyResponse{}
}
diff --git a/api/custom/federation.go b/api/custom/federation.go
index 4f50ee093a6eb1e8ebe7394b1038ad4ec570c5cd..84051d1388e075c9d2eefbc6bb581e4a175e81de 100644
--- a/api/custom/federation.go
+++ b/api/custom/federation.go
@@ -2,21 +2,25 @@ package custom
import (
"encoding/json"
- "github.com/getsentry/sentry-go"
"io/ioutil"
"net/http"
- "github.com/gorilla/mux"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/matrix"
)
-func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
+func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ serverName := _routers.GetParam("serverName", r)
- serverName := params["serverName"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
@@ -26,7 +30,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user api.U
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError(err.Error())
+ return _responses.InternalServerError(err.Error())
}
versionUrl := url + "/_matrix/federation/v1/version"
@@ -34,14 +38,14 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user api.U
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError(err.Error())
+ return _responses.InternalServerError(err.Error())
}
c, err := ioutil.ReadAll(versionResponse.Body)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError(err.Error())
+ return _responses.InternalServerError(err.Error())
}
out := make(map[string]interface{})
@@ -49,12 +53,12 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user api.U
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError(err.Error())
+ return _responses.InternalServerError(err.Error())
}
resp := make(map[string]interface{})
resp["base_url"] = url
resp["hostname"] = hostname
resp["versions_response"] = out
- return &api.DoNotCacheResponse{Payload: resp}
+ return &_responses.DoNotCacheResponse{Payload: resp}
}
diff --git a/api/custom/health.go b/api/custom/health.go
index cab5bf9996372d36f74606209034d38c17904bc0..f890788bf562a6b70b92563b1e5c26d27fb27616 100644
--- a/api/custom/health.go
+++ b/api/custom/health.go
@@ -3,7 +3,8 @@ package custom
import (
"net/http"
- "github.com/turt2live/matrix-media-repo/api"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
"github.com/turt2live/matrix-media-repo/common/rcontext"
)
@@ -12,8 +13,8 @@ type HealthzResponse struct {
Status string `json:"status"`
}
-func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- return &api.DoNotCacheResponse{
+func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ return &_responses.DoNotCacheResponse{
Payload: &HealthzResponse{
OK: true,
Status: "Probably not dead",
diff --git a/api/custom/imports.go b/api/custom/imports.go
index 3e51ef26e998d8ec70731aebafba982dbd7a3e67..532bd089dd03f00224ab5cf4159945a2d603c7bd 100644
--- a/api/custom/imports.go
+++ b/api/custom/imports.go
@@ -2,13 +2,15 @@ package custom
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"net/http"
- "github.com/gorilla/mux"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/data_controller"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type ImportStarted struct {
@@ -16,60 +18,64 @@ type ImportStarted struct {
TaskID int `json:"task_id"`
}
-func StartImport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- defer cleanup.DumpAndCloseStream(r.Body)
+ defer stream_util.DumpAndCloseStream(r.Body)
task, importId, err := data_controller.StartImport(r.Body, rctx)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("fatal error starting import")
+ return _responses.InternalServerError("fatal error starting import")
}
- return &api.DoNotCacheResponse{Payload: &ImportStarted{
+ return &_responses.DoNotCacheResponse{Payload: &ImportStarted{
TaskID: task.ID,
ImportID: importId,
}}
}
-func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ importId := _routers.GetParam("importId", r)
- importId := params["importId"]
+ if !_routers.ServerNameRegex.MatchString(importId) {
+ return _responses.BadRequest("invalid import ID")
+ }
- defer cleanup.DumpAndCloseStream(r.Body)
+ defer stream_util.DumpAndCloseStream(r.Body)
_, err := data_controller.AppendToImport(importId, r.Body, false)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("fatal error appending to import")
+ return _responses.InternalServerError("fatal error appending to import")
}
- return &api.DoNotCacheResponse{Payload: &api.EmptyResponse{}}
+ return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}}
}
-func StopImport(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Archiving.Enabled {
- return api.BadRequest("archiving is not enabled")
+ return _responses.BadRequest("archiving is not enabled")
}
- params := mux.Vars(r)
+ importId := _routers.GetParam("importId", r)
- importId := params["importId"]
+ if !_routers.ServerNameRegex.MatchString(importId) {
+ return _responses.BadRequest("invalid import ID")
+ }
err := data_controller.StopImport(importId)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("fatal error stopping import")
+ return _responses.InternalServerError("fatal error stopping import")
}
- return &api.DoNotCacheResponse{Payload: &api.EmptyResponse{}}
+ return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}}
}
diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go
index a4923330a57bd4de1891730d471a637bff94e255..b820ad167b3136551723781ce7626fb0acef4481 100644
--- a/api/custom/media_attributes.go
+++ b/api/custom/media_attributes.go
@@ -7,23 +7,24 @@ import (
"net/http"
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/matrix"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type Attributes struct {
Purpose string `json:"purpose"`
}
-func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user api.UserInfo) bool {
+func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user _apimeta.UserInfo) bool {
isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
if isGlobalAdmin {
return true
@@ -36,11 +37,13 @@ func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin s
return isLocalAdmin
}
-func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
+func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ origin := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
- origin := params["server"]
- mediaId := params["mediaId"]
+ if !_routers.ServerNameRegex.MatchString(origin) {
+ return _responses.BadRequest("invalid origin")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"origin": origin,
@@ -48,7 +51,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
})
if !canChangeAttributes(rctx, r, origin, user) {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
// Check to see if the media exists
@@ -57,10 +60,10 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if err != nil && err != sql.ErrNoRows {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get media record")
+ return _responses.InternalServerError("failed to get media record")
}
if media == nil || err == sql.ErrNoRows {
- return api.NotFoundError()
+ return _responses.NotFoundError()
}
db := storage.GetDatabase().GetMediaAttributesStore(rctx)
@@ -69,19 +72,21 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get attributes")
+ return _responses.InternalServerError("failed to get attributes")
}
- return &api.DoNotCacheResponse{Payload: &Attributes{
+ return &_responses.DoNotCacheResponse{Payload: &Attributes{
Purpose: attrs.Purpose,
}}
}
-func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
+func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ origin := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
- origin := params["server"]
- mediaId := params["mediaId"]
+ if !_routers.ServerNameRegex.MatchString(origin) {
+ return _responses.BadRequest("invalid origin")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"origin": origin,
@@ -89,15 +94,15 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
})
if !canChangeAttributes(rctx, r, origin, user) {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
- defer cleanup.DumpAndCloseStream(r.Body)
+ defer stream_util.DumpAndCloseStream(r.Body)
b, err := ioutil.ReadAll(r.Body)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to read attributes")
+ return _responses.InternalServerError("failed to read attributes")
}
newAttrs := &Attributes{}
@@ -105,7 +110,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to parse attributes")
+ return _responses.InternalServerError("failed to parse attributes")
}
db := storage.GetDatabase().GetMediaAttributesStore(rctx)
@@ -114,20 +119,20 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get attributes")
+ return _responses.InternalServerError("failed to get attributes")
}
if attrs.Purpose != newAttrs.Purpose {
if !util.ArrayContains(types.AllPurposes, newAttrs.Purpose) {
- return api.BadRequest("unknown purpose")
+ return _responses.BadRequest("unknown purpose")
}
err = db.UpsertPurpose(origin, mediaId, newAttrs.Purpose)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to update attributes: purpose")
+ return _responses.InternalServerError("failed to update attributes: purpose")
}
}
- return &api.DoNotCacheResponse{Payload: newAttrs}
+ return &_responses.DoNotCacheResponse{Payload: newAttrs}
}
diff --git a/api/custom/purge.go b/api/custom/purge.go
index e07523271c96534ac9ab2487d096a0b7a8e5d323..703cae8213a7b47945898a042b329a833f6a5212 100644
--- a/api/custom/purge.go
+++ b/api/custom/purge.go
@@ -2,13 +2,15 @@ package custom
import (
"database/sql"
- "github.com/getsentry/sentry-go"
"net/http"
"strconv"
- "github.com/gorilla/mux"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
@@ -22,14 +24,14 @@ type MediaPurgedResponse struct {
NumRemoved int `json:"total_removed"`
}
-func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
beforeTsStr := r.URL.Query().Get("before_ts")
if beforeTsStr == "" {
- return api.BadRequest("Missing before_ts argument")
+ return _responses.BadRequest("Missing before_ts argument")
}
beforeTs, err := strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
rctx = rctx.LogWithFields(logrus.Fields{
@@ -41,20 +43,22 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user api.Us
if err != nil {
rctx.Log.Error("Error purging remote media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Error purging remote media")
+ return _responses.InternalServerError("Error purging remote media")
}
- return &api.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}}
+ return &_responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}}
}
-func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
localServerName := r.Host
- params := mux.Vars(r)
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
- server := params["server"]
- mediaId := params["mediaId"]
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"server": server,
@@ -64,41 +68,41 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user a
// If the user is NOT a global admin, ensure they are speaking to the right server
if !isGlobalAdmin {
if server != localServerName {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
// If the user is NOT a local admin, ensure they uploaded the content in the first place
if !isLocalAdmin {
db := storage.GetDatabase().GetMediaStore(rctx)
m, err := db.Get(server, mediaId)
if err == sql.ErrNoRows {
- return api.NotFoundError()
+ return _responses.NotFoundError()
}
if err != nil {
rctx.Log.Error("Error checking ownership of media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error checking media ownership")
+ return _responses.InternalServerError("error checking media ownership")
}
if m.UserId != user.UserId {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
}
}
err := maintenance_controller.PurgeMedia(server, mediaId, rctx)
if err == sql.ErrNoRows || err == common.ErrMediaNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
}
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}}
}
-func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
localServerName := r.Host
var affected []*types.Media
@@ -109,13 +113,13 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user api.Us
} else if isLocalAdmin {
affected, err = maintenance_controller.PurgeQuarantinedFor(localServerName, rctx)
} else {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
mxcs := make([]string, 0)
@@ -123,17 +127,17 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user api.Us
mxcs = append(mxcs, a.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
}
-func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
var err error
beforeTs := util.NowMillis()
beforeTsStr := r.URL.Query().Get("before_ts")
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
@@ -142,7 +146,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if includeLocalStr != "" {
includeLocal, err = strconv.ParseBool(includeLocalStr)
if err != nil {
- return api.BadRequest("Error parsing include_local: " + err.Error())
+ return _responses.BadRequest("Error parsing include_local: " + err.Error())
}
}
@@ -156,7 +160,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
mxcs := make([]string, 0)
@@ -164,13 +168,13 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserI
mxcs = append(mxcs, a.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
}
-func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
if !isGlobalAdmin && !isLocalAdmin {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
var err error
@@ -179,13 +183,11 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
- params := mux.Vars(r)
-
- userId := params["userId"]
+ userId := _routers.GetParam("userId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"userId": userId,
@@ -196,11 +198,11 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error("Error parsing user ID (" + userId + "): " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error parsing user ID")
+ return _responses.InternalServerError("error parsing user ID")
}
if !isGlobalAdmin && userDomain != r.Host {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
affected, err := maintenance_controller.PurgeUserMedia(userId, beforeTs, rctx)
@@ -208,7 +210,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
mxcs := make([]string, 0)
@@ -216,13 +218,13 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
mxcs = append(mxcs, a.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
}
-func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
if !isGlobalAdmin && !isLocalAdmin {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
var err error
@@ -231,13 +233,11 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
- params := mux.Vars(r)
-
- roomId := params["roomId"]
+ roomId := _routers.GetParam("roomId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"roomId": roomId,
@@ -248,7 +248,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error("Error while listing media in the room: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error retrieving media in room")
+ return _responses.InternalServerError("error retrieving media in room")
}
mxcs := make([]string, 0)
@@ -288,7 +288,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
mxcs = make([]string, 0)
@@ -296,13 +296,13 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
mxcs = append(mxcs, a.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
}
-func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
if !isGlobalAdmin && !isLocalAdmin {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
var err error
@@ -311,13 +311,15 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.Us
if beforeTsStr != "" {
beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64)
if err != nil {
- return api.BadRequest("Error parsing before_ts: " + err.Error())
+ return _responses.BadRequest("Error parsing before_ts: " + err.Error())
}
}
- params := mux.Vars(r)
+ serverName := _routers.GetParam("serverName", r)
- serverName := params["serverName"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
@@ -325,7 +327,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.Us
})
if !isGlobalAdmin && serverName != r.Host {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
affected, err := maintenance_controller.PurgeDomainMedia(serverName, beforeTs, rctx)
@@ -333,7 +335,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.Us
if err != nil {
rctx.Log.Error("Error purging media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error purging media")
+ return _responses.InternalServerError("error purging media")
}
mxcs := make([]string, 0)
@@ -341,5 +343,5 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.Us
mxcs = append(mxcs, a.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
}
diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go
index 5571ced3e67abddabb4e1420ee78389aa2d7e7fe..4716a1d9b9b39ae076660c835919406653242c00 100644
--- a/api/custom/quarantine.go
+++ b/api/custom/quarantine.go
@@ -2,12 +2,14 @@ package custom
import (
"database/sql"
- "github.com/getsentry/sentry-go"
"net/http"
- "github.com/gorilla/mux"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/matrix"
@@ -23,15 +25,13 @@ type MediaQuarantinedResponse struct {
// Developer note: This isn't broken out into a dedicated controller class because the logic is slightly
// too complex to do so. If anything, the logic should be improved and moved.
-func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user)
if !canQuarantine {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
- params := mux.Vars(r)
-
- roomId := params["roomId"]
+ roomId := _routers.GetParam("roomId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"roomId": roomId,
@@ -42,7 +42,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api
if err != nil {
rctx.Log.Error("Error while listing media in the room: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error retrieving media in room")
+ return _responses.InternalServerError("error retrieving media in room")
}
var mxcs []string
@@ -55,7 +55,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api
if err != nil {
rctx.Log.Error("Error parsing MXC URI (" + mxc + "): " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error parsing mxc uri")
+ return _responses.InternalServerError("error parsing mxc uri")
}
if !allowOtherHosts && r.Host != server {
@@ -71,18 +71,16 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api
total += resp.(*MediaQuarantinedResponse).NumQuarantined
}
- return &api.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
+ return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
}
-func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user)
if !canQuarantine {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
- params := mux.Vars(r)
-
- userId := params["userId"]
+ userId := _routers.GetParam("userId", r)
rctx = rctx.LogWithFields(logrus.Fields{
"userId": userId,
@@ -93,11 +91,11 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api
if err != nil {
rctx.Log.Error("Error parsing user ID (" + userId + "): " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error parsing user ID")
+ return _responses.InternalServerError("error parsing user ID")
}
if !allowOtherHosts && userDomain != r.Host {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
db := storage.GetDatabase().GetMediaStore(rctx)
@@ -105,7 +103,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api
if err != nil {
rctx.Log.Error("Error while listing media for the user: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error retrieving media for user")
+ return _responses.InternalServerError("error retrieving media for user")
}
total := 0
@@ -118,18 +116,20 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api
total += resp.(*MediaQuarantinedResponse).NumQuarantined
}
- return &api.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
+ return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
}
-func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user)
if !canQuarantine {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
- params := mux.Vars(r)
+ serverName := _routers.GetParam("serverName", r)
- serverName := params["serverName"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
@@ -137,7 +137,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a
})
if !allowOtherHosts && serverName != r.Host {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
db := storage.GetDatabase().GetMediaStore(rctx)
@@ -145,7 +145,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a
if err != nil {
rctx.Log.Error("Error while listing media for the server: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error retrieving media for server")
+ return _responses.InternalServerError("error retrieving media for server")
}
total := 0
@@ -158,19 +158,21 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a
total += resp.(*MediaQuarantinedResponse).NumQuarantined
}
- return &api.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
+ return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}}
}
-func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user)
if !canQuarantine {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
- params := mux.Vars(r)
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
- server := params["server"]
- mediaId := params["mediaId"]
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"server": server,
@@ -179,11 +181,11 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user api.Use
})
if !allowOtherHosts && r.Host != server {
- return api.BadRequest("unable to quarantine media on other homeservers")
+ return _responses.BadRequest("unable to quarantine media on other homeservers")
}
resp, _ := doQuarantine(rctx, server, mediaId, allowOtherHosts)
- return &api.DoNotCacheResponse{Payload: resp}
+ return &_responses.DoNotCacheResponse{Payload: resp}
}
func doQuarantine(ctx rcontext.RequestContext, origin string, mediaId string, allowOtherHosts bool) (interface{}, bool) {
@@ -197,7 +199,7 @@ func doQuarantine(ctx rcontext.RequestContext, origin string, mediaId string, al
ctx.Log.Error("Error fetching media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error quarantining media"), false
+ return _responses.InternalServerError("error quarantining media"), false
}
return doQuarantineOn(media, allowOtherHosts, ctx)
@@ -210,7 +212,7 @@ func doQuarantineOn(media *types.Media, allowOtherHosts bool, ctx rcontext.Reque
if err != nil {
ctx.Log.Error("Error while getting attributes for media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Error quarantining media"), false
+ return _responses.InternalServerError("Error quarantining media"), false
}
if attr.Purpose == types.PurposePinned {
ctx.Log.Warn("Refusing to quarantine media due to it being pinned")
@@ -225,7 +227,7 @@ func doQuarantineOn(media *types.Media, allowOtherHosts bool, ctx rcontext.Reque
if err != nil {
ctx.Log.Error("Error quarantining media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Error quarantining media"), false
+ return _responses.InternalServerError("Error quarantining media"), false
}
return &MediaQuarantinedResponse{NumQuarantined: num}, true
@@ -258,7 +260,7 @@ func setMediaQuarantined(media *types.Media, isQuarantined bool, allowOtherHosts
return numQuarantined, nil
}
-func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) (bool, bool, bool) {
+func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) (bool, bool, bool) {
isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
canQuarantine := isGlobalAdmin
allowOtherHosts := isGlobalAdmin
diff --git a/api/custom/tasks.go b/api/custom/tasks.go
index 299330f6361f5a1a9fde53e9e920996706159166..0a390957994e843cd405e3b50e7f7c5d3bbbafd3 100644
--- a/api/custom/tasks.go
+++ b/api/custom/tasks.go
@@ -2,12 +2,14 @@ package custom
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"net/http"
"strconv"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/storage"
)
@@ -21,14 +23,12 @@ type TaskStatus struct {
IsFinished bool `json:"is_finished"`
}
-func GetTask(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- taskIdStr := params["taskId"]
+func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ taskIdStr := _routers.GetParam("taskId", r)
taskId, err := strconv.Atoi(taskIdStr)
if err != nil {
rctx.Log.Error(err)
- return api.BadRequest("invalid task ID")
+ return _responses.BadRequest("invalid task ID")
}
rctx = rctx.LogWithFields(logrus.Fields{
@@ -41,10 +41,10 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) i
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("failed to get task information")
+ return _responses.InternalServerError("failed to get task information")
}
- return &api.DoNotCacheResponse{Payload: &TaskStatus{
+ return &_responses.DoNotCacheResponse{Payload: &TaskStatus{
TaskID: task.ID,
Name: task.Name,
Params: task.Params,
@@ -54,14 +54,14 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) i
}}
}
-func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
db := storage.GetDatabase().GetMetadataStore(rctx)
tasks, err := db.GetAllBackgroundTasks()
if err != nil {
logrus.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get background tasks")
+ return _responses.InternalServerError("Failed to get background tasks")
}
statusObjs := make([]*TaskStatus, 0)
@@ -76,17 +76,17 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
})
}
- return &api.DoNotCacheResponse{Payload: statusObjs}
+ return &_responses.DoNotCacheResponse{Payload: statusObjs}
}
-func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
db := storage.GetDatabase().GetMetadataStore(rctx)
tasks, err := db.GetAllBackgroundTasks()
if err != nil {
logrus.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get background tasks")
+ return _responses.InternalServerError("Failed to get background tasks")
}
statusObjs := make([]*TaskStatus, 0)
@@ -104,5 +104,5 @@ func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user api
})
}
- return &api.DoNotCacheResponse{Payload: statusObjs}
+ return &_responses.DoNotCacheResponse{Payload: statusObjs}
}
diff --git a/api/custom/usage.go b/api/custom/usage.go
index d954de460743a4886e9a67d1b83a976a31e2b386..98f25ef3cc0c7e0e675dc86800fec05899cee42f 100644
--- a/api/custom/usage.go
+++ b/api/custom/usage.go
@@ -3,14 +3,16 @@ package custom
import (
"encoding/json"
"fmt"
- "github.com/getsentry/sentry-go"
"net/http"
"strconv"
"strings"
- "github.com/gorilla/mux"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/stores"
@@ -51,10 +53,12 @@ type MediaUsageEntry struct {
CreatedTs int64 `json:"created_ts"`
}
-func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
+func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ serverName := _routers.GetParam("serverName", r)
- serverName := params["serverName"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
@@ -66,17 +70,17 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user api.User
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get byte usage for server")
+ return _responses.InternalServerError("Failed to get byte usage for server")
}
mediaCount, thumbCount, err := db.GetCountUsageForServer(serverName)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get count usage for server")
+ return _responses.InternalServerError("Failed to get count usage for server")
}
- return &api.DoNotCacheResponse{
+ return &_responses.DoNotCacheResponse{
Payload: &CountsUsageResponse{
RawBytes: &UsageInfo{
MinimalUsageInfo: &MinimalUsageInfo{
@@ -96,12 +100,14 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user api.User
}
}
-func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- serverName := params["serverName"]
+func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ serverName := _routers.GetParam("serverName", r)
userIds := r.URL.Query()["user_id"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
+
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
})
@@ -119,7 +125,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get media records for users")
+ return _responses.InternalServerError("Failed to get media records for users")
}
parsed := make(map[string]*UserUsageEntry)
@@ -150,15 +156,17 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user api.UserIn
entry.UploadedMxcs = append(entry.UploadedMxcs, media.MxcUri())
}
- return &api.DoNotCacheResponse{Payload: parsed}
+ return &_responses.DoNotCacheResponse{Payload: parsed}
}
-func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- serverName := params["serverName"]
+func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ serverName := _routers.GetParam("serverName", r)
mxcs := r.URL.Query()["mxc"]
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
+
rctx = rctx.LogWithFields(logrus.Fields{
"serverName": serverName,
})
@@ -176,11 +184,11 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user api.Use
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Error parsing MXC " + mxc)
+ return _responses.InternalServerError("Error parsing MXC " + mxc)
}
if o != serverName {
- return api.BadRequest("MXC URIs must match the requested server")
+ return _responses.BadRequest("MXC URIs must match the requested server")
}
split = append(split, i)
@@ -191,7 +199,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user api.Use
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get media records for users")
+ return _responses.InternalServerError("Failed to get media records for users")
}
parsed := make(map[string]*MediaUsageEntry)
@@ -210,21 +218,24 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user api.Use
}
}
- return &api.DoNotCacheResponse{Payload: parsed}
+ return &_responses.DoNotCacheResponse{Payload: parsed}
}
// GetUsersUsageStats attempts to provide a loose equivalent to this Synapse admin end-point:
// https://matrix-org.github.io/synapse/develop/admin_api/statistics.html#users-media-usage-statistics
-func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
+func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
qs := r.URL.Query()
var err error
- serverName := params["serverName"]
+ serverName := _routers.GetParam("serverName", r)
+
+ if !_routers.ServerNameRegex.MatchString(serverName) {
+ return _responses.BadRequest("invalid server name")
+ }
- isGlobalAdmin, isLocalAdmin := api.GetRequestUserAdminStatus(r, rctx, user)
+ isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user)
if !isGlobalAdmin && (serverName != r.Host || !isLocalAdmin) {
- return api.AuthFailed()
+ return _responses.AuthFailed()
}
orderBy := qs.Get("order_by")
@@ -234,7 +245,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if !util.ArrayContains(stores.UsersUsageStatsSorts, orderBy) {
acceptedValsStr, _ := json.Marshal(stores.UsersUsageStatsSorts)
acceptedValsStr = []byte(strings.ReplaceAll(string(acceptedValsStr), "\"", "'"))
- return api.BadRequest(
+ return _responses.BadRequest(
fmt.Sprintf("Query parameter 'order_by' must be one of %s", acceptedValsStr))
}
@@ -242,7 +253,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if len(qs["from"]) > 0 {
start, err = strconv.ParseInt(qs.Get("from"), 10, 64)
if err != nil || start < 0 {
- return api.BadRequest("Query parameter 'from' must be a non-negative integer")
+ return _responses.BadRequest("Query parameter 'from' must be a non-negative integer")
}
}
@@ -250,7 +261,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if len(qs["limit"]) > 0 {
limit, err = strconv.ParseInt(qs.Get("limit"), 10, 64)
if err != nil || limit < 0 {
- return api.BadRequest("Query parameter 'limit' must be a non-negative integer")
+ return _responses.BadRequest("Query parameter 'limit' must be a non-negative integer")
}
}
@@ -259,7 +270,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if len(qs["from_ts"]) > 0 {
fromTS, err = strconv.ParseInt(qs.Get("from_ts"), 10, 64)
if err != nil || fromTS < 0 {
- return api.BadRequest("Query parameter 'from_ts' must be a non-negative integer")
+ return _responses.BadRequest("Query parameter 'from_ts' must be a non-negative integer")
}
}
@@ -267,15 +278,15 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if len(qs["until_ts"]) > 0 {
untilTS, err = strconv.ParseInt(qs.Get("until_ts"), 10, 64)
if err != nil || untilTS < 0 {
- return api.BadRequest("Query parameter 'until_ts' must be a non-negative integer")
+ return _responses.BadRequest("Query parameter 'until_ts' must be a non-negative integer")
} else if untilTS <= fromTS {
- return api.BadRequest("Query parameter 'until_ts' must be greater than 'from_ts'")
+ return _responses.BadRequest("Query parameter 'until_ts' must be greater than 'from_ts'")
}
}
searchTerm := qs.Get("search_term")
if searchTerm == "" && len(qs["search_term"]) > 0 {
- return api.BadRequest("Query parameter 'search_term' cannot be an empty string")
+ return _responses.BadRequest("Query parameter 'search_term' cannot be an empty string")
}
isAscendingOrder := true
@@ -285,7 +296,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
} else if direction == "b" {
isAscendingOrder = false
} else {
- return api.BadRequest("Query parameter 'dir' must be one of ['f', 'b']")
+ return _responses.BadRequest("Query parameter 'dir' must be one of ['f', 'b']")
}
rctx = rctx.LogWithFields(logrus.Fields{
@@ -314,7 +325,7 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("Failed to get users' usage stats on specified server")
+ return _responses.InternalServerError("Failed to get users' usage stats on specified server")
}
var users []map[string]interface{}
@@ -338,5 +349,5 @@ func GetUsersUsageStats(r *http.Request, rctx rcontext.RequestContext, user api.
result["next_token"] = start + int64(len(stats))
}
- return &api.DoNotCacheResponse{Payload: result}
+ return &_responses.DoNotCacheResponse{Payload: result}
}
diff --git a/api/custom/version.go b/api/custom/version.go
index f9bd53b93a3e43bcf9ef004e8ce4fa1f8e951368..dbcff87cb61150beb0cd15a1916de53ccb421775 100644
--- a/api/custom/version.go
+++ b/api/custom/version.go
@@ -3,15 +3,16 @@ package custom
import (
"net/http"
- "github.com/turt2live/matrix-media-repo/api"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/common/version"
)
-func GetVersion(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func GetVersion(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
unstableFeatures := make(map[string]bool)
- return &api.DoNotCacheResponse{
+ return &_responses.DoNotCacheResponse{
Payload: map[string]interface{}{
"Version": version.Version,
"GitCommit": version.GitCommit,
diff --git a/api/general_handlers.go b/api/general_handlers.go
deleted file mode 100644
index 1c20046008ff2b91b0d3b44c37fd8d5c6b2d5da4..0000000000000000000000000000000000000000
--- a/api/general_handlers.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package api
-
-import (
- "net/http"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
-)
-
-func NotFoundHandler(r *http.Request, rctx rcontext.RequestContext) interface{} {
- return NotFoundError()
-}
-
-func MethodNotAllowedHandler(r *http.Request, rctx rcontext.RequestContext) interface{} {
- return MethodNotAllowed()
-}
-
-func EmptyResponseHandler(r *http.Request, rctx rcontext.RequestContext) interface{} {
- return &EmptyResponse{}
-}
diff --git a/api/r0/download.go b/api/r0/download.go
index d84c74ec5b5f6a2d99f22a3ef27caf6132ca5d18..5b988220de6df5399e445e0ebd471356bd9af2a1 100644
--- a/api/r0/download.go
+++ b/api/r0/download.go
@@ -1,35 +1,32 @@
package r0
import (
- "github.com/getsentry/sentry-go"
- "io"
"net/http"
"strconv"
- "github.com/gorilla/mux"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/download_controller"
)
-type DownloadMediaResponse struct {
- ContentType string
- Filename string
- SizeBytes int64
- Data io.ReadCloser
- TargetDisposition string
-}
+type DownloadMediaResponse = _responses.DownloadResponse
-func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- server := params["server"]
- mediaId := params["mediaId"]
- filename := params["filename"]
+func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
+ filename := _routers.GetParam("filename", r)
allowRemote := r.URL.Query().Get("allow_remote")
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
+
targetDisposition := r.URL.Query().Get("org.matrix.msc2702.asAttachment")
if targetDisposition == "true" {
targetDisposition = "attachment"
@@ -43,7 +40,7 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserI
if allowRemote != "" {
parsedFlag, err := strconv.ParseBool(allowRemote)
if err != nil {
- return api.InternalServerError("allow_remote flag does not appear to be a boolean")
+ return _responses.InternalServerError("allow_remote flag does not appear to be a boolean")
}
downloadRemote = parsedFlag
}
@@ -58,15 +55,15 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserI
streamedMedia, err := download_controller.GetMedia(server, mediaId, downloadRemote, false, rctx)
if err != nil {
if err == common.ErrMediaNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
} else if err == common.ErrMediaTooLarge {
- return api.RequestTooLarge()
+ return _responses.RequestTooLarge()
} else if err == common.ErrMediaQuarantined {
- return api.NotFoundError() // We lie for security
+ return _responses.NotFoundError() // We lie for security
}
rctx.Log.Error("Unexpected error locating media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
if filename == "" {
diff --git a/api/r0/identicon.go b/api/r0/identicon.go
index 9fa756e3bcadfd817695de58826ef1f3d027cdc3..18387ccd34f139342d7375a7d223511c80ab2042 100644
--- a/api/r0/identicon.go
+++ b/api/r0/identicon.go
@@ -3,31 +3,28 @@ package r0
import (
"bytes"
"crypto/md5"
- "github.com/getsentry/sentry-go"
"image/color"
"io"
"net/http"
"strconv"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"github.com/cupcake/sigil/gen"
"github.com/disintegration/imaging"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
)
-type IdenticonResponse struct {
- Avatar io.Reader
-}
-
-func Identicon(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.Identicons.Enabled {
- return api.NotFoundError()
+ return _responses.NotFoundError()
}
- params := mux.Vars(r)
- seed := params["seed"]
+ seed := _routers.GetParam("seed", r)
var err error
width := 96
@@ -38,14 +35,14 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
if widthStr != "" {
width, err = strconv.Atoi(widthStr)
if err != nil {
- return api.InternalServerError("Error parsing width: " + err.Error())
+ return _responses.InternalServerError("Error parsing width: " + err.Error())
}
height = width
}
if heightStr != "" {
height, err = strconv.Atoi(heightStr)
if err != nil {
- return api.InternalServerError("Error parsing height: " + err.Error())
+ return _responses.InternalServerError("Error parsing height: " + err.Error())
}
}
@@ -86,10 +83,16 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
if err != nil {
rctx.Log.Error("Error generating image:" + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("error generating identicon")
+ return _responses.InternalServerError("error generating identicon")
}
- return &IdenticonResponse{Avatar: imgData}
+ return &_responses.DownloadResponse{
+ ContentType: "image/png",
+ Filename: string(hashed) + ".png",
+ SizeBytes: 0,
+ Data: io.NopCloser(imgData),
+ TargetDisposition: "inline",
+ }
}
func rgb(r, g, b uint8) color.NRGBA {
diff --git a/api/r0/logout.go b/api/r0/logout.go
index fc92f986844754ad014974dc347a458cfe3defae..3a81eefe4ab5e9428ead7aac7e907b887d91296f 100644
--- a/api/r0/logout.go
+++ b/api/r0/logout.go
@@ -2,29 +2,31 @@ package r0
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+
"net/http"
- "github.com/turt2live/matrix-media-repo/api"
- "github.com/turt2live/matrix-media-repo/api/auth_cache"
+ "github.com/turt2live/matrix-media-repo/api/_auth_cache"
"github.com/turt2live/matrix-media-repo/common/rcontext"
)
-func Logout(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- err := auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId)
+func Logout(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ err := _auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("unable to logout")
+ return _responses.InternalServerError("unable to logout")
}
- return api.EmptyResponse{}
+ return _responses.EmptyResponse{}
}
-func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- err := auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId)
+func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ err := _auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return api.InternalServerError("unable to logout")
+ return _responses.InternalServerError("unable to logout")
}
- return api.EmptyResponse{}
+ return _responses.EmptyResponse{}
}
diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go
index 98eac2d42150cfad77864e88c768721105fd65ee..c761a4f0908cb97083e537696b04660fabfe77ba 100644
--- a/api/r0/preview_url.go
+++ b/api/r0/preview_url.go
@@ -2,11 +2,13 @@ package r0
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+
"net/http"
"strconv"
"strings"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/preview_controller"
@@ -26,9 +28,9 @@ type MatrixOpenGraph struct {
ImageHeight int `json:"og:image:height,omitempty"`
}
-func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
if !rctx.Config.UrlPreviews.Enabled {
- return api.NotFoundError()
+ return _responses.NotFoundError()
}
params := r.URL.Query()
@@ -42,16 +44,16 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo
ts, err = strconv.ParseInt(tsStr, 10, 64)
if err != nil {
rctx.Log.Error("Error parsing ts: " + err.Error())
- return api.BadRequest(err.Error())
+ return _responses.BadRequest(err.Error())
}
}
// Validate the URL
if urlStr == "" {
- return api.BadRequest("No url provided")
+ return _responses.BadRequest("No url provided")
}
if strings.Index(urlStr, "http://") != 0 && strings.Index(urlStr, "https://") != 0 {
- return api.BadRequest("Scheme not accepted")
+ return _responses.BadRequest("Scheme not accepted")
}
languageHeader := rctx.Config.UrlPreviews.DefaultLanguage
@@ -62,12 +64,12 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo
preview, err := preview_controller.GetPreview(urlStr, r.Host, user.UserId, ts, languageHeader, rctx)
if err != nil {
if err == common.ErrMediaNotFound || err == common.ErrHostNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
} else if err == common.ErrInvalidHost || err == common.ErrHostBlacklisted {
- return api.BadRequest(err.Error())
+ return _responses.BadRequest(err.Error())
} else {
sentry.CaptureException(err)
- return api.InternalServerError("unexpected error during request")
+ return _responses.InternalServerError("unexpected error during request")
}
}
diff --git a/api/r0/public_config.go b/api/r0/public_config.go
index 66d313f2e386fd02c96f43aa16d0aa87666701e6..fc9bf3d6830c9795039411814740556120818726 100644
--- a/api/r0/public_config.go
+++ b/api/r0/public_config.go
@@ -3,7 +3,7 @@ package r0
import (
"net/http"
- "github.com/turt2live/matrix-media-repo/api"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
"github.com/turt2live/matrix-media-repo/common/rcontext"
)
@@ -11,7 +11,7 @@ type PublicConfigResponse struct {
UploadMaxSize int64 `json:"m.upload.size,omitempty"`
}
-func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
uploadSize := rctx.Config.Uploads.ReportedMaxSizeBytes
if uploadSize == 0 {
uploadSize = rctx.Config.Uploads.MaxSizeBytes
diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go
index b1b73f01a6a8b072c53eed0961a157b363c6e691..7e56131e9fccb8de2c4143c7bc2fb2db6fc04426 100644
--- a/api/r0/thumbnail.go
+++ b/api/r0/thumbnail.go
@@ -2,29 +2,33 @@ package r0
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+
"net/http"
"strconv"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller"
)
-func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- server := params["server"]
- mediaId := params["mediaId"]
+func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
allowRemote := r.URL.Query().Get("allow_remote")
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
+
downloadRemote := true
if allowRemote != "" {
parsedFlag, err := strconv.ParseBool(allowRemote)
if err != nil {
- return api.BadRequest("allow_remote flag does not appear to be a boolean")
+ return _responses.BadRequest("allow_remote flag does not appear to be a boolean")
}
downloadRemote = parsedFlag
}
@@ -44,7 +48,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
}
if widthStr == "" || heightStr == "" {
- return api.BadRequest("Width and height are required")
+ return _responses.BadRequest("Width and height are required")
}
width := 0
@@ -54,21 +58,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
if widthStr != "" {
parsedWidth, err := strconv.Atoi(widthStr)
if err != nil {
- return api.BadRequest("Width does not appear to be an integer")
+ return _responses.BadRequest("Width does not appear to be an integer")
}
width = parsedWidth
}
if heightStr != "" {
parsedHeight, err := strconv.Atoi(heightStr)
if err != nil {
- return api.BadRequest("Height does not appear to be an integer")
+ return _responses.BadRequest("Height does not appear to be an integer")
}
height = parsedHeight
}
if animatedStr != "" {
parsedFlag, err := strconv.ParseBool(animatedStr)
if err != nil {
- return api.BadRequest("Animated flag does not appear to be a boolean")
+ return _responses.BadRequest("Animated flag does not appear to be a boolean")
}
animated = parsedFlag
}
@@ -84,19 +88,19 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user api.User
})
if width <= 0 || height <= 0 {
- return api.BadRequest("Width and height must be greater than zero")
+ return _responses.BadRequest("Width and height must be greater than zero")
}
streamedThumbnail, err := thumbnail_controller.GetThumbnail(server, mediaId, width, height, animated, method, downloadRemote, rctx)
if err != nil {
if err == common.ErrMediaNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
} else if err == common.ErrMediaTooLarge {
- return api.RequestTooLarge()
+ return _responses.RequestTooLarge()
}
rctx.Log.Error("Unexpected error locating media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
return &DownloadMediaResponse{
diff --git a/api/r0/upload.go b/api/r0/upload.go
index 63b0170beb0578b4cdfea0fa07a5add47b8b7dd2..1db122493f1b80e7d4c9fa146fe63997fd1b7317 100644
--- a/api/r0/upload.go
+++ b/api/r0/upload.go
@@ -2,19 +2,21 @@ package r0
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"io"
"io/ioutil"
"net/http"
"path/filepath"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/info_controller"
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
"github.com/turt2live/matrix-media-repo/quota"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type MediaUploadedResponse struct {
@@ -22,9 +24,9 @@ type MediaUploadedResponse struct {
Blurhash string `json:"xyz.amorgan.blurhash,omitempty"`
}
-func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
filename := filepath.Base(r.URL.Query().Get("filename"))
- defer cleanup.DumpAndCloseStream(r.Body)
+ defer stream_util.DumpAndCloseStream(r.Body)
rctx = rctx.LogWithFields(logrus.Fields{
"filename": filename,
@@ -37,12 +39,12 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf
if upload_controller.IsRequestTooLarge(r.ContentLength, r.Header.Get("Content-Length"), rctx) {
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
- return api.RequestTooLarge()
+ return _responses.RequestTooLarge()
}
if upload_controller.IsRequestTooSmall(r.ContentLength, r.Header.Get("Content-Length"), rctx) {
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
- return api.RequestTooSmall()
+ return _responses.RequestTooSmall()
}
inQuota, err := quota.IsUserWithinQuota(rctx, user.UserId)
@@ -50,11 +52,11 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
rctx.Log.Error("Unexpected error checking quota: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
if !inQuota {
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
- return api.QuotaExceeded()
+ return _responses.QuotaExceeded()
}
contentLength := upload_controller.EstimateContentLength(r.ContentLength, r.Header.Get("Content-Length"))
@@ -64,12 +66,12 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
if err == common.ErrMediaQuarantined {
- return api.BadRequest("This file is not permitted on this server")
+ return _responses.BadRequest("This file is not permitted on this server")
}
rctx.Log.Error("Unexpected error storing media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
if rctx.Config.Features.MSC2448Blurhash.Enabled && r.URL.Query().Get("xyz.amorgan.generate_blurhash") == "true" {
diff --git a/api/router.go b/api/router.go
new file mode 100644
index 0000000000000000000000000000000000000000..8e49a33b59270434473832e0a26724193994c971
--- /dev/null
+++ b/api/router.go
@@ -0,0 +1,78 @@
+package api
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+
+ "github.com/getsentry/sentry-go"
+ "github.com/julienschmidt/httprouter"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+)
+
+func buildPrimaryRouter() *httprouter.Router {
+ router := httprouter.New()
+ router.RedirectTrailingSlash = false // spec compliance
+ router.RedirectFixedPath = false // don't fix case
+ router.MethodNotAllowed = http.HandlerFunc(methodNotAllowedFn)
+ router.NotFound = http.HandlerFunc(notFoundFn)
+ //router.GlobalOPTIONS = http.HandlerFunc(corsFn)
+ router.PanicHandler = panicFn
+ return router
+}
+
+func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ if b, err := json.Marshal(_responses.MethodNotAllowed()); err != nil {
+ panic(errors.New("error preparing MethodNotAllowed: " + err.Error()))
+ } else {
+ if _, err = w.Write(b); err != nil {
+ panic(errors.New("error sending MethodNotAllowed: " + err.Error()))
+ }
+ }
+}
+
+func notFoundFn(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusNotFound)
+ if b, err := json.Marshal(_responses.NotFoundError()); err != nil {
+ panic(errors.New("error preparing NotFound: " + err.Error()))
+ } else {
+ if _, err = w.Write(b); err != nil {
+ panic(errors.New("error sending NotFound: " + err.Error()))
+ }
+ }
+}
+
+//func corsFn(w http.ResponseWriter, r *http.Request) {
+// header := w.Header()
+// if header.Get("Access-Control-Request-Method") != "" {
+// header.Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+// header.Set("Access-Control-Allow-Origin", "Access-Control-Allow-Origin")
+// }
+//
+// w.WriteHeader(http.StatusNoContent)
+//}
+
+func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) {
+ logrus.Errorf("Panic received on %s %s: %s", r.Method, r.URL.String(), i)
+
+ if e, ok := i.(error); ok {
+ sentry.CaptureException(e)
+ } else {
+ sentry.CaptureMessage(fmt.Sprintf("Unknown panic received: %T %s %+v", i, i, i))
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusInternalServerError)
+ if b, err := json.Marshal(_responses.InternalServerError("unexpected error")); err != nil {
+ panic(errors.New("error preparing InternalServerError: " + err.Error()))
+ } else {
+ if _, err = w.Write(b); err != nil {
+ panic(errors.New("error sending InternalServerError: " + err.Error()))
+ }
+ }
+}
diff --git a/api/routes.go b/api/routes.go
new file mode 100644
index 0000000000000000000000000000000000000000..7b81d67afad8ee105a448594a525736a61bad7cd
--- /dev/null
+++ b/api/routes.go
@@ -0,0 +1,124 @@
+package api
+
+import (
+ "fmt"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/julienschmidt/httprouter"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+ "github.com/turt2live/matrix-media-repo/api/custom"
+ "github.com/turt2live/matrix-media-repo/api/r0"
+ "github.com/turt2live/matrix-media-repo/api/unstable"
+ "github.com/turt2live/matrix-media-repo/api/webserver/debug"
+)
+
+const PrefixMedia = "/_matrix/media"
+const PrefixClient = "/_matrix/client"
+
+func buildRoutes() http.Handler {
+ counter := &_routers.RequestCounter{}
+ router := buildPrimaryRouter()
+
+ pprofSecret := os.Getenv("MEDIA_PPROF_SECRET_KEY")
+ if pprofSecret != "" {
+ logrus.Warn("Enabling pprof/debug http endpoints")
+ debug.BindPprofEndpoints(router, pprofSecret)
+ }
+
+ // Standard (spec) features
+ register([]string{"POST"}, PrefixMedia, "upload", false, router, makeRoute(_routers.RequireAccessToken(r0.UploadMedia), "upload", false, counter))
+ downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMedia), "download", false, counter)
+ register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", false, router, downloadRoute)
+ register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", false, router, downloadRoute)
+ register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", false, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", false, counter))
+ register([]string{"GET"}, PrefixMedia, "preview_url", false, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", false, counter))
+ register([]string{"GET"}, PrefixMedia, "identicon/*seed", false, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", false, counter))
+ register([]string{"GET"}, PrefixMedia, "config", false, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", false, counter))
+ register([]string{"POST"}, PrefixClient, "logout", false, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", false, counter))
+ register([]string{"POST"}, PrefixClient, "logout/all", false, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", false, counter))
+
+ // Custom features
+ register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", true, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", false, counter))
+ register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", true, router, makeRoute(_routers.RequireAccessToken(unstable.MediaInfo), "info", false, counter))
+ purgeOneRoute := makeRoute(_routers.RequireAccessToken(custom.PurgeIndividualRecord), "purge_individual_media", false, counter)
+ register([]string{"DELETE"}, PrefixMedia, "download/:server/:mediaId", true, router, purgeOneRoute)
+
+ // Custom and top-level features
+ router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(_routers.OptionalAccessToken(custom.GetVersion), "get_version", false, counter))
+ healthzRoute := makeRoute(_routers.OptionalAccessToken(custom.GetHealthz), "healthz", true, counter)
+ router.Handler("GET", "/healthz", healthzRoute)
+ router.Handler("HEAD", "/healthz", healthzRoute)
+
+ // All admin routes are unstable only
+ purgeRemoteRoute := makeRoute(_routers.RequireRepoAdmin(custom.PurgeRemoteMedia), "purge_remote_media", false, counter)
+ register([]string{"POST"}, PrefixMedia, "admin/purge_remote", true, router, purgeRemoteRoute)
+ register([]string{"POST"}, PrefixMedia, "admin/purge/remote", true, router, purgeRemoteRoute)
+ register([]string{"POST"}, PrefixClient, "admin/purge_media_cache", true, router, purgeRemoteRoute) // synapse compat
+ register([]string{"POST"}, PrefixMedia, "admin/purge/media/:server/:mediaId", true, router, purgeOneRoute)
+ register([]string{"POST"}, PrefixMedia, "admin/purge/old", true, router, makeRoute(_routers.RequireRepoAdmin(custom.PurgeOldMedia), "purge_old_media", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/purge/quarantined", true, router, makeRoute(_routers.RequireAccessToken(custom.PurgeQuarantined), "purge_quarantined", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/purge/user/:userId", true, router, makeRoute(_routers.RequireAccessToken(custom.PurgeUserMedia), "purge_user_media", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/purge/room/:roomId", true, router, makeRoute(_routers.RequireAccessToken(custom.PurgeRoomMedia), "purge_room_media", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/purge/server/:serverName", true, router, makeRoute(_routers.RequireAccessToken(custom.PurgeDomainMedia), "purge_domain_media", false, counter))
+ quarantineRoomRoute := makeRoute(_routers.RequireAccessToken(custom.QuarantineRoomMedia), "quarantine_room", false, counter)
+ register([]string{"POST"}, PrefixMedia, "admin/quarantine/room/:roomId", true, router, quarantineRoomRoute)
+ register([]string{"POST"}, PrefixClient, "admin/quarantine_media/:roomId", true, router, quarantineRoomRoute) // synapse compat
+ register([]string{"POST"}, PrefixMedia, "admin/quarantine/user/:userId", true, router, makeRoute(_routers.RequireAccessToken(custom.QuarantineUserMedia), "quarantine_user", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/quarantine/server/:serverName", true, router, makeRoute(_routers.RequireAccessToken(custom.QuarantineDomainMedia), "quarantine_domain", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/quarantine/media/:server/:mediaId", true, router, makeRoute(_routers.RequireAccessToken(custom.QuarantineMedia), "quarantine_media", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/datastores/:datastoreId/size_estimate", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastoreStorageEstimate), "get_storage_estimate", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/datastores/:sourceDsId/transfer_to/:targetDsId", true, router, makeRoute(_routers.RequireRepoAdmin(custom.MigrateBetweenDatastores), "datastore_transfer", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/datastores", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastores), "list_datastores", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/federation/test/:serverName", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetFederationInfo), "federation_test", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDomainUsage), "domain_usage", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUserUsage), "user_usage", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users-stats", true, router, makeRoute(_routers.RequireAccessToken(custom.GetUsersUsageStats), "users_usage_stats", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/uploads", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUploadsUsage), "uploads_usage", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/task/:taskId", true, router, makeRoute(_routers.RequireRepoAdmin(custom.GetTask), "get_background_task", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/tasks/all", true, router, makeRoute(_routers.RequireRepoAdmin(custom.ListAllTasks), "list_all_background_tasks", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/tasks/unfinished", true, router, makeRoute(_routers.RequireRepoAdmin(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/user/:userId/export", true, router, makeRoute(_routers.RequireAccessToken(custom.ExportUserData), "export_user_data", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/server/:serverName/export", true, router, makeRoute(_routers.RequireAccessToken(custom.ExportServerData), "export_server_data", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/view", true, router, makeRoute(_routers.OptionalAccessToken(custom.ViewExport), "view_export", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/metadata", true, router, makeRoute(_routers.OptionalAccessToken(custom.GetExportMetadata), "get_export_metadata", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/part/:partId", true, router, makeRoute(_routers.OptionalAccessToken(custom.DownloadExportPart), "download_export_part", false, counter))
+ register([]string{"DELETE"}, PrefixMedia, "admin/export/:exportId/delete", true, router, makeRoute(_routers.OptionalAccessToken(custom.DeleteExport), "delete_export", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/import", true, router, makeRoute(_routers.RequireRepoAdmin(custom.StartImport), "start_import", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/import/:importId/part", true, router, makeRoute(_routers.RequireRepoAdmin(custom.AppendToImport), "append_to_import", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/import/:importId/close", true, router, makeRoute(_routers.RequireRepoAdmin(custom.StopImport), "stop_import", false, counter))
+ register([]string{"GET"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", true, router, makeRoute(_routers.RequireAccessToken(custom.GetAttributes), "get_media_attributes", false, counter))
+ register([]string{"POST"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", true, router, makeRoute(_routers.RequireAccessToken(custom.SetAttributes), "set_media_attributes", false, counter))
+
+ // TODO: Register pprof
+
+ return router
+}
+
+func makeRoute(generator _routers.GeneratorFn, name string, ignoreHost bool, counter *_routers.RequestCounter) http.Handler {
+ return _routers.NewInstallMetadataRouter(ignoreHost, name, counter,
+ _routers.NewInstallHeadersRouter(
+ _routers.NewHostRouter(
+ _routers.NewMetricsRequestRouter(
+ _routers.NewRContextRouter(generator, _routers.NewMetricsResponseRouter(nil)),
+ ),
+ ),
+ ))
+}
+
+var versions = []string{"r0", "v1", "v3", "unstable", "unstable/io.t2bot.media"}
+
+func register(methods []string, prefix string, postfix string, unstableOnly bool, router *httprouter.Router, handler http.Handler) {
+ for _, method := range methods {
+ for _, version := range versions {
+ if unstableOnly && !strings.HasPrefix(version, "unstable") {
+ continue
+ }
+ path := fmt.Sprintf("%s/%s/%s", prefix, version, postfix)
+ router.Handler(method, path, handler)
+ logrus.Debug("Registering route: ", method, path)
+ }
+ }
+}
diff --git a/api/unstable/info.go b/api/unstable/info.go
index 448e0183f6e044a8db513fdd5072babf9fb7fe72..e47542e32b159b5f7074f01329e15fc9afdb4cba 100644
--- a/api/unstable/info.go
+++ b/api/unstable/info.go
@@ -3,23 +3,25 @@ package unstable
import (
"bytes"
"database/sql"
- "github.com/getsentry/sentry-go"
"io/ioutil"
"net/http"
"strconv"
"strings"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/disintegration/imaging"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/download_controller"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/thumbnailing"
"github.com/turt2live/matrix-media-repo/thumbnailing/i"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
@@ -47,18 +49,20 @@ type MediaInfoResponse struct {
NumChannels int `json:"num_channels,omitempty"`
}
-func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- server := params["server"]
- mediaId := params["mediaId"]
+func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
allowRemote := r.URL.Query().Get("allow_remote")
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
+
downloadRemote := true
if allowRemote != "" {
parsedFlag, err := strconv.ParseBool(allowRemote)
if err != nil {
- return api.InternalServerError("allow_remote flag does not appear to be a boolean")
+ return _responses.InternalServerError("allow_remote flag does not appear to be a boolean")
}
downloadRemote = parsedFlag
}
@@ -72,23 +76,23 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
streamedMedia, err := download_controller.GetMedia(server, mediaId, downloadRemote, true, rctx)
if err != nil {
if err == common.ErrMediaNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
} else if err == common.ErrMediaTooLarge {
- return api.RequestTooLarge()
+ return _responses.RequestTooLarge()
} else if err == common.ErrMediaQuarantined {
- return api.NotFoundError() // We lie for security
+ return _responses.NotFoundError() // We lie for security
}
rctx.Log.Error("Unexpected error locating media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
- defer cleanup.DumpAndCloseStream(streamedMedia.Stream)
+ defer stream_util.DumpAndCloseStream(streamedMedia.Stream)
b, err := ioutil.ReadAll(streamedMedia.Stream)
if err != nil {
rctx.Log.Error("Unexpected error processing media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
response := &MediaInfoResponse{
@@ -111,7 +115,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
if err != nil && err != sql.ErrNoRows {
rctx.Log.Error("Unexpected error locating media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
if thumbs != nil && len(thumbs) > 0 {
diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go
index 30d5f26fb20c2d29cf3ebb9aae86f04d19cc0e67..74166e60af6778a7cf350aaeea6bbdc54d2d3b2e 100644
--- a/api/unstable/local_copy.go
+++ b/api/unstable/local_copy.go
@@ -2,32 +2,36 @@ package unstable
import (
"github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/api/_apimeta"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/api/_routers"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"net/http"
"strconv"
- "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/api/r0"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/download_controller"
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
-func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
- params := mux.Vars(r)
-
- server := params["server"]
- mediaId := params["mediaId"]
+func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+ server := _routers.GetParam("server", r)
+ mediaId := _routers.GetParam("mediaId", r)
allowRemote := r.URL.Query().Get("allow_remote")
+ if !_routers.ServerNameRegex.MatchString(server) {
+ return _responses.BadRequest("invalid server ID")
+ }
+
downloadRemote := true
if allowRemote != "" {
parsedFlag, err := strconv.ParseBool(allowRemote)
if err != nil {
- return api.InternalServerError("allow_remote flag does not appear to be a boolean")
+ return _responses.InternalServerError("allow_remote flag does not appear to be a boolean")
}
downloadRemote = parsedFlag
}
@@ -43,17 +47,17 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
streamedMedia, err := download_controller.GetMedia(server, mediaId, downloadRemote, true, rctx)
if err != nil {
if err == common.ErrMediaNotFound {
- return api.NotFoundError()
+ return _responses.NotFoundError()
} else if err == common.ErrMediaTooLarge {
- return api.RequestTooLarge()
+ return _responses.RequestTooLarge()
} else if err == common.ErrMediaQuarantined {
- return api.NotFoundError() // We lie for security
+ return _responses.NotFoundError() // We lie for security
}
rctx.Log.Error("Unexpected error locating media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
- defer cleanup.DumpAndCloseStream(streamedMedia.Stream)
+ defer stream_util.DumpAndCloseStream(streamedMedia.Stream)
// Don't clone the media if it's already available on this domain
if streamedMedia.KnownMedia.Origin == r.Host {
@@ -64,7 +68,7 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
if err != nil {
rctx.Log.Error("Unexpected error storing media: " + err.Error())
sentry.CaptureException(err)
- return api.InternalServerError("Unexpected Error")
+ return _responses.InternalServerError("Unexpected Error")
}
return &r0.MediaUploadedResponse{ContentUri: newMedia.MxcUri()}
diff --git a/api/webserver.go b/api/webserver.go
new file mode 100644
index 0000000000000000000000000000000000000000..ed9ae05c4ee4602fe468acc785dae4083662b21a
--- /dev/null
+++ b/api/webserver.go
@@ -0,0 +1,90 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "net/http"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/didip/tollbooth"
+ "github.com/getsentry/sentry-go"
+ sentryhttp "github.com/getsentry/sentry-go/http"
+ "github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/api/_responses"
+ "github.com/turt2live/matrix-media-repo/common/config"
+)
+
+var srv *http.Server
+var waitGroup = &sync.WaitGroup{}
+var reload = false
+
+func Init() *sync.WaitGroup {
+ address := net.JoinHostPort(config.Get().General.BindAddress, strconv.Itoa(config.Get().General.Port))
+
+ //defer func() {
+ // if err := recover(); err != nil {
+ // logrus.Fatal(err)
+ // }
+ //}()
+
+ handler := buildRoutes()
+
+ if config.Get().RateLimit.Enabled {
+ logrus.Debug("Enabling rate limit")
+ limiter := tollbooth.NewLimiter(0, nil)
+ limiter.SetIPLookups([]string{"X-Forwarded-For", "X-Real-IP", "RemoteAddr"})
+ limiter.SetTokenBucketExpirationTTL(time.Hour)
+ limiter.SetBurst(config.Get().RateLimit.BurstCount)
+ limiter.SetMax(config.Get().RateLimit.RequestsPerSecond)
+
+ b, _ := json.Marshal(_responses.RateLimitReached())
+ limiter.SetMessage(string(b))
+ limiter.SetMessageContentType("application/json")
+
+ handler = tollbooth.LimitHandler(limiter, handler)
+ }
+
+ // Note: we bind Sentry here to ensure we capture *everything*
+ sentryHandler := sentryhttp.New(sentryhttp.Options{})
+ srv = &http.Server{Addr: address, Handler: sentryHandler.Handle(handler)}
+ reload = false
+
+ go func() {
+ logrus.WithField("address", address).Info("Started up. Listening at http://" + address)
+ if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+ sentry.CaptureException(err)
+ logrus.Fatal(err)
+ }
+
+ // Only notify the main thread that we're done if we're actually done
+ srv = nil
+ if !reload {
+ waitGroup.Done()
+ }
+ }()
+
+ return waitGroup
+}
+
+func Reload() {
+ reload = true
+
+ // Stop the server first
+ Stop()
+
+ // Reload the web server, ignoring the wait group (because we don't care to wait here)
+ Init()
+}
+
+func Stop() {
+ if srv != nil {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := srv.Shutdown(ctx); err != nil {
+ panic(err)
+ }
+ }
+}
diff --git a/api/webserver/debug/pprof.go b/api/webserver/debug/pprof.go
index 0635b96b019e1e54f33fd5a9194ffd1aea0db9a1..ba22f0abf7ad55990147060faca84c210c62f615 100644
--- a/api/webserver/debug/pprof.go
+++ b/api/webserver/debug/pprof.go
@@ -4,36 +4,50 @@ import (
"encoding/json"
"net/http"
"net/http/pprof"
+
+ "github.com/julienschmidt/httprouter"
)
-func BindPprofEndpoints(httpMux *http.ServeMux, secret string) {
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/allocs", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/block", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/cmdline", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/goroutine", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/heap", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/mutex", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/profile", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/threadcreate", pprofServe(pprof.Index, secret))
- httpMux.HandleFunc("/_matrix/media/unstable/io.t2bot/debug/pprof/trace", pprofServe(pprof.Index, secret))
+func BindPprofEndpoints(httpMux *httprouter.Router, secret string) {
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/allocs", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/block", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/cmdline", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/goroutine", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/heap", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/mutex", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/profile", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/threadcreate", pprofServe(pprof.Index, secret))
+ httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/trace", pprofServe(pprof.Index, secret))
}
-func pprofServe(fn func(http.ResponseWriter, *http.Request), secret string) func(http.ResponseWriter, *http.Request) {
- return func(w http.ResponseWriter, r *http.Request) {
- auth := r.Header.Get("Authorization")
- if auth != ("Bearer " + secret) {
- // Order is important: Set headers before sending responses
- w.Header().Set("Content-Type", "application/json; charset=UTF-8")
- w.WriteHeader(http.StatusUnauthorized)
-
- encoder := json.NewEncoder(w)
- encoder.Encode(&map[string]bool{"success": false})
- return
- }
-
- // otherwise authed fine
- r.URL.Path = r.URL.Path[len("/_matrix/media/unstable/io.t2bot"):]
- fn(w, r)
+type generatorFn = func(w http.ResponseWriter, r *http.Request)
+
+type requestContainer struct {
+ secret string
+ fn generatorFn
+}
+
+func pprofServe(fn generatorFn, secret string) http.Handler {
+ return &requestContainer{
+ secret: secret,
+ fn: fn,
}
}
+
+func (c *requestContainer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ auth := r.Header.Get("Authorization")
+ if auth != ("Bearer " + c.secret) {
+ // Order is important: Set headers before sending responses
+ w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+ w.WriteHeader(http.StatusUnauthorized)
+
+ encoder := json.NewEncoder(w)
+ _ = encoder.Encode(&map[string]bool{"success": false})
+ return
+ }
+
+ // otherwise authed fine
+ r.URL.Path = r.URL.Path[len("/_matrix/media/unstable/io.t2bot"):]
+ c.fn(w, r)
+}
diff --git a/api/webserver/request_counter.go b/api/webserver/request_counter.go
deleted file mode 100644
index bc411c58b61f74484015d05f9d1dec2f043fd8c6..0000000000000000000000000000000000000000
--- a/api/webserver/request_counter.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package webserver
-
-import "strconv"
-
-type requestCounter struct {
- lastId uint64
-}
-
-func (c *requestCounter) GetNextId() string {
- strId := strconv.FormatUint(c.lastId, 10)
- c.lastId = c.lastId + 1
-
- return "REQ-" + strId
-}
diff --git a/api/webserver/route_handler.go b/api/webserver/route_handler.go
deleted file mode 100644
index b67cf515ed06aed5c42828dee6998d8a4b60fe80..0000000000000000000000000000000000000000
--- a/api/webserver/route_handler.go
+++ /dev/null
@@ -1,402 +0,0 @@
-package webserver
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "math"
- "mime"
- "net"
- "net/http"
- "net/url"
- "strconv"
- "strings"
-
- "github.com/getsentry/sentry-go"
-
- "github.com/alioygur/is"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sebest/xff"
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
- "github.com/turt2live/matrix-media-repo/api/r0"
- "github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/metrics"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type handler struct {
- h func(r *http.Request, ctx rcontext.RequestContext) interface{}
- action string
- reqCounter *requestCounter
- ignoreHost bool
-}
-
-func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- isUsingForwardedHost := false
- if r.Header.Get("X-Forwarded-Host") != "" && config.Get().General.UseForwardedHost {
- r.Host = r.Header.Get("X-Forwarded-Host")
- isUsingForwardedHost = true
- }
- r.Host = strings.Split(r.Host, ":")[0]
-
- var raddr string
- if config.Get().General.TrustAnyForward {
- raddr = r.Header.Get("X-Forwarded-For")
- } else {
- raddr = xff.GetRemoteAddr(r)
- }
- if raddr == "" {
- raddr = r.RemoteAddr
- }
-
- host, _, err := net.SplitHostPort(raddr)
- if err != nil {
- logrus.Error(err)
- sentry.CaptureException(err)
- host = raddr
- }
- r.RemoteAddr = host
-
- contextLog := logrus.WithFields(logrus.Fields{
- "method": r.Method,
- "host": r.Host,
- "usingForwardedHost": isUsingForwardedHost,
- "resource": r.URL.Path,
- "contentType": r.Header.Get("Content-Type"),
- "contentLength": r.ContentLength,
- "queryString": util.GetLogSafeQueryString(r),
- "requestId": h.reqCounter.GetNextId(),
- "remoteAddr": r.RemoteAddr,
- })
- contextLog.Info("Received request")
-
- // Send CORS and other basic headers
- w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
- w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
- w.Header().Set("Access-Control-Allow-Origin", "*")
- w.Header().Set("Content-Security-Policy", "sandbox; default-src 'none'; script-src 'none'; plugin-types application/pdf; style-src 'unsafe-inline'; media-src 'self'; object-src 'self';")
- w.Header().Set("Cross-Origin-Resource-Policy", "cross-origin")
- w.Header().Set("X-Content-Security-Policy", "sandbox;")
- w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex")
- w.Header().Set("Server", "matrix-media-repo")
-
- // Process response
- var res interface{} = api.AuthFailed()
- var rctx rcontext.RequestContext
- if util.IsServerOurs(r.Host) || h.ignoreHost {
- contextLog.Info("Host is valid - processing request")
- cfg := config.GetDomain(r.Host)
- if h.ignoreHost {
- dc := config.DomainConfigFrom(*config.Get())
- cfg = &dc
- }
-
- // Build a context that can be used throughout the remainder of the app
- // This is kinda annoying, but it's better than trying to pass our own
- // thing throughout the layers.
- ctx := r.Context()
- ctx = context.WithValue(ctx, "mr.logger", contextLog)
- ctx = context.WithValue(ctx, "mr.serverConfig", cfg)
- ctx = context.WithValue(ctx, "mr.request", r)
- rctx = rcontext.RequestContext{Context: ctx, Log: contextLog, Config: *cfg, Request: r}
- r = r.WithContext(rctx)
-
- metrics.HttpRequests.With(prometheus.Labels{
- "host": r.Host,
- "action": h.action,
- "method": r.Method,
- }).Inc()
- res = h.h(r, rctx)
- if res == nil {
- res = &api.EmptyResponse{}
- }
- } else {
- metrics.InvalidHttpRequests.With(prometheus.Labels{
- "action": h.action,
- "method": r.Method,
- }).Inc()
- contextLog.Warn("The server name provided in the Host header is not configured, or the request was made directly to the media repo instead of through your reverse proxy. This request is being rejected.")
- }
- if res == nil {
- res = api.InternalServerError("Error processing response")
- }
-
- switch result := res.(type) {
- case *api.DoNotCacheResponse:
- res = result.Payload
- break
- }
-
- htmlRes, isHtml := res.(*api.HtmlResponse)
- if isHtml {
- contextLog.Info(fmt.Sprintf("Replying with result: %T %+v", res, fmt.Sprintf("<%d chars of html>", len(htmlRes.HTML))))
- } else {
- contextLog.Info(fmt.Sprintf("Replying with result: %T %+v", res, res))
- }
-
- statusCode := http.StatusOK
- switch result := res.(type) {
- case *api.ErrorResponse:
- switch result.InternalCode {
- case common.ErrCodeUnknownToken:
- statusCode = http.StatusUnauthorized
- break
- case common.ErrCodeNotFound:
- statusCode = http.StatusNotFound
- break
- case common.ErrCodeMediaTooLarge:
- statusCode = http.StatusRequestEntityTooLarge
- break
- case common.ErrCodeBadRequest:
- statusCode = http.StatusBadRequest
- break
- case common.ErrCodeMethodNotAllowed:
- statusCode = http.StatusMethodNotAllowed
- break
- case common.ErrCodeForbidden:
- statusCode = http.StatusForbidden
- break
- default: // Treat as unknown (a generic server error)
- statusCode = http.StatusInternalServerError
- break
- }
- break
- case *r0.DownloadMediaResponse:
- // XXX: This range parsing isn't perfect, but works fine enough for now
- rangeStart := int64(0)
- rangeEnd := int64(0)
- grabBytes := int64(0)
- doRange := false
- if r.Header.Get("Range") != "" && result.SizeBytes > 0 && rctx.Request != nil && config.Get().Redis.Enabled {
- rnge := r.Header.Get("Range")
- if !strings.HasPrefix(rnge, "bytes=") {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper range units")
- break
- }
- if !strings.Contains(rnge, ",") && !strings.HasPrefix(rnge, "bytes=-") {
- parts := strings.Split(rnge[len("bytes="):], "-")
- if len(parts) <= 2 {
- rstart, err := strconv.ParseInt(parts[0], 10, 64)
- if err != nil {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper start of range")
- break
- }
-
- if rstart < 0 {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper start of range: negative")
- break
- }
-
- rend := int64(-1)
- if len(parts) > 1 && parts[1] != "" {
- rend, err = strconv.ParseInt(parts[1], 10, 64)
- if err != nil {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper end of range")
- break
- }
-
- if rend < 1 {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper end of range: negative")
- break
- }
-
- if rend >= result.SizeBytes {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Improper end of range: out of bounds")
- break
- }
-
- if rend <= rstart {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Start must be before end")
- break
- }
-
- if (rstart + rend) >= result.SizeBytes {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Range too large")
- break
- }
-
- grabBytes = rend - rstart
- } else {
- add := int64(10485760) // 10mb default
- if rctx.Config.Downloads.DefaultRangeChunkSizeBytes > 0 {
- add = rctx.Config.Downloads.DefaultRangeChunkSizeBytes
- }
- rend = int64(math.Min(float64(rstart+add), float64(result.SizeBytes-1)))
- grabBytes = (rend - rstart) + 1
- }
-
- rangeStart = rstart
- rangeEnd = rend
-
- if (rangeEnd-rangeStart) <= 0 || grabBytes <= 0 {
- statusCode = http.StatusRequestedRangeNotSatisfiable
- res = api.BadRequest("Range invalid at last pass")
- break
- }
-
- doRange = true
- }
- }
- }
-
- metrics.HttpResponses.With(prometheus.Labels{
- "host": r.Host,
- "action": h.action,
- "method": r.Method,
- "statusCode": strconv.Itoa(http.StatusOK),
- }).Inc()
-
- contentType := result.ContentType
- mediaType, params, err := mime.ParseMediaType(result.ContentType)
- if err != nil {
- sentry.CaptureException(err)
- contextLog.Warn("Failed to parse content type header for media on reply: " + err.Error())
- } else {
- // TODO: Maybe we only strip the charset from images? Is it valid to have the param on other types?
- if !strings.HasPrefix(mediaType, "text/") && mediaType != "application/json" {
- delete(params, "charset")
- }
- contentType = mime.FormatMediaType(mediaType, params)
- }
-
- w.Header().Set("Cache-Control", "private, max-age=259200") // 3 days
- w.Header().Set("Content-Type", contentType)
- if result.SizeBytes > 0 {
- if config.Get().Redis.Enabled {
- w.Header().Set("Accept-Ranges", "bytes")
- }
- w.Header().Set("Content-Length", fmt.Sprint(result.SizeBytes))
- }
- disposition := result.TargetDisposition
- if disposition == "" {
- disposition = "inline"
- } else if disposition == "infer" {
- if result.ContentType == "" {
- disposition = "attachment"
- } else {
- if util.HasAnyPrefix(result.ContentType, []string{"image/", "audio/", "video/", "text/plain"}) {
- disposition = "inline"
- } else {
- disposition = "attachment"
- }
- }
- }
- fname := result.Filename
- if fname == "" {
- exts, err := mime.ExtensionsByType(result.ContentType)
- if err != nil {
- exts = nil
- contextLog.Warn("Unexpected error inferring file extension: " + err.Error())
- sentry.CaptureException(err)
- }
- ext := ""
- if exts != nil && len(exts) > 0 {
- ext = exts[0]
- }
- fname = "file" + ext
- }
- if is.ASCII(result.Filename) {
- w.Header().Set("Content-Disposition", disposition+"; filename="+url.QueryEscape(fname))
- } else {
- w.Header().Set("Content-Disposition", disposition+"; filename*=utf-8''"+url.QueryEscape(fname))
- }
-
- defer result.Data.Close()
-
- if doRange {
- _, err = io.CopyN(ioutil.Discard, result.Data, rangeStart)
- if err != nil {
- // Should only blow up this request
- panic(err)
- }
-
- expectedBytes := grabBytes
- w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", rangeStart, rangeEnd, result.SizeBytes))
- w.Header().Set("Content-Length", fmt.Sprint(expectedBytes))
- w.WriteHeader(http.StatusPartialContent)
- b, err := io.CopyN(w, result.Data, expectedBytes)
- if err != nil {
- // Should only blow up this request
- panic(err)
- }
-
- // Discard anything that remains
- _, _ = io.Copy(ioutil.Discard, result.Data)
-
- if expectedBytes > 0 && b != expectedBytes {
- // Should only blow up this request
- panic(errors.New("mismatch transfer size"))
- }
- } else {
- writeResponseData(w, result.Data, result.SizeBytes)
- }
- return // Prevent sending conflicting responses
- case *r0.IdenticonResponse:
- metrics.HttpResponses.With(prometheus.Labels{
- "host": r.Host,
- "action": h.action,
- "method": r.Method,
- "statusCode": strconv.Itoa(http.StatusOK),
- }).Inc()
- w.Header().Set("Cache-Control", "private, max-age=604800") // 7 days
- w.Header().Set("Content-Type", "image/png")
- writeResponseData(w, result.Avatar, 0)
- return // Prevent sending conflicting responses
- case *api.HtmlResponse:
- metrics.HttpResponses.With(prometheus.Labels{
- "host": r.Host,
- "action": h.action,
- "method": r.Method,
- "statusCode": strconv.Itoa(http.StatusOK),
- }).Inc()
- w.Header().Set("Cache-Control", "private, max-age=259200") // 3 days
- w.Header().Set("Content-Type", "text/html; charset=UTF-8")
- w.Header().Set("Content-Security-Policy", "") // We're serving HTML, so take away the CSP
- w.Header().Set("X-Content-Security-Policy", "") // We're serving HTML, so take away the CSP
- io.Copy(w, bytes.NewBuffer([]byte(result.HTML)))
- return
- default:
- break
- }
-
- metrics.HttpResponses.With(prometheus.Labels{
- "host": r.Host,
- "action": h.action,
- "method": r.Method,
- "statusCode": strconv.Itoa(statusCode),
- }).Inc()
-
- // Order is important: Set headers before sending responses
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(statusCode)
-
- encoder := json.NewEncoder(w)
- encoder.Encode(res)
-}
-
-func writeResponseData(w http.ResponseWriter, s io.Reader, expectedBytes int64) {
- b, err := io.Copy(w, s)
- if err != nil {
- // Should only blow up this request
- panic(err)
- }
- if expectedBytes > 0 && b != expectedBytes {
- // Should only blow up this request
- panic(errors.New("mismatch transfer size"))
- }
-}
diff --git a/api/webserver/webserver.go b/api/webserver/webserver.go
deleted file mode 100644
index 0bb083484666e7c05fb6431bc8d9ca74e0046210..0000000000000000000000000000000000000000
--- a/api/webserver/webserver.go
+++ /dev/null
@@ -1,243 +0,0 @@
-package webserver
-
-import (
- "context"
- "encoding/json"
- "net"
- "net/http"
- "os"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/getsentry/sentry-go"
- sentryhttp "github.com/getsentry/sentry-go/http"
-
- "github.com/didip/tollbooth"
- "github.com/gorilla/mux"
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api"
- "github.com/turt2live/matrix-media-repo/api/custom"
- "github.com/turt2live/matrix-media-repo/api/r0"
- "github.com/turt2live/matrix-media-repo/api/unstable"
- "github.com/turt2live/matrix-media-repo/api/webserver/debug"
- "github.com/turt2live/matrix-media-repo/common/config"
-)
-
-type route struct {
- method string
- handler handler
-}
-
-type definedRoute struct {
- path string
- route route
-}
-
-var srv *http.Server
-var waitGroup = &sync.WaitGroup{}
-var reload = false
-
-func Init() *sync.WaitGroup {
- rtr := mux.NewRouter()
- counter := &requestCounter{}
-
- optionsHandler := handler{api.EmptyResponseHandler, "options_request", counter, false}
- uploadHandler := handler{api.AccessTokenRequiredRoute(r0.UploadMedia), "upload", counter, false}
- downloadHandler := handler{api.AccessTokenOptionalRoute(r0.DownloadMedia), "download", counter, false}
- thumbnailHandler := handler{api.AccessTokenOptionalRoute(r0.ThumbnailMedia), "thumbnail", counter, false}
- previewUrlHandler := handler{api.AccessTokenRequiredRoute(r0.PreviewUrl), "url_preview", counter, false}
- identiconHandler := handler{api.AccessTokenOptionalRoute(r0.Identicon), "identicon", counter, false}
- purgeRemote := handler{api.RepoAdminRoute(custom.PurgeRemoteMedia), "purge_remote_media", counter, false}
- purgeOneHandler := handler{api.AccessTokenRequiredRoute(custom.PurgeIndividualRecord), "purge_individual_media", counter, false}
- purgeQuarantinedHandler := handler{api.AccessTokenRequiredRoute(custom.PurgeQuarantined), "purge_quarantined", counter, false}
- purgeUserMediaHandler := handler{api.AccessTokenRequiredRoute(custom.PurgeUserMedia), "purge_user_media", counter, false}
- purgeRoomHandler := handler{api.AccessTokenRequiredRoute(custom.PurgeRoomMedia), "purge_room_media", counter, false}
- purgeDomainHandler := handler{api.AccessTokenRequiredRoute(custom.PurgeDomainMedia), "purge_domain_media", counter, false}
- purgeOldHandler := handler{api.RepoAdminRoute(custom.PurgeOldMedia), "purge_old_media", counter, false}
- quarantineHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineMedia), "quarantine_media", counter, false}
- quarantineRoomHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineRoomMedia), "quarantine_room", counter, false}
- quarantineUserHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineUserMedia), "quarantine_user", counter, false}
- quarantineDomainHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineDomainMedia), "quarantine_domain", counter, false}
- localCopyHandler := handler{api.AccessTokenRequiredRoute(unstable.LocalCopy), "local_copy", counter, false}
- infoHandler := handler{api.AccessTokenRequiredRoute(unstable.MediaInfo), "info", counter, false}
- configHandler := handler{api.AccessTokenRequiredRoute(r0.PublicConfig), "config", counter, false}
- storageEstimateHandler := handler{api.RepoAdminRoute(custom.GetDatastoreStorageEstimate), "get_storage_estimate", counter, false}
- datastoreListHandler := handler{api.RepoAdminRoute(custom.GetDatastores), "list_datastores", counter, false}
- dsTransferHandler := handler{api.RepoAdminRoute(custom.MigrateBetweenDatastores), "datastore_transfer", counter, false}
- fedTestHandler := handler{api.RepoAdminRoute(custom.GetFederationInfo), "federation_test", counter, false}
- healthzHandler := handler{api.AccessTokenOptionalRoute(custom.GetHealthz), "healthz", counter, true}
- domainUsageHandler := handler{api.RepoAdminRoute(custom.GetDomainUsage), "domain_usage", counter, false}
- userUsageHandler := handler{api.RepoAdminRoute(custom.GetUserUsage), "user_usage", counter, false}
- uploadsUsageHandler := handler{api.RepoAdminRoute(custom.GetUploadsUsage), "uploads_usage", counter, false}
- usersUsageStatsHandler := handler{api.AccessTokenRequiredRoute(custom.GetUsersUsageStats), "users_usage_stats", counter, false}
- getBackgroundTaskHandler := handler{api.RepoAdminRoute(custom.GetTask), "get_background_task", counter, false}
- listAllBackgroundTasksHandler := handler{api.RepoAdminRoute(custom.ListAllTasks), "list_all_background_tasks", counter, false}
- listUnfinishedBackgroundTasksHandler := handler{api.RepoAdminRoute(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", counter, false}
- exportUserDataHandler := handler{api.AccessTokenRequiredRoute(custom.ExportUserData), "export_user_data", counter, false}
- exportServerDataHandler := handler{api.AccessTokenRequiredRoute(custom.ExportServerData), "export_server_data", counter, false}
- viewExportHandler := handler{api.AccessTokenOptionalRoute(custom.ViewExport), "view_export", counter, false}
- getExportMetadataHandler := handler{api.AccessTokenOptionalRoute(custom.GetExportMetadata), "get_export_metadata", counter, false}
- downloadExportPartHandler := handler{api.AccessTokenOptionalRoute(custom.DownloadExportPart), "download_export_part", counter, false}
- deleteExportHandler := handler{api.AccessTokenOptionalRoute(custom.DeleteExport), "delete_export", counter, false}
- startImportHandler := handler{api.RepoAdminRoute(custom.StartImport), "start_import", counter, false}
- appendToImportHandler := handler{api.RepoAdminRoute(custom.AppendToImport), "append_to_import", counter, false}
- stopImportHandler := handler{api.RepoAdminRoute(custom.StopImport), "stop_import", counter, false}
- versionHandler := handler{api.AccessTokenOptionalRoute(custom.GetVersion), "get_version", counter, false}
- logoutHandler := handler{api.AccessTokenRequiredRoute(r0.Logout), "logout", counter, false}
- logoutAllHandler := handler{api.AccessTokenRequiredRoute(r0.LogoutAll), "logout_all", counter, false}
- getMediaAttrsHandler := handler{api.AccessTokenRequiredRoute(custom.GetAttributes), "get_media_attributes", counter, false}
- setMediaAttrsHandler := handler{api.AccessTokenRequiredRoute(custom.SetAttributes), "set_media_attributes", counter, false}
-
- routes := make([]definedRoute, 0)
- // r0 is typically clients and v1 is typically servers. v1 is deprecated.
- // unstable is, well, unstable. unstable/io.t2bot.media is to comply with MSC2324
- // v3 is Matrix 1.1 stuff
- versions := []string{"r0", "v1", "v3", "unstable", "unstable/io.t2bot.media"}
-
- // Things that don't need a version
- routes = append(routes, definedRoute{"/_matrix/media/version", route{"GET", versionHandler}})
-
- for _, version := range versions {
- // Standard routes we have to handle
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/upload", route{"POST", uploadHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/download/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}/{filename:.+}", route{"GET", downloadHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/download/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"GET", downloadHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/thumbnail/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"GET", thumbnailHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/preview_url", route{"GET", previewUrlHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/identicon/{seed:.*}", route{"GET", identiconHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/config", route{"GET", configHandler}})
- routes = append(routes, definedRoute{"/_matrix/client/" + version + "/logout", route{"POST", logoutHandler}})
- routes = append(routes, definedRoute{"/_matrix/client/" + version + "/logout/all", route{"POST", logoutAllHandler}})
-
- // Routes that we define but are not part of the spec (management)
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge_remote", route{"POST", purgeRemote}}) // deprecated
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/remote", route{"POST", purgeRemote}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/quarantined", route{"POST", purgeQuarantinedHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/user/{userId:[^/]+}", route{"POST", purgeUserMediaHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/room/{roomId:[^/]+}", route{"POST", purgeRoomHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/server/{serverName:[^/]+}", route{"POST", purgeDomainHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/old", route{"POST", purgeOldHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/purge/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"POST", purgeOneHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/room/{roomId:[^/]+}/quarantine", route{"POST", quarantineRoomHandler}}) // deprecated
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/quarantine/room/{roomId:[^/]+}", route{"POST", quarantineRoomHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/quarantine/user/{userId:[^/]+}", route{"POST", quarantineUserHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/quarantine/server/{serverName:[^/]+}", route{"POST", quarantineDomainHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/quarantine/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"POST", quarantineHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/datastores/{datastoreId:[^/]+}/size_estimate", route{"GET", storageEstimateHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/datastores", route{"GET", datastoreListHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/datastores/{sourceDsId:[^/]+}/transfer_to/{targetDsId:[^/]+}", route{"POST", dsTransferHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/federation/test/{serverName:[a-zA-Z0-9.:\\-_]+}", route{"GET", fedTestHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/usage/{serverName:[a-zA-Z0-9.:\\-_]+}", route{"GET", domainUsageHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/usage/{serverName:[a-zA-Z0-9.:\\-_]+}/users", route{"GET", userUsageHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/usage/{serverName:[a-zA-Z0-9.:\\-_]+}/users-stats", route{"GET", usersUsageStatsHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/usage/{serverName:[a-zA-Z0-9.:\\-_]+}/uploads", route{"GET", uploadsUsageHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/tasks/{taskId:[0-9]+}", route{"GET", getBackgroundTaskHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/tasks/all", route{"GET", listAllBackgroundTasksHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/tasks/unfinished", route{"GET", listUnfinishedBackgroundTasksHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/user/{userId:[^/]+}/export", route{"POST", exportUserDataHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/server/{serverName:[^/]+}/export", route{"POST", exportServerDataHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/export/{exportId:[a-zA-Z0-9.:\\-_]+}/view", route{"GET", viewExportHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/export/{exportId:[a-zA-Z0-9.:\\-_]+}/metadata", route{"GET", getExportMetadataHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/export/{exportId:[a-zA-Z0-9.:\\-_]+}/part/{partId:[0-9]+}", route{"GET", downloadExportPartHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/export/{exportId:[a-zA-Z0-9.:\\-_]+}/delete", route{"DELETE", deleteExportHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/import", route{"POST", startImportHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/import/{importId:[a-zA-Z0-9.:\\-_]+}/part", route{"POST", appendToImportHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/import/{importId:[a-zA-Z0-9.:\\-_]+}/close", route{"POST", stopImportHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/media/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}/attributes", route{"GET", getMediaAttrsHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/admin/media/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}/attributes/set", route{"POST", setMediaAttrsHandler}})
-
- // Routes that we should handle but aren't in the media namespace (synapse compat)
- routes = append(routes, definedRoute{"/_matrix/client/" + version + "/admin/purge_media_cache", route{"POST", purgeRemote}})
- routes = append(routes, definedRoute{"/_matrix/client/" + version + "/admin/quarantine_media/{roomId:[^/]+}", route{"POST", quarantineRoomHandler}})
-
- if strings.Index(version, "unstable") == 0 {
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/local_copy/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"GET", localCopyHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/info/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"GET", infoHandler}})
- routes = append(routes, definedRoute{"/_matrix/media/" + version + "/download/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[^/]+}", route{"DELETE", purgeOneHandler}})
- }
- }
-
- for _, def := range routes {
- logrus.Info("Registering route: " + def.route.method + " " + def.path)
- rtr.Handle(def.path, def.route.handler).Methods(def.route.method)
- rtr.Handle(def.path, optionsHandler).Methods("OPTIONS")
-
- // This is a hack to a ensure that trailing slashes also match the routes correctly
- rtr.Handle(def.path+"/", def.route.handler).Methods(def.route.method)
- rtr.Handle(def.path+"/", optionsHandler).Methods("OPTIONS")
- }
-
- // Health check endpoints
- rtr.Handle("/healthz", healthzHandler).Methods("OPTIONS", "GET", "HEAD")
-
- rtr.NotFoundHandler = handler{api.NotFoundHandler, "not_found", counter, true}
- rtr.MethodNotAllowedHandler = handler{api.MethodNotAllowedHandler, "method_not_allowed", counter, true}
-
- var handler http.Handler = rtr
- if config.Get().RateLimit.Enabled {
- logrus.Info("Enabling rate limit")
- limiter := tollbooth.NewLimiter(0, nil)
- limiter.SetIPLookups([]string{"X-Forwarded-For", "X-Real-IP", "RemoteAddr"})
- limiter.SetTokenBucketExpirationTTL(time.Hour)
- limiter.SetBurst(config.Get().RateLimit.BurstCount)
- limiter.SetMax(config.Get().RateLimit.RequestsPerSecond)
-
- b, _ := json.Marshal(api.RateLimitReached())
- limiter.SetMessage(string(b))
- limiter.SetMessageContentType("application/json")
-
- handler = tollbooth.LimitHandler(limiter, rtr)
- }
-
- address := net.JoinHostPort(config.Get().General.BindAddress, strconv.Itoa(config.Get().General.Port))
- httpMux := http.NewServeMux()
- httpMux.Handle("/", handler)
-
- pprofSecret := os.Getenv("MEDIA_PPROF_SECRET_KEY")
- if pprofSecret != "" {
- logrus.Warn("Enabling pprof endpoints")
- debug.BindPprofEndpoints(httpMux, pprofSecret)
- }
-
- sentryHandler := sentryhttp.New(sentryhttp.Options{})
- srv = &http.Server{Addr: address, Handler: sentryHandler.Handle(httpMux)}
- reload = false
-
- go func() {
- logrus.WithField("address", address).Info("Started up. Listening at http://" + address)
- if err := srv.ListenAndServe(); err != http.ErrServerClosed {
- sentry.CaptureException(err)
- logrus.Fatal(err)
- }
-
- // Only notify the main thread that we're done if we're actually done
- srv = nil
- if !reload {
- waitGroup.Done()
- }
- }()
-
- return waitGroup
-}
-
-func Reload() {
- reload = true
-
- // Stop the server first
- Stop()
-
- // Reload the web server, ignoring the wait group (because we don't care to wait here)
- Init()
-}
-
-func Stop() {
- if srv != nil {
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- if err := srv.Shutdown(ctx); err != nil {
- panic(err)
- }
- }
-}
diff --git a/archival/v2_export.go b/archival/v2_export.go
index f1ce9ca6d9196e743daea95612ea8bf88099dc66..0e504fe65dacbe29bbd0bde9055cb9d3225efb1a 100644
--- a/archival/v2_export.go
+++ b/archival/v2_export.go
@@ -15,6 +15,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/templating"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
type V2ArchiveWriter interface {
@@ -49,7 +50,7 @@ func NewV2Export(exportId string, entity string, partSize int64, writer V2Archiv
entity: entity,
writer: writer,
partSize: partSize,
- ctx: ctx,
+ ctx: ctx,
indexModel: &templating.ExportIndexModel{
Entity: entity,
ExportID: exportId,
@@ -91,7 +92,7 @@ func (e *V2ArchiveExport) persistTar() error {
archiver.Name = "export-manifest.tar"
}
- if _, err := io.Copy(archiver, util.ClonedBufReader(*e.currentTarBytes)); err != nil {
+ if _, err := io.Copy(archiver, stream_util.ClonedBufReader(*e.currentTarBytes)); err != nil {
return err
}
_ = archiver.Close()
diff --git a/cluster/idgen.go b/cluster/idgen.go
index 54c8db15a5c53a5b7f147d35d75edae06096a277..e74c2e847b4cb335484a096dd98868ec750992ec 100644
--- a/cluster/idgen.go
+++ b/cluster/idgen.go
@@ -3,12 +3,13 @@ package cluster
import (
"errors"
"fmt"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
"io/ioutil"
"net/http"
"time"
+
+ "github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func GetId() (string, error) {
@@ -26,7 +27,7 @@ func GetId() (string, error) {
if err != nil {
return "", err
}
- defer cleanup.DumpAndCloseStream(res.Body)
+ defer stream_util.DumpAndCloseStream(res.Body)
contents, err := ioutil.ReadAll(res.Body)
if err != nil {
diff --git a/cmd/export_synapse_for_import/main.go b/cmd/export_synapse_for_import/main.go
index 1a9cb49f6675b63777002411458e907b24914942..8b5e6546dfeb889050124660b344d20288293ac4 100644
--- a/cmd/export_synapse_for_import/main.go
+++ b/cmd/export_synapse_for_import/main.go
@@ -19,6 +19,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/synapse"
"github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
"golang.org/x/crypto/ssh/terminal"
)
@@ -126,7 +127,7 @@ func main() {
_ = f.Close()
temp := bytes.NewBuffer(d.Bytes())
- sha256, err := util.GetSha256HashOfStream(ioutil.NopCloser(temp))
+ sha256, err := stream_util.GetSha256HashOfStream(ioutil.NopCloser(temp))
if err != nil {
logrus.Fatal(err)
}
diff --git a/cmd/gdpr_export/main.go b/cmd/gdpr_export/main.go
index 4b0e4bfad049caf72a8d0fa7785abfeeead17af1..0b5887e06dec24d228750797c8d30c79028880d5 100644
--- a/cmd/gdpr_export/main.go
+++ b/cmd/gdpr_export/main.go
@@ -17,7 +17,7 @@ import (
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func main() {
@@ -123,8 +123,8 @@ func main() {
if err != nil {
panic(err)
}
- cleanup.DumpAndCloseStream(f)
- cleanup.DumpAndCloseStream(s)
+ stream_util.DumpAndCloseStream(f)
+ stream_util.DumpAndCloseStream(s)
}
logrus.Info("Deleting export now that it has been dumped")
diff --git a/cmd/import_synapse/main.go b/cmd/import_synapse/main.go
index 3c97b89c54404beae545d8c6a6f986fbeedd7175..4c8b246a6f40dcc8ac96b5b44dce3f53f115e6c4 100644
--- a/cmd/import_synapse/main.go
+++ b/cmd/import_synapse/main.go
@@ -22,7 +22,7 @@ import (
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/synapse"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
"golang.org/x/crypto/ssh/terminal"
)
@@ -159,7 +159,7 @@ func fetchMedia(req interface{}) interface{} {
logrus.Error(err.Error())
return nil
}
- defer cleanup.DumpAndCloseStream(body)
+ defer stream_util.DumpAndCloseStream(body)
_, err = upload_controller.StoreDirect(nil, body, -1, record.ContentType, record.UploadName, record.UserId, payload.serverName, record.MediaId, common.KindLocalMedia, ctx, false)
if err != nil {
diff --git a/cmd/media_repo/main.go b/cmd/media_repo/main.go
index ba828cb814e150e39583be0a5b5503ecd4a62939..970b1f8244f947e5be0725fb71732c8c1876e45f 100644
--- a/cmd/media_repo/main.go
+++ b/cmd/media_repo/main.go
@@ -3,9 +3,13 @@ package main
import (
"flag"
"fmt"
+ "os"
+ "os/signal"
+ "time"
+
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/api/webserver"
+ "github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/assets"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/logging"
@@ -14,9 +18,6 @@ import (
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/tasks"
- "os"
- "os/signal"
- "time"
)
func main() {
@@ -90,7 +91,7 @@ func main() {
logrus.Info("Starting media repository...")
metrics.Init()
- web := webserver.Init()
+ web := api.Init()
// Set up a function to stop everything
stopAllButWeb := func() {
@@ -117,7 +118,7 @@ func main() {
stopAllButWeb()
logrus.Info("Stopping web server...")
- webserver.Stop()
+ api.Stop()
}()
// Wait for the web server to exit nicely
diff --git a/cmd/media_repo/reloads.go b/cmd/media_repo/reloads.go
index 3e733e04b7b6f0f75fc2860e7fa3c64a55b84ca9..79cb16f4be101461585f25e38ab93905ab148b47 100644
--- a/cmd/media_repo/reloads.go
+++ b/cmd/media_repo/reloads.go
@@ -1,8 +1,8 @@
package main
import (
- "github.com/turt2live/matrix-media-repo/api/auth_cache"
- "github.com/turt2live/matrix-media-repo/api/webserver"
+ "github.com/turt2live/matrix-media-repo/api"
+ "github.com/turt2live/matrix-media-repo/api/_auth_cache"
"github.com/turt2live/matrix-media-repo/common/globals"
"github.com/turt2live/matrix-media-repo/common/runtime"
"github.com/turt2live/matrix-media-repo/internal_cache"
@@ -41,7 +41,7 @@ func reloadWebOnChan(reloadChan chan bool) {
for {
shouldReload := <-reloadChan
if shouldReload {
- webserver.Reload()
+ api.Reload()
} else {
return // received stop
}
@@ -114,7 +114,7 @@ func reloadAccessTokensOnChan(reloadChan chan bool) {
for {
shouldReload := <-reloadChan
if shouldReload {
- auth_cache.FlushCache()
+ _auth_cache.FlushCache()
}
}
}()
diff --git a/common/config/access.go b/common/config/access.go
index 9b4c80ce5da7c279af24917e548d4a887cfdec55..06f0e6560d10a189f549d9a64b1da7105663d4dd 100644
--- a/common/config/access.go
+++ b/common/config/access.go
@@ -9,7 +9,7 @@ import (
"sync"
"github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
"gopkg.in/yaml.v3"
)
@@ -105,7 +105,7 @@ func reloadConfig() (*MainRepoConfig, map[string]*DomainRepoConfig, error) {
if err != nil {
return nil, nil, err
}
- defer cleanup.DumpAndCloseStream(f)
+ defer stream_util.DumpAndCloseStream(f)
buffer, err := ioutil.ReadAll(f)
if err != nil {
diff --git a/controllers/data_controller/export_controller.go b/controllers/data_controller/export_controller.go
index de3fcca12558309022ef13d6fce6eba41ce903c7..2b50affcbbb91b7376afd8e78f68da303b59e99f 100644
--- a/controllers/data_controller/export_controller.go
+++ b/controllers/data_controller/export_controller.go
@@ -7,11 +7,13 @@ import (
"context"
"encoding/json"
"fmt"
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io"
"time"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/dustin/go-humanize"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
@@ -21,7 +23,6 @@ import (
"github.com/turt2live/matrix-media-repo/templating"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type ManifestRecord struct {
@@ -123,7 +124,7 @@ func StartUserExport(userId string, s3urls bool, includeData bool, ctx rcontext.
ctx.Context = context.Background()
db := storage.GetDatabase().GetMetadataStore(ctx)
- ds, err := datastore.PickDatastore(common.KindArchives, ctx, )
+ ds, err := datastore.PickDatastore(common.KindArchives, ctx)
if err != nil {
ctx.Log.Error(err)
sentry.CaptureException(err)
@@ -176,7 +177,7 @@ func compileArchive(exportId string, entityId string, archiveDs *datastore.Datas
gzipBytes := bytes.Buffer{}
archiver := gzip.NewWriter(&gzipBytes)
archiver.Name = fmt.Sprintf("export-part-%d.tar", part)
- _, err := io.Copy(archiver, util.BufferToStream(bytes.NewBuffer(currentTarBytes.Bytes())))
+ _, err := io.Copy(archiver, stream_util.BufferToStream(bytes.NewBuffer(currentTarBytes.Bytes())))
if err != nil {
return err
}
@@ -185,7 +186,7 @@ func compileArchive(exportId string, entityId string, archiveDs *datastore.Datas
ctx.Log.Info("Uploading compressed tar file")
buf := bytes.NewBuffer(gzipBytes.Bytes())
size := int64(buf.Len())
- obj, err := archiveDs.UploadFile(util.BufferToStream(buf), size, ctx)
+ obj, err := archiveDs.UploadFile(stream_util.BufferToStream(buf), size, ctx)
if err != nil {
return err
}
@@ -313,7 +314,7 @@ func compileArchive(exportId string, entityId string, archiveDs *datastore.Datas
}
ctx.Log.Info("Writing manifest")
- err = putFile("manifest.json", int64(len(b)), time.Now(), util.BufferToStream(bytes.NewBuffer(b)))
+ err = putFile("manifest.json", int64(len(b)), time.Now(), stream_util.BufferToStream(bytes.NewBuffer(b)))
if err != nil {
ctx.Log.Error(err)
sentry.CaptureException(err)
@@ -335,7 +336,7 @@ func compileArchive(exportId string, entityId string, archiveDs *datastore.Datas
sentry.CaptureException(err)
return
}
- err = putFile("index.html", int64(html.Len()), time.Now(), util.BufferToStream(bytes.NewBuffer(html.Bytes())))
+ err = putFile("index.html", int64(html.Len()), time.Now(), stream_util.BufferToStream(bytes.NewBuffer(html.Bytes())))
if err != nil {
ctx.Log.Error(err)
sentry.CaptureException(err)
@@ -357,12 +358,12 @@ func compileArchive(exportId string, entityId string, archiveDs *datastore.Datas
_, err = io.Copy(&b, s)
if err != nil {
ctx.Log.Error(err)
- cleanup.DumpAndCloseStream(s)
+ stream_util.DumpAndCloseStream(s)
sentry.CaptureException(err)
continue
}
- cleanup.DumpAndCloseStream(s)
- s = util.BufferToStream(bytes.NewBuffer(b.Bytes()))
+ stream_util.DumpAndCloseStream(s)
+ s = stream_util.BufferToStream(bytes.NewBuffer(b.Bytes()))
ctx.Log.Info("Archiving ", m.MxcUri())
err = putFile(archivedName(m), m.SizeBytes, time.Unix(0, m.CreationTs*int64(time.Millisecond)), s)
diff --git a/controllers/data_controller/import_controller.go b/controllers/data_controller/import_controller.go
index eceb34774ef1c047f9d970b75e92cc4a2fdaf9bf..57587dc207398e5a6a76a216828219dd192c125e 100644
--- a/controllers/data_controller/import_controller.go
+++ b/controllers/data_controller/import_controller.go
@@ -8,12 +8,14 @@ import (
"database/sql"
"encoding/json"
"errors"
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io"
"net/http"
"sync"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
@@ -336,7 +338,7 @@ func doImport(updateChannel chan *importUpdate, taskId int, importId string, ctx
buf, found := fileMap[record.ArchivedName]
if found {
ctx.Log.Info("Using file from memory")
- closer := util.BufferToStream(buf)
+ closer := stream_util.BufferToStream(buf)
_, err := upload_controller.StoreDirect(nil, closer, record.SizeBytes, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx, true)
if err != nil {
ctx.Log.Errorf("Error importing file: %s", err.Error())
diff --git a/controllers/download_controller/download_controller.go b/controllers/download_controller/download_controller.go
index 4d7f354bccfbd1d6a124b3b238e46c96de9a020f..3c70c715a2eb13baa3091ba1b11d3ec5dda0c854 100644
--- a/controllers/download_controller/download_controller.go
+++ b/controllers/download_controller/download_controller.go
@@ -5,11 +5,13 @@ import (
"database/sql"
"errors"
"fmt"
- "github.com/getsentry/sentry-go"
"io"
"io/ioutil"
"time"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/disintegration/imaging"
"github.com/patrickmn/go-cache"
"github.com/turt2live/matrix-media-repo/common"
@@ -21,7 +23,6 @@ import (
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
var localCache = cache.New(30*time.Second, 60*time.Second)
@@ -70,7 +71,7 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
if media != nil {
if media.Quarantined {
ctx.Log.Warn("Quarantined media accessed")
- defer cleanup.DumpAndCloseStream(minMedia.Stream)
+ defer stream_util.DumpAndCloseStream(minMedia.Stream)
if ctx.Config.Quarantine.ReplaceDownloads {
ctx.Log.Info("Replacing thumbnail with a quarantined one")
@@ -84,7 +85,7 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
imaging.Encode(data, img, imaging.PNG)
return &types.MinimalMedia{
// Lie about all the details
- Stream: util.BufferToStream(data),
+ Stream: stream_util.BufferToStream(data),
ContentType: "image/png",
UploadName: "quarantine.png",
SizeBytes: int64(data.Len()),
@@ -131,7 +132,7 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, blockForMedia
rv := v.(*types.MinimalMedia)
vals := make([]interface{}, 0)
- streams := util.CloneReader(rv.Stream, count)
+ streams := stream_util.CloneReader(rv.Stream, count)
for i := 0; i < count; i++ {
if rv.KnownMedia != nil {
diff --git a/controllers/download_controller/download_resource_handler.go b/controllers/download_controller/download_resource_handler.go
index 35358ecdd260fe7253d829124454e7daed278cc9..cb592c1cd04fdfa10c660ea3874b028cf6832e43 100644
--- a/controllers/download_controller/download_resource_handler.go
+++ b/controllers/download_controller/download_resource_handler.go
@@ -2,8 +2,6 @@ package download_controller
import (
"errors"
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util"
"io"
"io/ioutil"
"mime"
@@ -11,6 +9,10 @@ import (
"sync"
"time"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/djherbis/stream"
"github.com/patrickmn/go-cache"
"github.com/prometheus/client_golang/prometheus"
@@ -22,7 +24,6 @@ import (
"github.com/turt2live/matrix-media-repo/matrix"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
"github.com/turt2live/matrix-media-repo/util/resource_handler"
)
@@ -153,7 +154,7 @@ func downloadResourceWorkFn(request *resource_handler.WorkRequest) (resp *worker
}
persistFile := func(fileStream io.ReadCloser, r *workerDownloadResponse) *workerDownloadResponse {
- defer cleanup.DumpAndCloseStream(fileStream)
+ defer stream_util.DumpAndCloseStream(fileStream)
userId := upload_controller.NoApplicableUploadUser
ms := stream.NewMemStream()
diff --git a/controllers/info_controller/info_controller.go b/controllers/info_controller/info_controller.go
index 281ec6063f277a49a0aecfad83c88ed15235cd63..98eb09699fd1aa1d15df2564dcf8884b69961054 100644
--- a/controllers/info_controller/info_controller.go
+++ b/controllers/info_controller/info_controller.go
@@ -10,7 +10,7 @@ import (
"github.com/turt2live/matrix-media-repo/controllers/download_controller"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func GetOrCalculateBlurhash(media *types.Media, rctx rcontext.RequestContext) (string, error) {
@@ -31,7 +31,7 @@ func GetOrCalculateBlurhash(media *types.Media, rctx rcontext.RequestContext) (s
if err != nil {
return "", err
}
- defer cleanup.DumpAndCloseStream(minMedia.Stream)
+ defer stream_util.DumpAndCloseStream(minMedia.Stream)
// No cached blurhash: calculate one
rctx.Log.Info("Decoding image for blurhash calculation")
diff --git a/controllers/preview_controller/preview_resource_handler.go b/controllers/preview_controller/preview_resource_handler.go
index 86c42b9d9d4e2a05a3760cf9771487a19ff7c01b..3230caa12be950f40f4d148ce1edeec185a3eea6 100644
--- a/controllers/preview_controller/preview_resource_handler.go
+++ b/controllers/preview_controller/preview_resource_handler.go
@@ -2,9 +2,11 @@ package preview_controller
import (
"fmt"
- "github.com/getsentry/sentry-go"
"sync"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/disintegration/imaging"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
@@ -17,7 +19,6 @@ import (
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
"github.com/turt2live/matrix-media-repo/util/resource_handler"
)
@@ -143,7 +144,7 @@ func urlPreviewWorkFn(request *resource_handler.WorkRequest) (resp *urlPreviewRe
ctx.Log.Warn("Non-fatal error streaming datastore file: " + err.Error())
sentry.CaptureException(err)
} else {
- defer cleanup.DumpAndCloseStream(mediaStream)
+ defer stream_util.DumpAndCloseStream(mediaStream)
img, err := imaging.Decode(mediaStream)
if err != nil {
ctx.Log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
diff --git a/controllers/preview_controller/previewers/calculated_previewer.go b/controllers/preview_controller/previewers/calculated_previewer.go
index d41f21c69b646178293135c9661ed642adab1bd1..a40672a38975e9984fd7cfd0125ab61ca95f8784 100644
--- a/controllers/preview_controller/previewers/calculated_previewer.go
+++ b/controllers/preview_controller/previewers/calculated_previewer.go
@@ -9,7 +9,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_types"
"github.com/turt2live/matrix-media-repo/metrics"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func GenerateCalculatedPreview(urlPayload *preview_types.UrlPayload, languageHeader string, ctx rcontext.RequestContext) (preview_types.PreviewResult, error) {
@@ -26,7 +26,7 @@ func GenerateCalculatedPreview(urlPayload *preview_types.UrlPayload, languageHea
return preview_types.PreviewResult{}, common.ErrMediaNotFound
}
- stream := util.BufferToStream(bytes2.NewBuffer(bytes))
+ stream := stream_util.BufferToStream(bytes2.NewBuffer(bytes))
img := &preview_types.PreviewImage{
Data: stream,
ContentType: contentType,
diff --git a/controllers/preview_controller/previewers/http.go b/controllers/preview_controller/previewers/http.go
index 7c6deb96104723cfc70acc0c61e457f6cf627806..d20ae455aef5aa334e65e8941a5db09575dc4641 100644
--- a/controllers/preview_controller/previewers/http.go
+++ b/controllers/preview_controller/previewers/http.go
@@ -18,7 +18,7 @@ import (
"github.com/turt2live/matrix-media-repo/controllers/preview_controller/acl"
"github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func doHttpGet(urlPayload *preview_types.UrlPayload, languageHeader string, ctx rcontext.RequestContext) (*http.Response, error) {
@@ -149,7 +149,7 @@ func downloadRawContent(urlPayload *preview_types.UrlPayload, supportedTypes []s
return nil, "", "", "", err
}
- defer cleanup.DumpAndCloseStream(resp.Body)
+ defer stream_util.DumpAndCloseStream(resp.Body)
contentType := resp.Header.Get("Content-Type")
for _, supportedType := range supportedTypes {
diff --git a/controllers/thumbnail_controller/thumbnail_controller.go b/controllers/thumbnail_controller/thumbnail_controller.go
index 9b5e1212b928ccb8aac8e87b02089a973f08e4fe..1898b994d364ed03341a0a062100985d238053ba 100644
--- a/controllers/thumbnail_controller/thumbnail_controller.go
+++ b/controllers/thumbnail_controller/thumbnail_controller.go
@@ -4,10 +4,12 @@ import (
"bytes"
"database/sql"
"fmt"
- "github.com/getsentry/sentry-go"
"io/ioutil"
"time"
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/disintegration/imaging"
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
@@ -58,7 +60,7 @@ func GetThumbnail(origin string, mediaId string, desiredWidth int, desiredHeight
data := &bytes.Buffer{}
_ = imaging.Encode(data, img, imaging.PNG)
return &types.StreamedThumbnail{
- Stream: util.BufferToStream(data),
+ Stream: stream_util.BufferToStream(data),
Thumbnail: &types.Thumbnail{
// We lie about the details to ensure we keep our contract
Width: img.Bounds().Max.X,
@@ -165,7 +167,7 @@ func GetThumbnail(origin string, mediaId string, desiredWidth int, desiredHeight
rv := v.(*types.StreamedThumbnail)
vals := make([]interface{}, 0)
- streams := util.CloneReader(rv.Stream, count)
+ streams := stream_util.CloneReader(rv.Stream, count)
for i := 0; i < count; i++ {
internal_cache.Get().MarkDownload(rv.Thumbnail.Sha256Hash)
diff --git a/controllers/upload_controller/upload_controller.go b/controllers/upload_controller/upload_controller.go
index a1ec3248d77186ee929b1d9fd8f0a786dbee6278..b4fb52dc90865cc9b75db6763642a2e6a0254a33 100644
--- a/controllers/upload_controller/upload_controller.go
+++ b/controllers/upload_controller/upload_controller.go
@@ -8,6 +8,7 @@ import (
"github.com/getsentry/sentry-go"
"github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
@@ -20,7 +21,6 @@ import (
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
@@ -94,7 +94,7 @@ func EstimateContentLength(contentLength int64, contentLengthHeader string) int6
}
func UploadMedia(contents io.ReadCloser, contentLength int64, contentType string, filename string, userId string, origin string, ctx rcontext.RequestContext) (*types.Media, error) {
- defer cleanup.DumpAndCloseStream(contents)
+ defer stream_util.DumpAndCloseStream(contents)
var data io.ReadCloser
if ctx.Config.Uploads.MaxSizeBytes > 0 {
@@ -189,7 +189,7 @@ func StoreDirect(f *AlreadyUploadedFile, contents io.ReadCloser, expectedSize in
return nil, err
}
- fInfo, err := ds.UploadFile(util.BytesToStream(contentBytes), expectedSize, ctx)
+ fInfo, err := ds.UploadFile(stream_util.BytesToStream(contentBytes), expectedSize, ctx)
if err != nil {
return nil, err
}
diff --git a/docs/admin.md b/docs/admin.md
index b35b4092e1bfc8eca8affa26a3adfa5e414d311e..2823b08ec35689d10743c004c21ab8367b205ae9 100644
--- a/docs/admin.md
+++ b/docs/admin.md
@@ -46,7 +46,9 @@ This will delete all media that has previously been quarantined, local or remote
#### Purge individual record
-URL: `POST /_matrix/media/unstable/admin/purge/<server>/<media id>?access_token=your_access_token`
+URL: `POST /_matrix/media/unstable/admin/purge/media/<server>/<media id>?access_token=your_access_token`
+
+**Note**: Prior to v1.3, this endpoint did not require the `/media` component, but does now.
This will delete the media record, regardless of it being local or remote. Can be called by homeserver administrators and the uploader to delete it.
@@ -88,7 +90,9 @@ This API is unique in that it can allow administrators of configured homeservers
#### Quarantine a specific record
-URL: `POST /_matrix/media/unstable/admin/quarantine/<server>/<media id>?access_token=your_access_token`
+URL: `POST /_matrix/media/unstable/admin/quarantine/media/<server>/<media id>?access_token=your_access_token`
+
+**Note**: Prior to v1.3, this endpoint did not require the `/media` component, but does now.
The `<server>` and `<media id>` can be retrieved from an MXC URI (`mxc://<server>/<media id>`).
@@ -358,7 +362,9 @@ The response is a list of all unfinished tasks:
#### Getting information on a specific task
-URL: `GET /_matrix/media/unstable/admin/tasks/<task ID>`
+URL: `GET /_matrix/media/unstable/admin/task/<task ID>`
+
+**Note**: Prior to v1.3, this endpoint was "tasks" (plural). It is now singular.
The response is the status of the task:
```json
diff --git a/go.mod b/go.mod
index 706adc63a08a994a05ca83eb66ede320643a77fa..740d1fa501bc96e5cf3285a782f009c9216dd2ca 100644
--- a/go.mod
+++ b/go.mod
@@ -27,7 +27,6 @@ require (
github.com/getsentry/sentry-go v0.13.0
github.com/go-redis/redis/v9 v9.0.0-beta.2
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
- github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-hclog v1.2.2
github.com/hashicorp/go-plugin v1.4.4
github.com/k3a/html2text v1.0.8
@@ -50,6 +49,8 @@ require (
golang.org/x/net v0.0.0-20220812174116-3211cb980234
)
+require github.com/julienschmidt/httprouter v1.3.0
+
require (
github.com/Jeffail/gabs v1.4.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
diff --git a/go.sum b/go.sum
index 6cf276e173ad84eee5e442b2eefddd4fc9213d12..a052c1bccf4e69fcb3d47c22504bfd64420fb028 100644
--- a/go.sum
+++ b/go.sum
@@ -232,8 +232,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hajimehoshi/go-mp3 v0.3.0/go.mod h1:qMJj/CSDxx6CGHiZeCgbiq2DSUkbK0UbtXShQcnfyMM=
github.com/hajimehoshi/go-mp3 v0.3.3 h1:cWnfRdpye2m9ElSoVqneYRcpt/l3ijttgjMeQh+r+FE=
@@ -279,6 +277,7 @@ github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4Yc
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/k3a/html2text v1.0.8 h1:rVanLhKilpnJUJs/CNKWzMC4YaQINGxK0rSG8ssmnV0=
github.com/k3a/html2text v1.0.8/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
diff --git a/matrix/client_server.go b/matrix/client_server.go
index 7e20e62ab0dd977a5e16d35846fbf17be2f72975..cf70d451a465a2e2e87c06785dcb68dafa89cbd0 100644
--- a/matrix/client_server.go
+++ b/matrix/client_server.go
@@ -10,7 +10,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
// Based in part on https://github.com/matrix-org/gomatrix/blob/072b39f7fa6b40257b4eead8c958d71985c28bdd/client.go#L180-L243
@@ -48,7 +48,7 @@ func doRequest(ctx rcontext.RequestContext, method string, urlStr string, body i
if err != nil {
return err
}
- defer cleanup.DumpAndCloseStream(res.Body)
+ defer stream_util.DumpAndCloseStream(res.Body)
contents, err := ioutil.ReadAll(res.Body)
if err != nil {
diff --git a/pipline/upload_pipeline/pipeline.go b/pipline/upload_pipeline/pipeline.go
index a2c46d02e804eb17137227bcffce4ffebac247fb..0d10ad0219bc81b83bf901b5098e0cf666073418 100644
--- a/pipline/upload_pipeline/pipeline.go
+++ b/pipline/upload_pipeline/pipeline.go
@@ -8,11 +8,11 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func UploadMedia(ctx rcontext.RequestContext, origin string, mediaId string, r io.ReadCloser, contentType string, fileName string, userId string) (*types.Media, error) {
- defer cleanup.DumpAndCloseStream(r)
+ defer stream_util.DumpAndCloseStream(r)
// Step 1: Limit the stream's length
r = limitStreamLength(ctx, r)
@@ -63,4 +63,4 @@ func UploadMedia(ctx rcontext.RequestContext, origin string, mediaId string, r i
// TODO
return nil, errors.New("not yet implemented")
-}
\ No newline at end of file
+}
diff --git a/pipline/upload_pipeline/step_hash.go b/pipline/upload_pipeline/step_hash.go
index 66072ee7ce7a5763508309e35e60479f4571e1f1..eedab08cb9a0206db24ac61b78c874fa9913a0d0 100644
--- a/pipline/upload_pipeline/step_hash.go
+++ b/pipline/upload_pipeline/step_hash.go
@@ -4,9 +4,9 @@ import (
"io"
"github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func hashFile(ctx rcontext.RequestContext, r io.ReadCloser) (string, error) {
- return util.GetSha256HashOfStream(r)
+ return stream_util.GetSha256HashOfStream(r)
}
diff --git a/storage/datastore/ds_file/file_store.go b/storage/datastore/ds_file/file_store.go
index ce41ba6c6b70b5160d8e3408ed04b0e8fd7ea82d..74a5df07c20f2c79524050e91195ca2d543612dd 100644
--- a/storage/datastore/ds_file/file_store.go
+++ b/storage/datastore/ds_file/file_store.go
@@ -2,20 +2,21 @@ package ds_file
import (
"errors"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io"
"io/ioutil"
"os"
"path"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
func PersistFile(basePath string, file io.ReadCloser, ctx rcontext.RequestContext) (*types.ObjectInfo, error) {
- defer cleanup.DumpAndCloseStream(file)
+ defer stream_util.DumpAndCloseStream(file)
exists := true
var primaryContainer string
@@ -70,13 +71,13 @@ func PersistFile(basePath string, file io.ReadCloser, ctx rcontext.RequestContex
}
func PersistFileAtLocation(targetFile string, file io.ReadCloser, ctx rcontext.RequestContext) (int64, string, error) {
- defer cleanup.DumpAndCloseStream(file)
+ defer stream_util.DumpAndCloseStream(file)
f, err := os.OpenFile(targetFile, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return 0, "", err
}
- defer cleanup.DumpAndCloseStream(f)
+ defer stream_util.DumpAndCloseStream(f)
rfile, wfile := io.Pipe()
tr := io.TeeReader(file, wfile)
@@ -92,7 +93,7 @@ func PersistFileAtLocation(targetFile string, file io.ReadCloser, ctx rcontext.R
go func() {
defer wfile.Close()
ctx.Log.Info("Calculating hash of stream...")
- hash, hashErr = util.GetSha256HashOfStream(ioutil.NopCloser(tr))
+ hash, hashErr = stream_util.GetSha256HashOfStream(ioutil.NopCloser(tr))
ctx.Log.Info("Hash of file is ", hash)
done <- true
}()
diff --git a/storage/datastore/ds_s3/s3_store.go b/storage/datastore/ds_s3/s3_store.go
index f8d28af9504587677f82fa06f5ed77a17cefc4a8..511374ab83269eae9a981ea34fd676c36a1571b8 100644
--- a/storage/datastore/ds_s3/s3_store.go
+++ b/storage/datastore/ds_s3/s3_store.go
@@ -2,13 +2,15 @@ package ds_s3
import (
"fmt"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io"
"io/ioutil"
"os"
"strconv"
"strings"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/minio/minio-go/v6"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
@@ -17,8 +19,6 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
var stores = make(map[string]*s3Datastore)
@@ -131,7 +131,7 @@ func (s *s3Datastore) EnsureTempPathExists() error {
}
func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx rcontext.RequestContext) (*types.ObjectInfo, error) {
- defer cleanup.DumpAndCloseStream(file)
+ defer stream_util.DumpAndCloseStream(file)
objectName, err := ids.NewUniqueId()
if err != nil {
@@ -154,7 +154,7 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
go func() {
defer ws3.Close()
ctx.Log.Info("Calculating hash of stream...")
- hash, hashErr = util.GetSha256HashOfStream(ioutil.NopCloser(tr))
+ hash, hashErr = stream_util.GetSha256HashOfStream(ioutil.NopCloser(tr))
ctx.Log.Info("Hash of file is ", hash)
done <- true
}()
@@ -172,14 +172,14 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
}
defer os.Remove(f.Name())
expectedLength, uploadErr = io.Copy(f, rs3)
- cleanup.DumpAndCloseStream(f)
+ stream_util.DumpAndCloseStream(f)
f, uploadErr = os.Open(f.Name())
if uploadErr != nil {
done <- true
return
}
rs3 = f
- defer cleanup.DumpAndCloseStream(f)
+ defer stream_util.DumpAndCloseStream(f)
} else {
ctx.Log.Warn("Uploading content of unknown length to s3 - this could result in high memory usage")
expectedLength = -1
@@ -236,7 +236,7 @@ func (s *s3Datastore) ObjectExists(location string) bool {
}
func (s *s3Datastore) OverwriteObject(location string, stream io.ReadCloser) error {
- defer cleanup.DumpAndCloseStream(stream)
+ defer stream_util.DumpAndCloseStream(stream)
metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc()
_, err := s.client.PutObject(s.bucket, location, stream, -1, minio.PutObjectOptions{StorageClass: s.storageClass})
return err
diff --git a/thumbnailing/i/jpegxl.go b/thumbnailing/i/jpegxl.go
index 69e2282ef428603b16dcc57ebc415a3690592686..9294cc58f82cda090e9a2a2b2da10327a1f369fd 100644
--- a/thumbnailing/i/jpegxl.go
+++ b/thumbnailing/i/jpegxl.go
@@ -2,15 +2,16 @@ package i
import (
"errors"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io/ioutil"
"os"
"os/exec"
"path"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type jpegxlGenerator struct {
@@ -49,7 +50,7 @@ func (d jpegxlGenerator) GenerateThumbnail(b []byte, contentType string, width i
return nil, errors.New("jpegxl: error writing temp jpegxl file: " + err.Error())
}
_, _ = f.Write(b)
- cleanup.DumpAndCloseStream(f)
+ stream_util.DumpAndCloseStream(f)
err = exec.Command("convert", tempFile1, tempFile2).Run()
if err != nil {
diff --git a/thumbnailing/i/mp4.go b/thumbnailing/i/mp4.go
index 5fe9d73f0c32aeeda341cc28afaaf227346723e6..e6cceb521dc16be0f75f61c420a95cefa8b1896c 100644
--- a/thumbnailing/i/mp4.go
+++ b/thumbnailing/i/mp4.go
@@ -2,16 +2,17 @@ package i
import (
"errors"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io/ioutil"
"os"
"os/exec"
"path"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type mp4Generator struct {
@@ -50,7 +51,7 @@ func (d mp4Generator) GenerateThumbnail(b []byte, contentType string, width int,
return nil, errors.New("mp4: error writing temp video file: " + err.Error())
}
_, _ = f.Write(b)
- cleanup.DumpAndCloseStream(f)
+ stream_util.DumpAndCloseStream(f)
err = exec.Command("ffmpeg", "-i", tempFile1, "-vf", "select=eq(n\\,0)", tempFile2).Run()
if err != nil {
diff --git a/thumbnailing/i/svg.go b/thumbnailing/i/svg.go
index f8d99753ae29981bf1fa76693b181855b187e265..98b622ac39cc381b9fbda68c6d74104f275e8c96 100644
--- a/thumbnailing/i/svg.go
+++ b/thumbnailing/i/svg.go
@@ -2,15 +2,16 @@ package i
import (
"errors"
- "github.com/turt2live/matrix-media-repo/util/ids"
"io/ioutil"
"os"
"os/exec"
"path"
+ "github.com/turt2live/matrix-media-repo/util/ids"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
+
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
type svgGenerator struct {
@@ -49,7 +50,7 @@ func (d svgGenerator) GenerateThumbnail(b []byte, contentType string, width int,
return nil, errors.New("svg: error writing temp svg file: " + err.Error())
}
_, _ = f.Write(b)
- cleanup.DumpAndCloseStream(f)
+ stream_util.DumpAndCloseStream(f)
err = exec.Command("convert", tempFile1, tempFile2).Run()
if err != nil {
diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go
index 2d124592a8b9945a4a2db38fb894adf5163f0525..0897a209a4cffa85b3cd29357f03e6441e64698d 100644
--- a/thumbnailing/thumbnail.go
+++ b/thumbnailing/thumbnail.go
@@ -7,12 +7,12 @@ import (
"reflect"
"github.com/turt2live/matrix-media-repo/common"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/i"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/util"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
)
var ErrUnsupported = errors.New("unsupported thumbnail type")
@@ -30,7 +30,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h
return nil, ErrUnsupported
}
- defer cleanup.DumpAndCloseStream(imgStream)
+ defer stream_util.DumpAndCloseStream(imgStream)
b, err := ioutil.ReadAll(imgStream)
if err != nil {
return nil, err
@@ -57,7 +57,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h
}
func GetGenerator(imgStream io.ReadCloser, contentType string, animated bool) (i.Generator, error) {
- defer cleanup.DumpAndCloseStream(imgStream)
+ defer stream_util.DumpAndCloseStream(imgStream)
b, err := ioutil.ReadAll(imgStream)
if err != nil {
return nil, err
diff --git a/util/cleanup/stream_cleanup.go b/util/cleanup/stream_cleanup.go
deleted file mode 100644
index 52056716c3e515e34c8830fd23584ac80d7a0123..0000000000000000000000000000000000000000
--- a/util/cleanup/stream_cleanup.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package cleanup
-
-import (
- "io"
- "io/ioutil"
-)
-
-func DumpAndCloseStream(r io.ReadCloser) {
- if r == nil {
- return // nothing to dump or close
- }
- _, _ = io.Copy(ioutil.Discard, r)
- _ = r.Close()
-}
diff --git a/util/files.go b/util/files.go
index 53f8896ded68a198e58ccc9dba437f967508c350..178ae1d8815cd4be638504bc2640705bf043c360 100644
--- a/util/files.go
+++ b/util/files.go
@@ -4,7 +4,7 @@ import (
"os"
"path"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
func FileExists(path string) (bool, error) {
@@ -33,7 +33,7 @@ func GetFileHash(filePath string) (string, error) {
if err != nil {
return "", err
}
- defer cleanup.DumpAndCloseStream(f)
+ defer stream_util.DumpAndCloseStream(f)
- return GetSha256HashOfStream(f)
+ return stream_util.GetSha256HashOfStream(f)
}
diff --git a/util/stream_util/streams.go b/util/stream_util/streams.go
new file mode 100644
index 0000000000000000000000000000000000000000..0717f8bbd6b110c4ed12d050fa3a972b4d3f9d5f
--- /dev/null
+++ b/util/stream_util/streams.go
@@ -0,0 +1,124 @@
+package stream_util
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "math"
+
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
+)
+
+func BufferToStream(buf *bytes.Buffer) io.ReadCloser {
+ newBuf := bytes.NewReader(buf.Bytes())
+ return ioutil.NopCloser(newBuf)
+}
+
+func BytesToStream(b []byte) io.ReadCloser {
+ return ioutil.NopCloser(bytes.NewBuffer(b))
+}
+
+func CloneReader(input io.ReadCloser, numReaders int) []io.ReadCloser {
+ readers := make([]io.ReadCloser, 0)
+ writers := make([]io.WriteCloser, 0)
+
+ for i := 0; i < numReaders; i++ {
+ r, w := io.Pipe()
+ readers = append(readers, r)
+ writers = append(writers, w)
+ }
+
+ go func() {
+ plainWriters := make([]io.Writer, 0)
+ for _, w := range writers {
+ defer w.Close()
+ plainWriters = append(plainWriters, w)
+ }
+
+ mw := io.MultiWriter(plainWriters...)
+ io.Copy(mw, input)
+ }()
+
+ return readers
+}
+
+func GetSha256HashOfStream(r io.ReadCloser) (string, error) {
+ defer DumpAndCloseStream(r)
+
+ hasher := sha256.New()
+
+ if _, err := io.Copy(hasher, r); err != nil {
+ return "", err
+ }
+
+ return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+func ClonedBufReader(buf bytes.Buffer) util_byte_seeker.ByteSeeker {
+ return util_byte_seeker.NewByteSeeker(buf.Bytes())
+}
+
+func ForceDiscard(r io.Reader, nBytes int64) error {
+ if nBytes == 0 {
+ return nil // weird call, but ok
+ }
+
+ buf := make([]byte, 128)
+
+ if nBytes < 0 {
+ for true {
+ _, err := r.Read(buf)
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+
+ read := int64(0)
+ for (nBytes - read) > 0 {
+ toRead := int(math.Min(float64(len(buf)), float64(nBytes-read)))
+ if toRead != len(buf) {
+ buf = make([]byte, toRead)
+ }
+ actuallyRead, err := r.Read(buf)
+ if err != nil {
+ return err
+ }
+ read += int64(actuallyRead)
+ if (nBytes - read) < 0 {
+ return errors.New(fmt.Sprintf("over-discarded from stream by %d bytes", nBytes-read))
+ }
+ }
+
+ return nil
+}
+
+func ManualSeekStream(r io.Reader, bytesStart int64, bytesToRead int64) (io.Reader, error) {
+ if sr, ok := r.(io.ReadSeeker); ok {
+ _, err := sr.Seek(bytesStart, io.SeekStart)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ err := ForceDiscard(r, bytesStart)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return io.LimitReader(r, bytesToRead), nil
+}
+
+func DumpAndCloseStream(r io.ReadCloser) {
+ if r == nil {
+ return // nothing to dump or close
+ }
+ _ = ForceDiscard(r, -1)
+ _ = r.Close()
+}
diff --git a/util/streams.go b/util/streams.go
deleted file mode 100644
index 9cafc8ba5d38db9f14893367246149513db74347..0000000000000000000000000000000000000000
--- a/util/streams.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package util
-
-import (
- "bytes"
- "crypto/sha256"
- "encoding/hex"
- "io"
- "io/ioutil"
-
- "github.com/turt2live/matrix-media-repo/util/cleanup"
- "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
-)
-
-func BufferToStream(buf *bytes.Buffer) io.ReadCloser {
- newBuf := bytes.NewReader(buf.Bytes())
- return ioutil.NopCloser(newBuf)
-}
-
-func BytesToStream(b []byte) io.ReadCloser {
- return ioutil.NopCloser(bytes.NewBuffer(b))
-}
-
-func CloneReader(input io.ReadCloser, numReaders int) []io.ReadCloser {
- readers := make([]io.ReadCloser, 0)
- writers := make([]io.WriteCloser, 0)
-
- for i := 0; i < numReaders; i++ {
- r, w := io.Pipe()
- readers = append(readers, r)
- writers = append(writers, w)
- }
-
- go func() {
- plainWriters := make([]io.Writer, 0)
- for _, w := range writers {
- defer w.Close()
- plainWriters = append(plainWriters, w)
- }
-
- mw := io.MultiWriter(plainWriters...)
- io.Copy(mw, input)
- }()
-
- return readers
-}
-
-func GetSha256HashOfStream(r io.ReadCloser) (string, error) {
- defer cleanup.DumpAndCloseStream(r)
-
- hasher := sha256.New()
-
- if _, err := io.Copy(hasher, r); err != nil {
- return "", err
- }
-
- return hex.EncodeToString(hasher.Sum(nil)), nil
-}
-
-func ClonedBufReader(buf bytes.Buffer) util_byte_seeker.ByteSeeker {
- return util_byte_seeker.NewByteSeeker(buf.Bytes())
-}
\ No newline at end of file
diff --git a/util/util_exif/exif.go b/util/util_exif/exif.go
index 4226576cd0c9ed0a7c0d748923196ea9df225613..47cdb2842952f283535b61d90f3739cfef661377 100644
--- a/util/util_exif/exif.go
+++ b/util/util_exif/exif.go
@@ -2,10 +2,11 @@ package util_exif
import (
"fmt"
+ "io"
+
"github.com/dsoprea/go-exif/v3"
"github.com/pkg/errors"
- "github.com/turt2live/matrix-media-repo/util/cleanup"
- "io"
+ "github.com/turt2live/matrix-media-repo/util/stream_util"
)
type ExifOrientation struct {
@@ -15,7 +16,7 @@ type ExifOrientation struct {
}
func GetExifOrientation(img io.ReadCloser) (*ExifOrientation, error) {
- defer cleanup.DumpAndCloseStream(img)
+ defer stream_util.DumpAndCloseStream(img)
rawExif, err := exif.SearchAndExtractExifWithReader(img)
if err != nil {