diff --git a/api/_routers/01-install_metadata.go b/api/_routers/01-install_metadata.go
index 2ab5b29b9d347ddebc2740399966cef5cc0615c1..8a9102e8c8de083eb1e52b28462a943ed5e05687 100644
--- a/api/_routers/01-install_metadata.go
+++ b/api/_routers/01-install_metadata.go
@@ -67,14 +67,6 @@ func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request
}
}
-func GetRequestId(r *http.Request) string {
- x, ok := r.Context().Value(requestIdCtxKey).(string)
- if !ok {
- return "REQ-ID-UNKNOWN"
- }
- return x
-}
-
func GetActionName(r *http.Request) string {
x, ok := r.Context().Value(actionNameCtxKey).(string)
if !ok {
diff --git a/api/_routers/97-optional-access-token.go b/api/_routers/97-optional-access-token.go
index 517ab346cd6e1d3901e34296f1786c0c12661be8..bbddbfdd70339b8eca58d806e160d8c019c90410 100644
--- a/api/_routers/97-optional-access-token.go
+++ b/api/_routers/97-optional-access-token.go
@@ -1,6 +1,7 @@
package _routers
import (
+ "errors"
"net/http"
"github.com/getsentry/sentry-go"
@@ -35,7 +36,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn {
appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId)
if err != nil {
- if err != matrix.ErrInvalidToken {
+ if !errors.Is(err, matrix.ErrInvalidToken) {
sentry.CaptureException(err)
ctx.Log.Error("Error verifying token: ", err)
return _responses.InternalServerError("unexpected error validating access token")
diff --git a/api/_routers/97-require-access-token.go b/api/_routers/97-require-access-token.go
index 8d4b1fecf3c1baa89603896c41734c3e15e7dd81..00880271dc96b66c1ac73758e87e32475ddde5dd 100644
--- a/api/_routers/97-require-access-token.go
+++ b/api/_routers/97-require-access-token.go
@@ -1,6 +1,7 @@
package _routers
import (
+ "errors"
"net/http"
"github.com/getsentry/sentry-go"
@@ -38,10 +39,10 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn {
appserviceUserId := util.GetAppserviceUserIdFromRequest(r)
userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId)
if err != nil || userId == "" {
- if err == matrix.ErrGuestToken {
+ if errors.Is(err, matrix.ErrGuestToken) {
return _responses.GuestAuthFailed()
}
- if err != nil && err != matrix.ErrInvalidToken {
+ if err != nil && !errors.Is(err, matrix.ErrInvalidToken) {
sentry.CaptureException(err)
ctx.Log.Error("Error verifying token: ", err)
return _responses.InternalServerError("unexpected error validating access token")
diff --git a/api/custom/imports.go b/api/custom/imports.go
index 20846616bdd323d300c273fb63a5ca0b12ff25d5..7faec339b767c5e2d60be0c3fa547ef8764f4e2c 100644
--- a/api/custom/imports.go
+++ b/api/custom/imports.go
@@ -1,6 +1,7 @@
package custom
import (
+ "errors"
"net/http"
"github.com/getsentry/sentry-go"
@@ -64,7 +65,7 @@ func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta
err := task_runner.AppendImportFile(rctx, importId, r.Body)
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
}
rctx.Log.Error(err)
@@ -91,7 +92,7 @@ func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use
err := task_runner.FinishImport(rctx, importId)
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
}
rctx.Log.Error(err)
diff --git a/api/custom/purge.go b/api/custom/purge.go
index 4d4ff3b90da29f71a07c11d7f286aaac617155d7..319401ce6f1a58823f7206e3bd3f084076535637 100644
--- a/api/custom/purge.go
+++ b/api/custom/purge.go
@@ -1,6 +1,7 @@
package custom
import (
+ "errors"
"net/http"
"strconv"
@@ -69,7 +70,7 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _
},
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
@@ -104,7 +105,7 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime
DbMedia: affected,
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
@@ -157,7 +158,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.
DbMedia: records,
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
@@ -214,7 +215,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
DbMedia: records,
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
@@ -287,7 +288,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
MxcUris: mxcs,
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
@@ -341,7 +342,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime
DbMedia: records,
})
if err != nil {
- if err == common.ErrWrongUser {
+ if errors.Is(err, common.ErrWrongUser) {
return _responses.AuthFailed()
}
rctx.Log.Error(err)
diff --git a/api/r0/download.go b/api/r0/download.go
index af4a23e158cd49e6ab89194e0b3f002148feab42..a948f22a8b7576cc35b659183c49ed42bdad528e 100644
--- a/api/r0/download.go
+++ b/api/r0/download.go
@@ -1,6 +1,7 @@
package r0
import (
+ "errors"
"net/http"
"strconv"
@@ -69,18 +70,18 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.
BlockForReadUntil: blockFor,
})
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
- } else if err == common.ErrMediaTooLarge {
+ } else if errors.Is(err, common.ErrMediaTooLarge) {
return _responses.RequestTooLarge()
- } else if err == common.ErrMediaQuarantined {
+ } else if errors.Is(err, common.ErrMediaQuarantined) {
rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil)
if stream != nil {
return _responses.MakeQuarantinedImageResponse(stream)
} else {
return _responses.NotFoundError() // We lie for security
}
- } else if err == common.ErrMediaNotYetUploaded {
+ } else if errors.Is(err, common.ErrMediaNotYetUploaded) {
return _responses.NotYetUploaded()
}
rctx.Log.Error("Unexpected error locating media: ", err)
diff --git a/api/r0/identicon.go b/api/r0/identicon.go
index 07d307becfeeeea80d295a90ac4c6dcaf8dec044..751947d61d556c9ff10ebb6a68122be4ba444cf0 100644
--- a/api/r0/identicon.go
+++ b/api/r0/identicon.go
@@ -80,7 +80,7 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User
}
rctx.Log.Info("Generating identicon")
- img := sig.Make(width, false, []byte(hashed))
+ img := sig.Make(width, false, hashed)
if width != height {
// Resize to the desired height
rctx.Log.Info("Resizing image to fit height")
diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go
index 0d7963e2a52e15600dd443156028c3db904a03f2..ba2503fdb74d4500f7c6726eef0e36aabb20be71 100644
--- a/api/r0/preview_url.go
+++ b/api/r0/preview_url.go
@@ -53,6 +53,7 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use
if urlStr == "" {
return _responses.BadRequest("No url provided")
}
+ //goland:noinspection HttpUrlsUsage
if strings.Index(urlStr, "http://") != 0 && strings.Index(urlStr, "https://") != 0 {
return _responses.BadRequest("Scheme not accepted")
}
@@ -76,9 +77,9 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use
}
}
if err != nil {
- if err == common.ErrMediaNotFound || err == common.ErrHostNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) || errors.Is(err, common.ErrHostNotFound) {
return _responses.NotFoundError()
- } else if err == common.ErrInvalidHost || err == common.ErrHostNotAllowed {
+ } else if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) {
return _responses.BadRequest(err.Error())
} else {
sentry.CaptureException(err)
diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go
index eee357cc7785347ee00f61b7ecec81bdc0c7be7e..2b4bb43c3e42020328b28387f82f3d6d69637fca 100644
--- a/api/r0/thumbnail.go
+++ b/api/r0/thumbnail.go
@@ -1,6 +1,7 @@
package r0
import (
+ "errors"
"net/http"
"strconv"
@@ -118,18 +119,18 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
Animated: animated,
})
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
- } else if err == common.ErrMediaTooLarge {
+ } else if errors.Is(err, common.ErrMediaTooLarge) {
return _responses.RequestTooLarge()
- } else if err == common.ErrMediaQuarantined {
+ } else if errors.Is(err, common.ErrMediaQuarantined) {
rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil)
if stream != nil {
return _responses.MakeQuarantinedImageResponse(stream)
} else {
return _responses.NotFoundError() // We lie for security
}
- } else if err == common.ErrMediaNotYetUploaded {
+ } else if errors.Is(err, common.ErrMediaNotYetUploaded) {
return _responses.NotYetUploaded()
}
rctx.Log.Error("Unexpected error locating media: ", err)
diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go
index 8d0bd6f31f967568177bf1c92422ad6154d61f3c..1e0e11ab3f3fae45372d6a61e0c90af436a5c633 100644
--- a/api/r0/upload_async.go
+++ b/api/r0/upload_async.go
@@ -1,6 +1,7 @@
package r0
import (
+ "errors"
"net/http"
"path/filepath"
@@ -47,21 +48,21 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apime
// Actually upload
media, err := pipeline_upload.ExecutePut(rctx, server, mediaId, r.Body, contentType, filename, user.UserId)
if err != nil {
- if err == common.ErrQuotaExceeded {
+ if errors.Is(err, common.ErrQuotaExceeded) {
return _responses.QuotaExceeded()
- } else if err == common.ErrAlreadyUploaded {
+ } else if errors.Is(err, common.ErrAlreadyUploaded) {
return _responses.ErrorResponse{
Code: common.ErrCodeCannotOverwrite,
Message: "This media has already been uploaded.",
InternalCode: common.ErrCodeCannotOverwrite,
}
- } else if err == common.ErrWrongUser {
+ } else if errors.Is(err, common.ErrWrongUser) {
return _responses.ErrorResponse{
Code: common.ErrCodeForbidden,
Message: "You do not have permission to upload this media.",
InternalCode: common.ErrCodeForbidden,
}
- } else if err == common.ErrExpired {
+ } else if errors.Is(err, common.ErrExpired) {
return _responses.ErrorResponse{
Code: common.ErrCodeNotFound,
Message: "Media expired or not found.",
diff --git a/api/r0/upload_sync.go b/api/r0/upload_sync.go
index b8cb7f326436a7b16ee9e9511f496d1d18d84c8d..e0ca2629d54a4a0f50a600de92c3ea501a4eb662 100644
--- a/api/r0/upload_sync.go
+++ b/api/r0/upload_sync.go
@@ -1,6 +1,7 @@
package r0
import (
+ "errors"
"net/http"
"path/filepath"
"strconv"
@@ -42,7 +43,7 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimet
// Actually upload
media, err := pipeline_upload.Execute(rctx, r.Host, "", r.Body, contentType, filename, user.UserId, datastores.LocalMediaKind)
if err != nil {
- if err == common.ErrQuotaExceeded {
+ if errors.Is(err, common.ErrQuotaExceeded) {
return _responses.QuotaExceeded()
}
rctx.Log.Error("Unexpected error uploading media: ", err)
diff --git a/api/router.go b/api/router.go
index 79430516b02c7bc0ebcbe45cc9ae1ade9dc8ba43..ebc6c8ac1916e7a5e8ddabf975c36e7d1478bf4d 100644
--- a/api/router.go
+++ b/api/router.go
@@ -57,6 +57,7 @@ func finishCorsFn(w http.ResponseWriter, r *http.Request) {
func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) {
logrus.Errorf("Panic received on %s %s: %s", r.Method, util.GetLogSafeUrl(r), i)
+ //goland:noinspection GoTypeAssertionOnErrors
if e, ok := i.(error); ok {
sentry.CaptureException(e)
} else {
diff --git a/api/unstable/info.go b/api/unstable/info.go
index b14171c268a075718d22d778a366a4fa85575a8f..7335c8ed9f71788767b361db1ec3bf814634f5b5 100644
--- a/api/unstable/info.go
+++ b/api/unstable/info.go
@@ -1,6 +1,7 @@
package unstable
import (
+ "errors"
"net/http"
"strconv"
"strings"
@@ -85,18 +86,18 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User
})
// Error handling copied from download endpoint
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
- } else if err == common.ErrMediaTooLarge {
+ } else if errors.Is(err, common.ErrMediaTooLarge) {
return _responses.RequestTooLarge()
- } else if err == common.ErrMediaQuarantined {
+ } else if errors.Is(err, common.ErrMediaQuarantined) {
rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil)
if stream != nil {
return _responses.MakeQuarantinedImageResponse(stream)
} else {
return _responses.NotFoundError() // We lie for security
}
- } else if err == common.ErrMediaNotYetUploaded {
+ } else if errors.Is(err, common.ErrMediaNotYetUploaded) {
return _responses.NotYetUploaded()
}
rctx.Log.Error("Unexpected error locating media: ", err)
diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go
index c22f34deab488eac369eeeda6b185248443d4655..55f28c357199bd5edde95b5ee5508873e8344304 100644
--- a/api/unstable/local_copy.go
+++ b/api/unstable/local_copy.go
@@ -1,6 +1,7 @@
package unstable
import (
+ "errors"
"net/http"
"strconv"
"time"
@@ -66,18 +67,18 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User
})
// Error handling copied from download endpoint
if err != nil {
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
return _responses.NotFoundError()
- } else if err == common.ErrMediaTooLarge {
+ } else if errors.Is(err, common.ErrMediaTooLarge) {
return _responses.RequestTooLarge()
- } else if err == common.ErrMediaQuarantined {
+ } else if errors.Is(err, common.ErrMediaQuarantined) {
rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil)
if stream != nil {
return _responses.MakeQuarantinedImageResponse(stream)
} else {
return _responses.NotFoundError() // We lie for security
}
- } else if err == common.ErrMediaNotYetUploaded {
+ } else if errors.Is(err, common.ErrMediaNotYetUploaded) {
return _responses.NotYetUploaded()
}
rctx.Log.Error("Unexpected error locating media: ", err)
@@ -88,7 +89,7 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User
record, err = pipeline_upload.Execute(rctx, server, mediaId, stream, record.ContentType, record.UploadName, user.UserId, datastores.LocalMediaKind)
// Error handling copied from upload(sync) endpoint
if err != nil {
- if err == common.ErrQuotaExceeded {
+ if errors.Is(err, common.ErrQuotaExceeded) {
return _responses.QuotaExceeded()
}
rctx.Log.Error("Unexpected error uploading media: ", err)
diff --git a/api/webserver.go b/api/webserver.go
index ed9ae05c4ee4602fe468acc785dae4083662b21a..a2c7fd59f666214b3ab12e04fbcba9e2017b059b 100644
--- a/api/webserver.go
+++ b/api/webserver.go
@@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
+ "errors"
"net"
"net/http"
"strconv"
@@ -53,8 +54,9 @@ func Init() *sync.WaitGroup {
reload = false
go func() {
+ //goland:noinspection HttpUrlsUsage
logrus.WithField("address", address).Info("Started up. Listening at http://" + address)
- if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+ if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
sentry.CaptureException(err)
logrus.Fatal(err)
}
diff --git a/archival/entity_export.go b/archival/entity_export.go
index 262c28d4759647215933774125347a193f7e493b..0d8330b3f67bcb496d0b7579d81a2c013cc15d31 100644
--- a/archival/entity_export.go
+++ b/archival/entity_export.go
@@ -1,6 +1,7 @@
package archival
import (
+ "errors"
"time"
"github.com/turt2live/matrix-media-repo/archival/v2archive"
@@ -46,10 +47,10 @@ func ExportEntityData(ctx rcontext.RequestContext, exportId string, entityId str
BlockForReadUntil: 1 * time.Minute,
RecordOnly: false,
})
- if err == common.ErrMediaQuarantined {
+ if errors.Is(err, common.ErrMediaQuarantined) {
ctx.Log.Warnf("%s is quarantined and will not be included in the export", mxc)
continue
- } else if err == common.ErrMediaNotYetUploaded {
+ } else if errors.Is(err, common.ErrMediaNotYetUploaded) {
ctx.Log.Debug("Media not uploaded yet - skipping")
continue
} else if err != nil {
diff --git a/common/LEGACY_media_kinds.go b/common/LEGACY_media_kinds.go
deleted file mode 100644
index a439543469637d3079037e35153238fced6d143e..0000000000000000000000000000000000000000
--- a/common/LEGACY_media_kinds.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package common
-
-type Kind string
-
-const KindLocalMedia Kind = "local_media"
-const KindRemoteMedia Kind = "remote_media"
-const KindThumbnails Kind = "thumbnails"
-const KindArchives Kind = "archives"
-const KindAll Kind = "all"
-
-func IsKind(have Kind, want Kind) bool {
- return have == want || have == KindAll
-}
-
-func HasKind(have []string, want Kind) bool {
- for _, k := range have {
- if IsKind(Kind(k), want) {
- return true
- }
- }
- return false
-}
diff --git a/common/errorcodes.go b/common/errorcodes.go
index 5d4d7b10035c82c2005f0db6d42072c2c755e2a0..2ba7fca2a5fff03d74a286e6175539fd9cb02cdd 100644
--- a/common/errorcodes.go
+++ b/common/errorcodes.go
@@ -1,8 +1,6 @@
package common
const ErrCodeInvalidHost = "M_INVALID_HOST"
-const ErrCodeHostNotFound = "M_HOST_NOT_FOUND"
-const ErrCodeHostNotAllowed = "M_HOST_NOT_ALLOWED"
const ErrCodeNotFound = "M_NOT_FOUND"
const ErrCodeUnknownToken = "M_UNKNOWN_TOKEN"
const ErrCodeNoGuests = "M_GUEST_ACCESS_FORBIDDEN"
diff --git a/common/globals/LEGACY_singleflight_groups.go b/common/globals/LEGACY_singleflight_groups.go
deleted file mode 100644
index 4f5fd266f599d11dd9a17cc8bf715572b01bc9be..0000000000000000000000000000000000000000
--- a/common/globals/LEGACY_singleflight_groups.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package globals
-
-import (
- "github.com/turt2live/matrix-media-repo/util/singleflight-counter"
-)
-
-var DefaultRequestGroup singleflight_counter.Group
diff --git a/controllers/LEGACY.txt b/controllers/LEGACY.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/controllers/data_controller/export_controller.go b/controllers/data_controller/export_controller.go
deleted file mode 100644
index 95e2053d6e856f28a260df46921a6f22614c7f74..0000000000000000000000000000000000000000
--- a/controllers/data_controller/export_controller.go
+++ /dev/null
@@ -1,397 +0,0 @@
-package data_controller
-
-import (
- "archive/tar"
- "bytes"
- "compress/gzip"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "time"
-
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util/ids"
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-
- "github.com/dustin/go-humanize"
- "github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/storage"
- "github.com/turt2live/matrix-media-repo/storage/datastore"
- "github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3"
- "github.com/turt2live/matrix-media-repo/templating"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type ManifestRecord struct {
- FileName string `json:"name"`
- ArchivedName string `json:"file_name"`
- SizeBytes int64 `json:"size_bytes"`
- ContentType string `json:"content_type"`
- S3Url string `json:"s3_url"`
- Sha256 string `json:"sha256"`
- Origin string `json:"origin"`
- MediaId string `json:"media_id"`
- CreatedTs int64 `json:"created_ts"`
- Uploader string `json:"uploader"`
-}
-
-type Manifest struct {
- Version int `json:"version"`
- EntityId string `json:"entity_id"`
- CreatedTs int64 `json:"created_ts"`
- Media map[string]*ManifestRecord `json:"media"`
-
- // Deprecated: for v1 manifests
- UserId string `json:"user_id,omitempty"`
-}
-
-func StartServerExport(serverName string, s3urls bool, includeData bool, ctx rcontext.RequestContext) (*types.BackgroundTask, string, error) {
- exportId, err := ids.NewUniqueId()
- if err != nil {
- return nil, "", err
- }
-
- db := storage.GetDatabase().GetMetadataStore(ctx)
- task, err := db.CreateBackgroundTask("export_data", map[string]interface{}{
- "server_name": serverName,
- "include_s3_urls": s3urls,
- "include_data": includeData,
- "export_id": exportId,
- })
-
- if err != nil {
- return nil, "", err
- }
-
- go func() {
- // Use a new context in the goroutine
- ctx.Context = context.Background()
- db := storage.GetDatabase().GetMetadataStore(ctx)
-
- ds, err := datastore.PickDatastore(string(common.KindArchives), ctx)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- mediaDb := storage.GetDatabase().GetMediaStore(ctx)
- media, err := mediaDb.GetAllMediaForServer(serverName)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- compileArchive(exportId, serverName, ds, media, s3urls, includeData, ctx)
-
- ctx.Log.Info("Finishing export task")
- err = db.FinishedBackgroundTask(task.ID)
- if err != nil {
- ctx.Log.Error(err)
- ctx.Log.Error("Failed to flag task as finished")
- sentry.CaptureException(err)
- }
- ctx.Log.Info("Finished export")
- }()
-
- return task, exportId, nil
-}
-
-func StartUserExport(userId string, s3urls bool, includeData bool, ctx rcontext.RequestContext) (*types.BackgroundTask, string, error) {
- exportId, err := ids.NewUniqueId()
- if err != nil {
- return nil, "", err
- }
-
- db := storage.GetDatabase().GetMetadataStore(ctx)
- task, err := db.CreateBackgroundTask("export_data", map[string]interface{}{
- "user_id": userId,
- "include_s3_urls": s3urls,
- "include_data": includeData,
- "export_id": exportId,
- })
-
- if err != nil {
- return nil, "", err
- }
-
- go func() {
- // Use a new context in the goroutine
- ctx.Context = context.Background()
- db := storage.GetDatabase().GetMetadataStore(ctx)
-
- ds, err := datastore.PickDatastore(string(common.KindArchives), ctx)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- mediaDb := storage.GetDatabase().GetMediaStore(ctx)
- media, err := mediaDb.GetMediaByUser(userId)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- compileArchive(exportId, userId, ds, media, s3urls, includeData, ctx)
-
- ctx.Log.Info("Finishing export task")
- err = db.FinishedBackgroundTask(task.ID)
- if err != nil {
- ctx.Log.Error(err)
- ctx.Log.Error("Failed to flag task as finished")
- sentry.CaptureException(err)
- }
- ctx.Log.Info("Finished export")
- }()
-
- return task, exportId, nil
-}
-
-func compileArchive(exportId string, entityId string, archiveDs *datastore.DatastoreRef, media []*types.Media, s3urls bool, includeData bool, ctx rcontext.RequestContext) {
- exportDb := storage.GetDatabase().GetExportStore(ctx)
- err := exportDb.InsertExport(exportId, entityId)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- var currentTar *tar.Writer
- var currentTarBytes bytes.Buffer
- part := 0
- parts := make([]*types.ObjectInfo, 0)
- currentSize := int64(0)
-
- persistTar := func() error {
- _ = currentTar.Close()
-
- // compress
- ctx.Log.Info("Compressing tar file")
- gzipBytes := bytes.Buffer{}
- archiver := gzip.NewWriter(&gzipBytes)
- archiver.Name = fmt.Sprintf("export-part-%d.tar", part)
- _, err := io.Copy(archiver, stream_util.BufferToStream(bytes.NewBuffer(currentTarBytes.Bytes())))
- if err != nil {
- return err
- }
- _ = archiver.Close()
-
- ctx.Log.Info("Uploading compressed tar file")
- buf := bytes.NewBuffer(gzipBytes.Bytes())
- size := int64(buf.Len())
- obj, err := archiveDs.UploadFile(stream_util.BufferToStream(buf), size, ctx)
- if err != nil {
- return err
- }
- parts = append(parts, obj)
-
- fname := fmt.Sprintf("export-part-%d.tgz", part)
- err = exportDb.InsertExportPart(exportId, part, size, fname, archiveDs.DatastoreId, obj.Location)
- if err != nil {
- return err
- }
-
- return nil
- }
-
- newTar := func() error {
- if part > 0 {
- ctx.Log.Info("Persisting complete tar file")
- err := persistTar()
- if err != nil {
- return err
- }
- }
-
- ctx.Log.Info("Starting new tar file")
- currentTarBytes = bytes.Buffer{}
- currentTar = tar.NewWriter(¤tTarBytes)
- part = part + 1
- currentSize = 0
-
- return nil
- }
-
- // Start the first tar file
- ctx.Log.Info("Creating first tar file")
- err = newTar()
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- putFile := func(name string, size int64, creationTime time.Time, file io.Reader) error {
- header := &tar.Header{
- Name: name,
- Size: size,
- Mode: int64(0644),
- ModTime: creationTime,
- }
- err := currentTar.WriteHeader(header)
- if err != nil {
- ctx.Log.Error("error writing header")
- return err
- }
-
- i, err := io.Copy(currentTar, file)
- if err != nil {
- ctx.Log.Error("error writing file")
- return err
- }
-
- currentSize += i
-
- return nil
- }
-
- archivedName := func(m *types.Media) string {
- // TODO: Pick the right extension for the file type
- return fmt.Sprintf("%s__%s.obj", m.Origin, m.MediaId)
- }
-
- // Build a manifest first (JSON)
- ctx.Log.Info("Building manifest")
- indexModel := &templating.ExportIndexModel{
- Entity: entityId,
- ExportID: exportId,
- Media: make([]*templating.ExportIndexMediaModel, 0),
- }
- mediaManifest := make(map[string]*ManifestRecord)
- for _, m := range media {
- var s3url string
- if s3urls {
- s3url, err = ds_s3.GetS3URL(m.DatastoreId, m.Location)
- if err != nil {
- ctx.Log.Warn(err)
- }
- }
- mediaManifest[m.MxcUri()] = &ManifestRecord{
- ArchivedName: archivedName(m),
- FileName: m.UploadName,
- SizeBytes: m.SizeBytes,
- ContentType: m.ContentType,
- S3Url: s3url,
- Sha256: m.Sha256Hash,
- Origin: m.Origin,
- MediaId: m.MediaId,
- CreatedTs: m.CreationTs,
- Uploader: m.UserId,
- }
- indexModel.Media = append(indexModel.Media, &templating.ExportIndexMediaModel{
- ExportID: exportId,
- ArchivedName: archivedName(m),
- FileName: m.UploadName,
- SizeBytes: m.SizeBytes,
- SizeBytesHuman: humanize.Bytes(uint64(m.SizeBytes)),
- Origin: m.Origin,
- MediaID: m.MediaId,
- Sha256Hash: m.Sha256Hash,
- ContentType: m.ContentType,
- UploadTs: m.CreationTs,
- UploadDateHuman: util.FromMillis(m.CreationTs).Format(time.UnixDate),
- Uploader: m.UserId,
- })
- }
- manifest := &Manifest{
- Version: 2,
- EntityId: entityId,
- CreatedTs: util.NowMillis(),
- Media: mediaManifest,
- }
- b, err := json.Marshal(manifest)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- ctx.Log.Info("Writing manifest")
- err = putFile("manifest.json", int64(len(b)), time.Now(), stream_util.BufferToStream(bytes.NewBuffer(b)))
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- if includeData {
- ctx.Log.Info("Building and writing index")
- t, err := templating.GetTemplate("export_index")
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
- html := bytes.Buffer{}
- err = t.Execute(&html, indexModel)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
- err = putFile("index.html", int64(html.Len()), time.Now(), stream_util.BufferToStream(bytes.NewBuffer(html.Bytes())))
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- ctx.Log.Info("Including data in the archive")
- for _, m := range media {
- ctx.Log.Info("Downloading ", m.MxcUri())
- s, err := datastore.DownloadStream(ctx, m.DatastoreId, m.Location)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- continue
- }
-
- ctx.Log.Infof("Copying %s to memory", m.MxcUri())
- b := bytes.Buffer{}
- _, err = io.Copy(&b, s)
- if err != nil {
- ctx.Log.Error(err)
- stream_util.DumpAndCloseStream(s)
- sentry.CaptureException(err)
- continue
- }
- stream_util.DumpAndCloseStream(s)
- s = stream_util.BufferToStream(bytes.NewBuffer(b.Bytes()))
-
- ctx.Log.Info("Archiving ", m.MxcUri())
- err = putFile(archivedName(m), m.SizeBytes, time.Unix(0, m.CreationTs*int64(time.Millisecond)), s)
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
-
- if currentSize >= ctx.Config.Archiving.TargetBytesPerPart {
- ctx.Log.Info("Rotating tar")
- err = newTar()
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
- }
- }
- }
-
- if currentSize > 0 {
- ctx.Log.Info("Persisting last tar")
- err = persistTar()
- if err != nil {
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- return
- }
- }
-}
diff --git a/controllers/data_controller/import_controller.go b/controllers/data_controller/import_controller.go
deleted file mode 100644
index a8bd419a511ebe6f2af8330b243fece81a09a6bc..0000000000000000000000000000000000000000
--- a/controllers/data_controller/import_controller.go
+++ /dev/null
@@ -1,414 +0,0 @@
-package data_controller
-
-import (
- "archive/tar"
- "bytes"
- "compress/gzip"
- "context"
- "database/sql"
- "encoding/json"
- "errors"
- "io"
- "net/http"
- "sync"
-
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util/ids"
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-
- "github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/controllers/upload_controller"
- "github.com/turt2live/matrix-media-repo/storage"
- "github.com/turt2live/matrix-media-repo/storage/datastore"
- "github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type importUpdate struct {
- stop bool
- fileMap map[string]*bytes.Buffer
- onDoneChan chan bool
-}
-
-var openImports = &sync.Map{} // importId => updateChan
-
-func StartImport(data io.Reader, ctx rcontext.RequestContext) (*types.BackgroundTask, string, error) {
- // Prepare the first update for the import (sync, so we can error)
- // We do this before anything else because if the archive is invalid then we shouldn't
- // even bother with an import.
- results, err := processArchive(data)
- if err != nil {
- return nil, "", err
- }
-
- importId, err := ids.NewUniqueId()
- if err != nil {
- return nil, "", err
- }
-
- db := storage.GetDatabase().GetMetadataStore(ctx)
- task, err := db.CreateBackgroundTask("import_data", map[string]interface{}{
- "import_id": importId,
- })
-
- if err != nil {
- return nil, "", err
- }
-
- // Start the import and send it its first update
- updateChan := make(chan *importUpdate)
- go doImport(updateChan, task.ID, importId, ctx)
- openImports.Store(importId, updateChan)
- updateChan <- &importUpdate{stop: false, fileMap: results}
-
- return task, importId, nil
-}
-
-func AppendToImport(importId string, data io.Reader, withReturnChan bool) (chan bool, error) {
- runningImport, ok := openImports.Load(importId)
- if !ok || runningImport == nil {
- return nil, errors.New("import not found or it has been closed")
- }
-
- results, err := processArchive(data)
- if err != nil {
- return nil, err
- }
-
- // Repeat the safety check - the archive processing can take a bit
- runningImport, ok = openImports.Load(importId)
- if !ok || runningImport == nil {
- return nil, nil
- }
-
- var doneChan chan bool
- if withReturnChan {
- doneChan = make(chan bool)
- }
- updateChan := runningImport.(chan *importUpdate)
- updateChan <- &importUpdate{stop: false, fileMap: results, onDoneChan: doneChan}
-
- return doneChan, nil
-}
-
-func StopImport(importId string) error {
- runningImport, ok := openImports.Load(importId)
- if !ok || runningImport == nil {
- return errors.New("import not found or it has been closed")
- }
-
- updateChan := runningImport.(chan *importUpdate)
- updateChan <- &importUpdate{stop: true, fileMap: make(map[string]*bytes.Buffer)}
-
- return nil
-}
-
-func processArchive(data io.Reader) (map[string]*bytes.Buffer, error) {
- archiver, err := gzip.NewReader(data)
- if err != nil {
- return nil, err
- }
-
- defer archiver.Close()
-
- tarFile := tar.NewReader(archiver)
- index := make(map[string]*bytes.Buffer)
- for {
- header, err := tarFile.Next()
- if err == io.EOF {
- break // we're done
- }
- if err != nil {
- return nil, err
- }
-
- if header == nil {
- continue // skip this weird file
- }
- if header.Typeflag != tar.TypeReg {
- continue // skip directories and other stuff
- }
-
- // Copy the file into our index
- buf := &bytes.Buffer{}
- _, err = io.Copy(buf, tarFile)
- if err != nil {
- return nil, err
- }
- buf = bytes.NewBuffer(buf.Bytes()) // clone to reset reader position
- index[header.Name] = buf
- }
-
- return index, nil
-}
-
-func doImport(updateChannel chan *importUpdate, taskId int, importId string, ctx rcontext.RequestContext) {
- defer close(updateChannel)
-
- // Use a new context in the goroutine
- ctx.Context = context.Background()
-
- ctx.Log.Info("Preparing for import...")
- fileMap := make(map[string]*bytes.Buffer)
- stopImport := false
- archiveManifest := &Manifest{}
- haveManifest := false
- imported := make(map[string]bool)
- db := storage.GetDatabase().GetMediaStore(ctx)
- var update *importUpdate
-
- for !stopImport {
- if update != nil && update.onDoneChan != nil {
- ctx.Log.Info("Flagging tar as completed")
- update.onDoneChan <- true
- }
- update = <-updateChannel
- if update.stop {
- ctx.Log.Info("Close requested")
- stopImport = true
- }
-
- // Populate files
- for name, fileBytes := range update.fileMap {
- if _, ok := fileMap[name]; ok {
- ctx.Log.Warnf("Duplicate file name, skipping: %s", name)
- continue // file already known to us
- }
- ctx.Log.Infof("Tracking file: %s", name)
- fileMap[name] = fileBytes
- }
-
- var manifestBuf *bytes.Buffer
- var ok bool
- if manifestBuf, ok = fileMap["manifest.json"]; !ok {
- ctx.Log.Info("No manifest found - waiting for more files")
- continue
- }
-
- if !haveManifest {
- haveManifest = true
- err := json.Unmarshal(manifestBuf.Bytes(), archiveManifest)
- if err != nil {
- ctx.Log.Error("Failed to parse manifest - giving up on import")
- ctx.Log.Error(err)
- sentry.CaptureException(err)
- break
- }
- if archiveManifest.Version != 1 && archiveManifest.Version != 2 {
- ctx.Log.Error("Unsupported archive version")
- break
- }
- if archiveManifest.Version == 1 {
- archiveManifest.EntityId = archiveManifest.UserId
- }
- if archiveManifest.EntityId == "" {
- ctx.Log.Error("Invalid manifest: no entity")
- break
- }
- if archiveManifest.Media == nil {
- ctx.Log.Error("Invalid manifest: no media")
- break
- }
- ctx.Log.Infof("Using manifest for %s (v%d) created %d", archiveManifest.EntityId, archiveManifest.Version, archiveManifest.CreatedTs)
- }
-
- if !haveManifest {
- // Without a manifest we can't import anything
- continue
- }
-
- toClear := make([]string, 0)
- doClear := true
- for mxc, record := range archiveManifest.Media {
- _, found := imported[mxc]
- if found {
- continue // already imported
- }
-
- _, err := db.Get(record.Origin, record.MediaId)
- if err == nil {
- ctx.Log.Info("Media already imported: " + record.Origin + "/" + record.MediaId)
-
- // flag as imported and move on
- imported[mxc] = true
- continue
- }
-
- userId := archiveManifest.EntityId
- if userId[0] != '@' {
- userId = "" // assume none for now
- }
-
- kind := common.KindLocalMedia
- serverName := archiveManifest.EntityId
- if userId != "" {
- _, s, err := util.SplitUserId(userId)
- if err != nil {
- ctx.Log.Errorf("Invalid user ID: %s", userId)
- serverName = ""
- } else {
- serverName = s
- }
- }
- if !util.IsServerOurs(serverName) {
- kind = common.KindRemoteMedia
- }
-
- ctx.Log.Infof("Attempting to import %s for %s", mxc, archiveManifest.EntityId)
- buf, found := fileMap[record.ArchivedName]
- if found {
- ctx.Log.Info("Using file from memory")
- closer := stream_util.BufferToStream(buf)
- _, err := upload_controller.StoreDirect(nil, closer, record.SizeBytes, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx, true)
- if err != nil {
- ctx.Log.Errorf("Error importing file: %s", err.Error())
- doClear = false // don't clear things on error
- sentry.CaptureException(err)
- continue
- }
- toClear = append(toClear, record.ArchivedName)
- } else if record.S3Url != "" {
- ctx.Log.Info("Using S3 URL")
- endpoint, bucket, location, err := ds_s3.ParseS3URL(record.S3Url)
- if err != nil {
- ctx.Log.Errorf("Error importing file: %s", err.Error())
- sentry.CaptureException(err)
- continue
- }
-
- ctx.Log.Infof("Seeing if a datastore for %s/%s exists", endpoint, bucket)
- datastores, err := datastore.GetAvailableDatastores(ctx)
- if err != nil {
- ctx.Log.Errorf("Error locating datastore: %s", err.Error())
- sentry.CaptureException(err)
- continue
- }
- imported := false
- for _, ds := range datastores {
- if ds.Type != "s3" {
- continue
- }
-
- tmplUrl, err := ds_s3.GetS3URL(ds.DatastoreId, location)
- if err != nil {
- ctx.Log.Errorf("Error investigating s3 datastore: %s", err.Error())
- sentry.CaptureException(err)
- continue
- }
- if tmplUrl == record.S3Url {
- ctx.Log.Infof("File matches! Assuming the file has been uploaded already")
-
- existingRecord, err := db.Get(record.Origin, record.MediaId)
- if err != nil && err != sql.ErrNoRows {
- ctx.Log.Errorf("Error testing file in database: %s", err.Error())
- sentry.CaptureException(err)
- break
- }
- if err != sql.ErrNoRows && existingRecord != nil {
- ctx.Log.Warnf("Media %s already exists - skipping without altering record", existingRecord.MxcUri())
- imported = true
- break
- }
-
- // Use the user ID (if any) as the uploader as a default. If this is an import
- // for a server then we use the recorded one, if any is available.
- uploader := userId
- if userId == "" {
- uploader = record.Uploader
- }
- media := &types.Media{
- Origin: record.Origin,
- MediaId: record.MediaId,
- UploadName: record.FileName,
- ContentType: record.ContentType,
- UserId: uploader,
- Sha256Hash: record.Sha256,
- SizeBytes: record.SizeBytes,
- DatastoreId: ds.DatastoreId,
- Location: location,
- CreationTs: record.CreatedTs,
- }
-
- err = db.Insert(media)
- if err != nil {
- ctx.Log.Errorf("Error creating media record: %s", err.Error())
- sentry.CaptureException(err)
- break
- }
-
- ctx.Log.Infof("Media %s has been imported", media.MxcUri())
- imported = true
- break
- }
- }
-
- if !imported {
- ctx.Log.Info("No datastore found - trying to upload by downloading first")
- r, err := http.DefaultClient.Get(record.S3Url)
- if err != nil {
- ctx.Log.Errorf("Error trying to download file from S3 via HTTP: ", err.Error())
- sentry.CaptureException(err)
- continue
- }
-
- _, err = upload_controller.StoreDirect(nil, r.Body, r.ContentLength, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx, true)
- if err != nil {
- ctx.Log.Errorf("Error importing file: %s", err.Error())
- sentry.CaptureException(err)
- continue
- }
- }
- } else {
- ctx.Log.Warn("Missing usable file for import - assuming it will show up in a future upload")
- continue
- }
-
- ctx.Log.Info("Counting file as imported")
- imported[mxc] = true
- }
-
- if doClear {
- ctx.Log.Info("Clearing up memory for imported files...")
- for _, f := range toClear {
- ctx.Log.Infof("Removing %s from memory", f)
- delete(fileMap, f)
- }
- }
-
- ctx.Log.Info("Checking for any unimported files...")
- missingAny := false
- for mxc, _ := range archiveManifest.Media {
- _, found := imported[mxc]
- if found {
- continue // already imported
- }
- missingAny = true
- break
- }
-
- if !missingAny {
- ctx.Log.Info("No more files to import - closing import")
- stopImport = true
- }
- }
-
- // Clean up the last tar file
- if update != nil && update.onDoneChan != nil {
- ctx.Log.Info("Flagging tar as completed")
- update.onDoneChan <- true
- }
-
- openImports.Delete(importId)
-
- ctx.Log.Info("Finishing import task")
- dbMeta := storage.GetDatabase().GetMetadataStore(ctx)
- err := dbMeta.FinishedBackgroundTask(taskId)
- if err != nil {
- ctx.Log.Error(err)
- ctx.Log.Error("Failed to flag task as finished")
- sentry.CaptureException(err)
- }
- ctx.Log.Info("Finished import")
-}
diff --git a/controllers/upload_controller/upload_controller.go b/controllers/upload_controller/upload_controller.go
deleted file mode 100644
index 2f684c92d7ddd21ecf1d629c72a665169b4d6b20..0000000000000000000000000000000000000000
--- a/controllers/upload_controller/upload_controller.go
+++ /dev/null
@@ -1,266 +0,0 @@
-package upload_controller
-
-import (
- "bytes"
- "errors"
- "io"
-
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/plugins"
- "github.com/turt2live/matrix-media-repo/storage"
- "github.com/turt2live/matrix-media-repo/storage/datastore"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-const NoApplicableUploadUser = ""
-
-type AlreadyUploadedFile struct {
- DS *datastore.DatastoreRef
- ObjectInfo *types.ObjectInfo
-}
-
-func trackUploadAsLastAccess(ctx rcontext.RequestContext, media *types.Media) {
- err := storage.GetDatabase().GetMetadataStore(ctx).UpsertLastAccess(media.Sha256Hash, util.NowMillis())
- if err != nil {
- logrus.Warn("Failed to upsert the last access time: ", err)
- }
-}
-
-func checkSpam(contents []byte, filename string, contentType string, userId string, origin string, mediaId string) error {
- spam, err := plugins.CheckForSpam(bytes.NewBuffer(contents), filename, contentType, userId, origin, mediaId)
- if err != nil {
- logrus.Warn("Error checking spam - assuming not spam: " + err.Error())
- sentry.CaptureException(err)
- return nil
- }
- if spam {
- return common.ErrMediaQuarantined
- }
- return nil
-}
-
-func StoreDirect(f *AlreadyUploadedFile, contents io.ReadCloser, expectedSize int64, contentType string, filename string, userId string, origin string, mediaId string, kind common.Kind, ctx rcontext.RequestContext, filterUserDuplicates bool) (*types.Media, error) {
- var err error
- var ds *datastore.DatastoreRef
- var info *types.ObjectInfo
- var contentBytes []byte
- if f == nil {
- dsPicked, err := datastore.PickDatastore(string(kind), ctx)
- if err != nil {
- return nil, err
- }
- ds = dsPicked
-
- contentBytes, err = io.ReadAll(contents)
- if err != nil {
- return nil, err
- }
-
- fInfo, err := ds.UploadFile(stream_util.BytesToStream(contentBytes), expectedSize, ctx)
- if err != nil {
- return nil, err
- }
- info = fInfo
- } else {
- ds = f.DS
- info = f.ObjectInfo
-
- // download the contents for antispam
- contents, err = ds.DownloadFile(info.Location)
- if err != nil {
- return nil, err
- }
- contentBytes, err = io.ReadAll(contents)
- if err != nil {
- return nil, err
- }
- }
-
- db := storage.GetDatabase().GetMediaStore(ctx)
- records, err := db.GetByHash(info.Sha256Hash)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
-
- if len(records) > 0 {
- ctx.Log.Info("Duplicate media for hash ", info.Sha256Hash)
-
- // If the user is a real user (ie: actually uploaded media), then we'll see if there's
- // an exact duplicate that we can return. Otherwise we'll just pick the first record and
- // clone that.
- if filterUserDuplicates && userId != NoApplicableUploadUser {
- for _, record := range records {
- if record.Quarantined {
- ctx.Log.Warn("User attempted to upload quarantined content - rejecting")
- return nil, common.ErrMediaQuarantined
- }
- if record.UserId == userId && record.Origin == origin && record.ContentType == contentType {
- ctx.Log.Info("User has already uploaded this media before - returning unaltered media record")
-
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
-
- trackUploadAsLastAccess(ctx, record)
- return record, nil
- }
- }
- }
-
- err = checkSpam(contentBytes, filename, contentType, userId, origin, mediaId)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
-
- // We'll use the location from the first record
- record := records[0]
- if record.Quarantined {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- ctx.Log.Warn("User attempted to upload quarantined content - rejecting")
- return nil, common.ErrMediaQuarantined
- }
-
- // Double check that we're not about to try and store a record we know about
- for _, knownRecord := range records {
- if knownRecord.Origin == origin && knownRecord.MediaId == mediaId {
- ctx.Log.Info("Duplicate media record found - returning unaltered record")
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- trackUploadAsLastAccess(ctx, knownRecord)
- return knownRecord, nil
- }
- }
-
- media := record
- media.Origin = origin
- media.MediaId = mediaId
- media.UserId = userId
- media.UploadName = filename
- media.ContentType = contentType
- media.CreationTs = util.NowMillis()
-
- err = db.Insert(media)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
-
- // If the media's file exists, we'll delete the temp file
- // If the media's file doesn't exist, we'll move the temp file to where the media expects it to be
- if media.DatastoreId != ds.DatastoreId && media.Location != info.Location {
- ds2, err := datastore.LocateDatastore(ctx, media.DatastoreId)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
- if !ds2.ObjectExists(media.Location) {
- stream, err := ds.DownloadFile(info.Location)
- if err != nil {
- return nil, err
- }
-
- err2 := ds2.OverwriteObject(media.Location, stream, ctx)
- if err2 != nil {
- ctx.Log.Warn("Error overwriting object", err2)
- sentry.CaptureException(err2)
- }
- err2 = ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- } else {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- }
- }
-
- trackUploadAsLastAccess(ctx, media)
- return media, nil
- }
-
- // The media doesn't already exist - save it as new
-
- if info.SizeBytes <= 0 {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, errors.New("file has no contents")
- }
-
- err = checkSpam(contentBytes, filename, contentType, userId, origin, mediaId)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
-
- ctx.Log.Info("Persisting new media record")
-
- media := &types.Media{
- Origin: origin,
- MediaId: mediaId,
- UploadName: filename,
- ContentType: contentType,
- UserId: userId,
- Sha256Hash: info.Sha256Hash,
- SizeBytes: info.SizeBytes,
- DatastoreId: ds.DatastoreId,
- Location: info.Location,
- CreationTs: util.NowMillis(),
- }
-
- err = db.Insert(media)
- if err != nil {
- err2 := ds.DeleteObject(info.Location) // delete temp object
- if err2 != nil {
- ctx.Log.Warn("Error deleting temporary upload", err2)
- sentry.CaptureException(err2)
- }
- return nil, err
- }
-
- trackUploadAsLastAccess(ctx, media)
- return media, nil
-}
diff --git a/database/table_blurhashes.go b/database/table_blurhashes.go
index 8bf0339dba9c9a276266a90a82fb0c24fa2967fd..2f4bfbde1010457af3e1238ac692bd1c2006d46f 100644
--- a/database/table_blurhashes.go
+++ b/database/table_blurhashes.go
@@ -55,7 +55,7 @@ func (s *blurhashesTableWithContext) Get(hash string) (string, error) {
row := s.statements.selectBlurhash.QueryRowContext(s.ctx, hash)
val := ""
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = ""
}
diff --git a/database/table_expiring_media.go b/database/table_expiring_media.go
index 416d1c37df07e0f6432916dddca4da117c1fd56d..5c3fac2bed4fa76e0007810797c9b049090f18b5 100644
--- a/database/table_expiring_media.go
+++ b/database/table_expiring_media.go
@@ -72,7 +72,7 @@ func (s *expiringMediaTableWithContext) ByUserCount(userId string) (int64, error
row := s.statements.selectExpiringMediaByUserCount.QueryRowContext(s.ctx, userId, util.NowMillis())
val := int64(0)
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = 0
}
@@ -83,7 +83,7 @@ func (s *expiringMediaTableWithContext) Get(origin string, mediaId string) (*DbE
row := s.statements.selectExpiringMediaById.QueryRowContext(s.ctx, origin, mediaId)
val := &DbExpiringMedia{}
err := row.Scan(&val.Origin, &val.MediaId, &val.UserId, &val.ExpiresTs)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
diff --git a/database/table_export_parts.go b/database/table_export_parts.go
index a6ecf447717aaa32e099c9f979316a66ae57f657..b82ad9b93cf1ad8ff579d10fa5f8470de6483ab4 100644
--- a/database/table_export_parts.go
+++ b/database/table_export_parts.go
@@ -64,7 +64,7 @@ func (s *exportPartsTableWithContext) GetForExport(exportId string) ([]*DbExport
results := make([]*DbExportPart, 0)
rows, err := s.statements.selectExportPartsById.QueryContext(s.ctx, exportId)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
@@ -83,7 +83,7 @@ func (s *exportPartsTableWithContext) Get(exportId string, partNum int) (*DbExpo
row := s.statements.selectExportPartById.QueryRowContext(s.ctx, exportId, partNum)
val := &DbExportPart{}
err := row.Scan(&val.ExportId, &val.PartNum, &val.SizeBytes, &val.FileName, &val.DatastoreId, &val.Location)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
diff --git a/database/table_exports.go b/database/table_exports.go
index ea306f305d1e6dbb91232268b2c9c4e57f4d8808..bf200df19df49806559e9ece50c0143bd880f164 100644
--- a/database/table_exports.go
+++ b/database/table_exports.go
@@ -60,7 +60,7 @@ func (s *exportsTableWithContext) GetEntity(exportId string) (string, error) {
row := s.statements.selectExportEntity.QueryRowContext(s.ctx, exportId)
val := ""
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = ""
}
diff --git a/database/table_media.go b/database/table_media.go
index edb811ec937ee78d76e3eae5cd3bb7c78e3fa5a4..5e3fcd2095c71281cc92d43ca27c28be4f254fc5 100644
--- a/database/table_media.go
+++ b/database/table_media.go
@@ -157,7 +157,7 @@ func (s *mediaTableWithContext) GetDistinctDatastoreIds() ([]string, error) {
results := make([]string, 0)
rows, err := s.statements.selectDistinctMediaDatastoreIds.QueryContext(s.ctx)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
@@ -179,7 +179,7 @@ func (s *mediaTableWithContext) IsHashQuarantined(sha256hash string) (bool, erro
row := s.statements.selectMediaIsQuarantinedByHash.QueryRowContext(s.ctx, sha256hash)
val := false
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = false
}
@@ -189,7 +189,7 @@ func (s *mediaTableWithContext) IsHashQuarantined(sha256hash string) (bool, erro
func (s *mediaTableWithContext) scanRows(rows *sql.Rows, err error) ([]*DbMedia, error) {
results := make([]*DbMedia, 0)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
@@ -253,7 +253,7 @@ func (s *mediaTableWithContext) GetById(origin string, mediaId string) (*DbMedia
row := s.statements.selectMediaById.QueryRowContext(s.ctx, origin, mediaId)
val := &DbMedia{Locatable: &Locatable{}}
err := row.Scan(&val.Origin, &val.MediaId, &val.UploadName, &val.ContentType, &val.UserId, &val.Sha256Hash, &val.SizeBytes, &val.CreationTs, &val.Quarantined, &val.DatastoreId, &val.Location)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
@@ -264,7 +264,7 @@ func (s *mediaTableWithContext) ByUserCount(userId string) (int64, error) {
row := s.statements.selectMediaByUserCount.QueryRowContext(s.ctx, userId)
val := int64(0)
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = 0
}
@@ -275,7 +275,7 @@ func (s *mediaTableWithContext) IdExists(origin string, mediaId string) (bool, e
row := s.statements.selectMediaExists.QueryRowContext(s.ctx, origin, mediaId)
val := false
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = false
}
@@ -286,7 +286,7 @@ func (s *mediaTableWithContext) LocationExists(datastoreId string, location stri
row := s.statements.selectMediaByLocationExists.QueryRowContext(s.ctx, datastoreId, location)
val := false
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = false
}
diff --git a/database/table_media_attributes.go b/database/table_media_attributes.go
index 2513fc55fa6ca8faf8a21cdc27dda98f3f9ab525..de50010f11486dfc4c332eada0cbc4c8f163c7a5 100644
--- a/database/table_media_attributes.go
+++ b/database/table_media_attributes.go
@@ -62,7 +62,7 @@ func (s *mediaAttributesTableWithContext) Get(origin string, mediaId string) (*D
row := s.statements.selectMediaAttributes.QueryRowContext(s.ctx, origin, mediaId)
val := &DbMediaAttributes{}
err := row.Scan(&val.Origin, &val.MediaId, &val.Purpose)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
diff --git a/database/table_reserved_media.go b/database/table_reserved_media.go
index 58047c68e65c62a0c3795ed448390cccc213d09d..d862644b2f5efdeb304f7a826906e07860718f54 100644
--- a/database/table_reserved_media.go
+++ b/database/table_reserved_media.go
@@ -56,7 +56,7 @@ func (s *reservedMediaTableWithContext) IdExists(origin string, mediaId string)
row := s.statements.selectReservedMediaExists.QueryRowContext(s.ctx, origin, mediaId)
val := false
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = false
}
diff --git a/database/table_tasks.go b/database/table_tasks.go
index 511094d8555cc462e91789711afe8ea0453e846e..eb82414da9e776e5f9231610f81e644056ef3926 100644
--- a/database/table_tasks.go
+++ b/database/table_tasks.go
@@ -83,7 +83,7 @@ func (s *tasksTableWithContext) Get(id int) (*DbTask, error) {
row := s.statements.selectTask.QueryRowContext(s.ctx, id)
val := &DbTask{}
err := row.Scan(&val.TaskId, &val.Name, &val.Params, &val.StartTs, &val.EndTs)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
@@ -98,7 +98,7 @@ func (s *tasksTableWithContext) GetAll(includingFinished bool) ([]*DbTask, error
}
rows, err := q.QueryContext(s.ctx)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
diff --git a/database/table_thumbnails.go b/database/table_thumbnails.go
index 67de1760f2bdd6152641a030ed99f5689f2ff556..280fa0d6ae3354e4e8b2fc05baf73f571b3d3012 100644
--- a/database/table_thumbnails.go
+++ b/database/table_thumbnails.go
@@ -93,7 +93,7 @@ func (s *thumbnailsTableWithContext) GetByParams(origin string, mediaId string,
row := s.statements.selectThumbnailByParams.QueryRowContext(s.ctx, origin, mediaId, width, height, method, animated)
val := &DbThumbnail{Locatable: &Locatable{}}
err := row.Scan(&val.Origin, &val.MediaId, &val.ContentType, &val.Width, &val.Height, &val.Method, &val.Animated, &val.Sha256Hash, &val.SizeBytes, &val.CreationTs, &val.DatastoreId, &val.Location)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = nil
}
@@ -103,7 +103,7 @@ func (s *thumbnailsTableWithContext) GetByParams(origin string, mediaId string,
func (s *thumbnailsTableWithContext) scanRows(rows *sql.Rows, err error) ([]*DbThumbnail, error) {
results := make([]*DbThumbnail, 0)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
@@ -140,7 +140,7 @@ func (s *thumbnailsTableWithContext) LocationExists(datastoreId string, location
row := s.statements.selectThumbnailByLocationExists.QueryRowContext(s.ctx, datastoreId, location)
val := false
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = false
}
diff --git a/database/table_url_previews.go b/database/table_url_previews.go
index e4a28ce97e86ce7230a5ae0182dc33f29ced3f24..633fa1bf9798f01c38820270435686fdc12a40ac 100644
--- a/database/table_url_previews.go
+++ b/database/table_url_previews.go
@@ -68,7 +68,7 @@ func (s *urlPreviewsTableWithContext) Get(url string, ts int64, languageHeader s
row := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, ts, languageHeader)
val := &DbUrlPreview{}
err := row.Scan(&val.Url, &val.ErrorCode, &val.BucketTs, &val.SiteUrl, &val.SiteName, &val.ResourceType, &val.Description, &val.Title, &val.ImageMxc, &val.ImageType, &val.ImageSize, &val.ImageWidth, &val.ImageHeight, &val.LanguageHeader)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return val, err
diff --git a/database/table_user_stats.go b/database/table_user_stats.go
index 3840ef25823aaedc077605fb05147a439020ec36..d94b49ecf1c84925c327e2d5f1afb27c656910a8 100644
--- a/database/table_user_stats.go
+++ b/database/table_user_stats.go
@@ -45,7 +45,7 @@ func (s *userStatsTableWithContext) UserUploadedBytes(userId string) (int64, err
row := s.statements.selectUserStatsUploadedBytes.QueryRowContext(s.ctx, userId)
val := int64(0)
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = 0
}
diff --git a/database/virtualtable_metadata.go b/database/virtualtable_metadata.go
index fac2c0ea9ae7de311d8af9547eddb6ecd5d68ca2..cf583fbfd051e430c15520c85090050da3b0b32d 100644
--- a/database/virtualtable_metadata.go
+++ b/database/virtualtable_metadata.go
@@ -104,7 +104,7 @@ func (s *metadataVirtualTableWithContext) EstimateDatastoreSize(datastoreId stri
row := s.statements.selectEstimatedDatastoreSize.QueryRowContext(s.ctx, datastoreId)
val := int64(0)
err := row.Scan(&val)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
val = 0
}
@@ -116,7 +116,7 @@ func (s *metadataVirtualTableWithContext) ByteUsageForServer(serverName string)
media := int64(0)
thumbs := int64(0)
err := row.Scan(&media, &thumbs)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
media = int64(0)
thumbs = int64(0)
@@ -129,7 +129,7 @@ func (s *metadataVirtualTableWithContext) CountUsageForServer(serverName string)
media := int64(0)
thumbs := int64(0)
err := row.Scan(&media, &thumbs)
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
err = nil
media = int64(0)
thumbs = int64(0)
@@ -183,7 +183,7 @@ func (s *metadataVirtualTableWithContext) UnoptimizedSynapseUserStatsPage(server
results := make([]*DbSynUserStat, 0)
rows, err := s.statements.db.QueryContext(s.ctx, sqlPageQ, sqlParams...)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, 0, nil
}
return nil, 0, err
@@ -203,7 +203,7 @@ func (s *metadataVirtualTableWithContext) UnoptimizedSynapseUserStatsPage(server
total := int64(0)
err = row.Scan(&total)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return make([]*DbSynUserStat, 0), 0, nil
}
return nil, 0, err
@@ -215,7 +215,7 @@ func (s *metadataVirtualTableWithContext) UnoptimizedSynapseUserStatsPage(server
func (s *metadataVirtualTableWithContext) scanLastAccess(rows *sql.Rows, err error) ([]*VirtLastAccess, error) {
results := make([]*VirtLastAccess, 0)
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return results, nil
}
return nil, err
diff --git a/datastores/upload.go b/datastores/upload.go
index a64b96206429616d2c5c4d90c2215f068d1a467c..813300b305502fd48c179c1f7f1beccf2fedfc99 100644
--- a/datastores/upload.go
+++ b/datastores/upload.go
@@ -49,12 +49,11 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read
metrics.S3Operations.With(prometheus.Labels{"operation": "StatObject"}).Inc()
_, err = s3c.client.StatObject(ctx.Context, s3c.bucket, objectName, minio.StatObjectOptions{})
if err != nil {
- if merr, ok := err.(minio.ErrorResponse); ok {
+ var merr minio.ErrorResponse
+ if errors.As(err, &merr) {
if merr.Code == "NoSuchKey" || merr.StatusCode == http.StatusNotFound {
exists = false
}
- } else {
- return "", err
}
}
}
diff --git a/docs/contrib/delegation.md b/docs/contrib/delegation.md
index 7e646a88f4469438b960d77383d29c4f0142b9bb..7359093829536c8ae12497a0f6a497426392cf2a 100644
--- a/docs/contrib/delegation.md
+++ b/docs/contrib/delegation.md
@@ -17,12 +17,12 @@ media-repo.yaml:
```yaml
repo:
useForwaredHost: true # See notes below
- ...
+ #...
homeservers:
- name: example.com
csApi: https://matrix.example.com # The base URL to where the homeserver can actually be reached
- ...
+ #...
```
A full sample config can be found [here](https://github.com/turt2live/matrix-media-repo/blob/master/config.sample.yaml).
diff --git a/go.mod b/go.mod
index b35c37496cbde25fd71788032e634ccb13b33e85..9e5eb8330d5d58a45d8236f1dec47d53f4c4cb8c 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,6 @@ go 1.19
require (
github.com/DavidHuie/gomigrate v0.0.0-20190826182718-4adc4b3de142
- github.com/Jeffail/tunny v0.1.4
github.com/PuerkitoBio/goquery v1.8.1
github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
github.com/alioygur/is v1.0.3
@@ -31,8 +30,6 @@ require (
github.com/kettek/apng v0.0.0-20220823221153-ff692776a607
github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
github.com/lib/pq v1.10.9
- github.com/minio/minio-go/v6 v6.0.57
- github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.16.0
@@ -86,7 +83,6 @@ require (
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mewkiz/flac v1.0.8 // indirect
github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 // indirect
- github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
diff --git a/go.sum b/go.sum
index 9b35ae6004c42801b540882a82637bc18109db22..0c0caa4891ab4e640be79a34470de6fdf25b0d1a 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,6 @@ github.com/DavidHuie/gomigrate v0.0.0-20190826182718-4adc4b3de142 h1:pfeJevnIXt4
github.com/DavidHuie/gomigrate v0.0.0-20190826182718-4adc4b3de142/go.mod h1:F3GZLX+VN44AjFiyKD8++nq8sVE0Sw3bOhhQ3mUffnM=
github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
-github.com/Jeffail/tunny v0.1.4 h1:chtpdz+nUtaYQeCKlNBg6GycFF/kGVHOr6A3cmzTJXs=
-github.com/Jeffail/tunny v0.1.4/go.mod h1:P8xAx4XQl0xsuhjX1DtfaMDCSuavzdb2rwbd0lk+fvo=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 h1:zvgtcRb2B5gynWjm+Fc9oJZPHXwmcgyH0xCcNm6Rmo4=
@@ -68,7 +66,6 @@ github.com/dsoprea/go-utility/v2 v2.0.0-20221003142440-7a1927d49d9d/go.mod h1:LV
github.com/dsoprea/go-utility/v2 v2.0.0-20221003160719-7bc88537c05e/go.mod h1:VZ7cB0pTjm1ADBWhJUOHESu4ZYy9JN+ZPqjfiW09EPU=
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 h1:DilThiXje0z+3UQ5YjYiSRRzVdtamFpvBQXKwMglWqw=
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349/go.mod h1:4GC5sXji84i/p+irqghpPFZBF8tRN/Q7+700G0/DLe8=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c h1:MEV1LrQtCBGacXajlT4CSuYWbZuLl/qaZVqwoOmwAbU=
@@ -185,7 +182,6 @@ github.com/jfreymuth/vorbis v1.0.2/go.mod h1:DoftRo4AznKnShRl1GxiTFCseHr4zR9BN3T
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
@@ -200,11 +196,9 @@ github.com/kettek/apng v0.0.0-20220823221153-ff692776a607/go.mod h1:x78/VRQYKuCf
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -240,32 +234,23 @@ github.com/mewkiz/flac v1.0.8/go.mod h1:l7dt5uFY724eKVkHQtAJAQSkhpC3helU3RDxN0ES
github.com/mewkiz/pkg v0.0.0-20190919212034-518ade7978e2/go.mod h1:3E2FUC/qYUfM8+r9zAwpeHJzqRVVMIYnpzD/clwWxyA=
github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 h1:tnAPMExbRERsyEYkmR1YjhTgDM0iqyiBYf8ojRXxdbA=
github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14/go.mod h1:QYCFBiH5q6XTHEbWhR0uhR3M9qNPoD2CSQzr0g75kE4=
-github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw=
-github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM=
github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI=
github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg=
-github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
-github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba h1:/Q5vvLs180BFH7u+Nakdrr1B9O9RAxVaIurFQy0c8QQ=
-github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -312,19 +297,16 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM=
github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
-github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -342,7 +324,6 @@ github.com/tebeka/strftime v0.1.3/go.mod h1:7wJm3dZlpr4l/oVK0t1HYIc4rMzQ2XJlOMIU
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -363,7 +344,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -395,7 +375,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -475,7 +454,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
diff --git a/homeserver_interop/synapse/db.go b/homeserver_interop/synapse/db.go
index 57494cec71025242ace5a5b7399e104afd9dfe1a..503299abcc01ca144ef488a97f78f2aee5a0c9ef 100644
--- a/homeserver_interop/synapse/db.go
+++ b/homeserver_interop/synapse/db.go
@@ -2,6 +2,7 @@ package synapse
import (
"database/sql"
+ "errors"
_ "github.com/lib/pq" // postgres driver
)
@@ -45,7 +46,7 @@ func OpenDatabase(connectionString string) (*SynDatabase, error) {
func (d *SynDatabase) GetAllMedia() ([]*LocalMedia, error) {
rows, err := d.statements.selectLocalMedia.Query()
if err != nil {
- if err == sql.ErrNoRows {
+ if errors.Is(err, sql.ErrNoRows) {
return []*LocalMedia{}, nil // no records
}
return nil, err
diff --git a/matrix/federation.go b/matrix/federation.go
index e76c58b3cdd3d4e7a070231d0792dc9dd5dd90c0..0ddda378dfd09d8fe28e5f8addabb00b204dddce 100644
--- a/matrix/federation.go
+++ b/matrix/federation.go
@@ -55,8 +55,9 @@ func getFederationBreaker(hostname string) *circuit.Breaker {
return cb
}
-// Note: URL lookups are not covered by the breaker because otherwise it might never close.
func GetServerApiUrl(hostname string) (string, string, error) {
+ // dev note: URL lookups are not covered by the breaker because otherwise it might never close.
+
logrus.Debug("Getting server API URL for " + hostname)
// Check to see if we've cached this hostname at all
diff --git a/metrics/webserver.go b/metrics/webserver.go
index d3ddfd3e043d4ae0b59f7cc7107bcdd1a68e9563..67bebb47daa98bb0dc237ddda9a22fd976dc4b9b 100644
--- a/metrics/webserver.go
+++ b/metrics/webserver.go
@@ -2,6 +2,7 @@ package metrics
import (
"context"
+ "errors"
"net"
"net/http"
"strconv"
@@ -32,8 +33,9 @@ func Init() {
address := net.JoinHostPort(config.Get().Metrics.BindAddress, strconv.Itoa(config.Get().Metrics.Port))
srv = &http.Server{Addr: address, Handler: rtr}
go func() {
+ //goland:noinspection HttpUrlsUsage
logrus.WithField("address", address).Info("Started metrics listener. Listening at http://" + address)
- if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+ if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
sentry.CaptureException(err)
logrus.Fatal(err)
}
diff --git a/pipelines/_steps/datastore_op/put_and_return_stream.go b/pipelines/_steps/datastore_op/put_and_return_stream.go
index 975c1d8075368af7ffa52ea55b614afb043b92cb..f2e975a34148aa0f04d1ee44635b03dab8f529d9 100644
--- a/pipelines/_steps/datastore_op/put_and_return_stream.go
+++ b/pipelines/_steps/datastore_op/put_and_return_stream.go
@@ -32,7 +32,9 @@ func PutAndReturnStream(ctx rcontext.RequestContext, origin string, mediaId stri
pr, pw := io.Pipe()
tee := io.TeeReader(input, pw)
- defer pw.CloseWithError(errors.New("failed to finish write"))
+ defer func(pw *io.PipeWriter, err error) {
+ _ = pw.CloseWithError(err)
+ }(pw, errors.New("failed to finish write"))
wg := new(sync.WaitGroup)
wg.Add(2)
diff --git a/pipelines/_steps/upload/generate_media_id.go b/pipelines/_steps/upload/generate_media_id.go
index 4fd86792ce22e60137b58b4bd53517f934b1cf99..01dd1b83d4f0497d4f79e2df22bf99cec04c16f3 100644
--- a/pipelines/_steps/upload/generate_media_id.go
+++ b/pipelines/_steps/upload/generate_media_id.go
@@ -20,7 +20,7 @@ func GenerateMediaId(ctx rcontext.RequestContext, origin string) (string, error)
var err error
var exists bool
attempts := 0
- for true {
+ for {
attempts += 1
if attempts > 10 {
return "", errors.New("internal limit reached: unable to generate media ID")
diff --git a/pipelines/_steps/url_preview/preview.go b/pipelines/_steps/url_preview/preview.go
index 3fc75b943f972b74fa35057802c355a61d68ffcb..31bd7a7f47e04a7a796a14090ac8aa3855944db4 100644
--- a/pipelines/_steps/url_preview/preview.go
+++ b/pipelines/_steps/url_preview/preview.go
@@ -1,6 +1,8 @@
package url_preview
import (
+ "errors"
+
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/pool"
"github.com/turt2live/matrix-media-repo/url_previewing/m"
@@ -26,13 +28,13 @@ func Preview(ctx rcontext.RequestContext, targetUrl *m.UrlPayload, languageHeade
}
// Try OpenGraph if that failed
- if err == m.ErrPreviewUnsupported {
+ if errors.Is(err, m.ErrPreviewUnsupported) {
ctx.Log.Debug("Trying OpenGraph previewer")
preview, err = p.GenerateOpenGraphPreview(targetUrl, languageHeader, ctx)
}
// Try scraping if that failed
- if err == m.ErrPreviewUnsupported {
+ if errors.Is(err, m.ErrPreviewUnsupported) {
ctx.Log.Debug("Trying built-in previewer")
preview, err = p.GenerateCalculatedPreview(targetUrl, languageHeader, ctx)
}
diff --git a/pipelines/_steps/url_preview/process.go b/pipelines/_steps/url_preview/process.go
index ed30cf8a58ee8af78500c55675a34a630e5f84fd..dfc241b72d6396c28199af7c992243116fe24086 100644
--- a/pipelines/_steps/url_preview/process.go
+++ b/pipelines/_steps/url_preview/process.go
@@ -1,6 +1,8 @@
package url_preview
import (
+ "errors"
+
"github.com/getsentry/sentry-go"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
@@ -12,11 +14,11 @@ func Process(ctx rcontext.RequestContext, previewUrl string, preview m.PreviewRe
previewDb := database.GetInstance().UrlPreviews.Prepare(ctx)
if err != nil {
- if err == m.ErrPreviewUnsupported {
+ if errors.Is(err, m.ErrPreviewUnsupported) {
err = common.ErrMediaNotFound
}
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
previewDb.InsertError(previewUrl, common.ErrCodeNotFound)
} else {
previewDb.InsertError(previewUrl, common.ErrCodeUnknown)
diff --git a/pipelines/pipeline_download/pipeline.go b/pipelines/pipeline_download/pipeline.go
index 0c5ebef3cd732a8f8e91994538b1519b28957671..f57de2fd449c27cb51eb26e61be0a688b3d824b7 100644
--- a/pipelines/pipeline_download/pipeline.go
+++ b/pipelines/pipeline_download/pipeline.go
@@ -103,7 +103,7 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, opts Do
return r, nil
})
- if err == common.ErrMediaQuarantined {
+ if errors.Is(err, common.ErrMediaQuarantined) {
cancel()
return nil, r, err
}
diff --git a/pipelines/pipeline_thumbnail/pipeline.go b/pipelines/pipeline_thumbnail/pipeline.go
index f8451caea6a5a069a68e1e8ae714dc9292aa67d3..9786eac96ea917be2caf93c53d5ff445372b42ad 100644
--- a/pipelines/pipeline_thumbnail/pipeline.go
+++ b/pipelines/pipeline_thumbnail/pipeline.go
@@ -75,7 +75,7 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, opts Th
// Step 4: Get the associated media record (without stream)
mediaRecord, dr, err := pipeline_download.Execute(ctx, origin, mediaId, opts.ImpliedDownloadOpts())
if err != nil {
- if err == common.ErrMediaQuarantined {
+ if errors.Is(err, common.ErrMediaQuarantined) {
go serveRecord(recordCh, nil) // async function to prevent deadlock
if dr != nil {
dr.Close()
@@ -116,7 +116,7 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, opts Th
// Step 7: Create a limited stream
return download.CreateLimitedStream(ctx, r, opts.StartByte, opts.EndByte)
})
- if err == common.ErrMediaQuarantined {
+ if errors.Is(err, common.ErrMediaQuarantined) {
cancel()
return nil, r, err
}
diff --git a/pipelines/pipeline_upload/pipeline.go b/pipelines/pipeline_upload/pipeline.go
index 8dd30f51259c68a1b7f40e4e66e1fcbefb20b69e..dd534ef837a4cb34b5b49a041cd9c591b26ceacd 100644
--- a/pipelines/pipeline_upload/pipeline.go
+++ b/pipelines/pipeline_upload/pipeline.go
@@ -83,8 +83,12 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, r io.Re
allWriters := io.MultiWriter(cacheW, bhW)
tee := io.TeeReader(reader, allWriters)
- defer bhW.CloseWithError(errors.New("failed to finish write"))
- defer cacheW.CloseWithError(errors.New("failed to finish write"))
+ defer func(bhW *io.PipeWriter, err error) {
+ _ = bhW.CloseWithError(err)
+ }(bhW, errors.New("failed to finish write"))
+ defer func(cacheW *io.PipeWriter, err error) {
+ _ = cacheW.CloseWithError(err)
+ }(cacheW, errors.New("failed to finish write"))
// Step 6: Check quarantine
if err = upload.CheckQuarantineStatus(ctx, sha256hash); err != nil {
diff --git a/plugins/mmr_plugin.go b/plugins/mmr_plugin.go
index 24bc040ea880133d884f11f68b84478049962041..3ef00dc55a5dc694d7781602d3eeda316d216ec0 100644
--- a/plugins/mmr_plugin.go
+++ b/plugins/mmr_plugin.go
@@ -53,7 +53,7 @@ func (p *mmrPlugin) Antispam() (plugin_interfaces.Antispam, error) {
}
p.antispamPlugin = raw.(plugin_interfaces.Antispam)
- p.antispamPlugin.HandleConfig(p.config)
+ _ = p.antispamPlugin.HandleConfig(p.config)
return p.antispamPlugin, nil
}
diff --git a/plugins/plugin_common/handshake.go b/plugins/plugin_common/handshake.go
index c0278fd7cddb40d3a50e4a858fdd20da4e3bf6b1..6a661ffc69a37d97d505c54084310328efb3ee33 100644
--- a/plugins/plugin_common/handshake.go
+++ b/plugins/plugin_common/handshake.go
@@ -4,9 +4,10 @@ import (
"github.com/hashicorp/go-plugin"
)
-// UX, not security
+// dev note: HandshakeConfig is for UX, not security
+
var HandshakeConfig = plugin.HandshakeConfig{
ProtocolVersion: 1,
MagicCookieKey: "MEDIA_REPO_PLUGIN",
MagicCookieValue: "hello world - I am a media repo",
-}
\ No newline at end of file
+}
diff --git a/pool/queue.go b/pool/queue.go
index 18cea5f6f0e63db811164ae0954238b5fd6fe321..9def5b7c86ff2549b7fbb221acfd8cd7edc88759 100644
--- a/pool/queue.go
+++ b/pool/queue.go
@@ -22,6 +22,7 @@ func NewQueue(workers int, name string) (*Queue, error) {
PanicHandler: func(err interface{}) {
logrus.Errorf("Panic from internal queue %s", name)
logrus.Error(err)
+ //goland:noinspection GoTypeAssertionOnErrors
if e, ok := err.(error); ok {
sentry.CaptureException(e)
}
diff --git a/redis_cache/LEGACY_redis.go b/redis_cache/LEGACY_redis.go
deleted file mode 100644
index d9d65fd216cb7773c5d4a6b6ed2c20e9bbb721b4..0000000000000000000000000000000000000000
--- a/redis_cache/LEGACY_redis.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package redis_cache
-
-import (
- "bytes"
- "context"
- "errors"
- "io"
- "time"
-
- "github.com/redis/go-redis/v9"
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
-)
-
-var ErrCacheMiss = errors.New("missed cache")
-var ErrCacheDown = errors.New("all shards appear to be down")
-
-type RedisCache struct {
- ring *redis.Ring
-}
-
-func NewCache(conf config.RedisConfig) *RedisCache {
- addresses := make(map[string]string)
- for _, c := range conf.Shards {
- addresses[c.Name] = c.Address
- }
- ring := redis.NewRing(&redis.RingOptions{
- Addrs: addresses,
- DialTimeout: 10 * time.Second,
- DB: conf.DbNum,
- })
-
- logrus.Info("Contacting Redis shards...")
- _ = ring.ForEachShard(context.Background(), func(ctx context.Context, client *redis.Client) error {
- logrus.Infof("Pinging %s", client.String())
- r, err := client.Ping(ctx).Result()
- if err != nil {
- return err
- }
- logrus.Infof("%s replied with: %s", client.String(), r)
- return nil
- })
-
- return &RedisCache{ring: ring}
-}
-
-func (c *RedisCache) Close() error {
- return c.ring.Close()
-}
-
-func (c *RedisCache) SetStream(ctx rcontext.RequestContext, key string, s io.Reader) error {
- b, err := io.ReadAll(s)
- if err != nil {
- return err
- }
- return c.SetBytes(ctx, key, b)
-}
-
-func (c *RedisCache) GetStream(ctx rcontext.RequestContext, key string) (io.Reader, error) {
- b, err := c.GetBytes(ctx, key)
- if err != nil {
- return nil, err
- }
- return bytes.NewBuffer(b), nil
-}
-
-func (c *RedisCache) SetBytes(ctx rcontext.RequestContext, key string, b []byte) error {
- if c.ring.PoolStats().TotalConns == 0 {
- return ErrCacheDown
- }
- _, err := c.ring.Set(ctx.Context, key, b, time.Duration(0)).Result() // no expiration (zero)
- if err != nil && c.ring.PoolStats().TotalConns == 0 {
- ctx.Log.Error(err)
- return ErrCacheDown
- }
- return err
-}
-
-func (c *RedisCache) GetBytes(ctx rcontext.RequestContext, key string) ([]byte, error) {
- if c.ring.PoolStats().TotalConns == 0 {
- return nil, ErrCacheDown
- }
- r := c.ring.Get(ctx.Context, key)
- if r.Err() != nil {
- if r.Err() == redis.Nil {
- return nil, ErrCacheMiss
- }
- if c.ring.PoolStats().TotalConns == 0 {
- ctx.Log.Error(r.Err())
- return nil, ErrCacheDown
- }
- return nil, r.Err()
- }
-
- b, err := r.Bytes()
- return b, err
-}
diff --git a/redislib/cache.go b/redislib/cache.go
index 3e3d25b5c7a9f28d0b9fddd54c897b87b99551c2..c95336db93ce6d0fd5a10c3fc06394536e66dec1 100644
--- a/redislib/cache.go
+++ b/redislib/cache.go
@@ -34,7 +34,7 @@ func StoreMedia(ctx rcontext.RequestContext, hash string, content io.Reader, siz
}
buf := make([]byte, appendBufferSize)
- for true {
+ for {
read, err := content.Read(buf)
if err == io.EOF {
break
diff --git a/storage/LEGACY.txt b/storage/LEGACY.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/storage/datastore/datastore.go b/storage/datastore/datastore.go
deleted file mode 100644
index 719a37838505d5a597cb32a9eb3d2f70b2cbfc72..0000000000000000000000000000000000000000
--- a/storage/datastore/datastore.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package datastore
-
-import (
- "errors"
- "fmt"
- "io"
-
- "github.com/getsentry/sentry-go"
- "github.com/turt2live/matrix-media-repo/common"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/storage"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-func GetAvailableDatastores(ctx rcontext.RequestContext) ([]*types.Datastore, error) {
- datastores := make([]*types.Datastore, 0)
- for _, ds := range ctx.Config.DataStores {
- uri := GetUriForDatastore(ds)
-
- dsInstance, err := storage.GetOrCreateDatastoreOfType(rcontext.Initial(), ds.Type, uri)
- if err != nil {
- return nil, err
- }
-
- datastores = append(datastores, dsInstance)
- }
-
- return datastores, nil
-}
-
-func LocateDatastore(ctx rcontext.RequestContext, datastoreId string) (*DatastoreRef, error) {
- ds, err := storage.GetDatabase().GetMediaStore(ctx).GetDatastore(datastoreId)
- if err != nil {
- return nil, err
- }
-
- conf, err := GetDatastoreConfig(ds)
- if err != nil {
- return nil, err
- }
-
- return newDatastoreRef(ds, conf), nil
-}
-
-func DownloadStream(ctx rcontext.RequestContext, datastoreId string, location string) (io.ReadCloser, error) {
- ref, err := LocateDatastore(ctx, datastoreId)
- if err != nil {
- return nil, err
- }
- return ref.DownloadFile(location)
-}
-
-func GetDatastoreConfig(ds *types.Datastore) (config.DatastoreConfig, error) {
- for _, dsConf := range config.UniqueDatastores() {
- if dsConf.Type == ds.Type && GetUriForDatastore(dsConf) == ds.Uri {
- return dsConf, nil
- }
- }
-
- return config.DatastoreConfig{}, errors.New("datastore not found")
-}
-
-func GetUriForDatastore(dsConf config.DatastoreConfig) string {
- if dsConf.Type == "file" {
- path, pathFound := dsConf.Options["path"]
- if !pathFound {
- sentry.CaptureException(errors.New("Missing 'path' on file datastore"))
- logrus.Fatal("Missing 'path' on file datastore")
- }
- return path
- } else if dsConf.Type == "s3" {
- endpoint, epFound := dsConf.Options["endpoint"]
- bucket, bucketFound := dsConf.Options["bucketName"]
- region, regionFound := dsConf.Options["region"]
- if !epFound || !bucketFound {
- sentry.CaptureException(errors.New("Missing 'endpoint' or 'bucketName' on s3 datastore"))
- logrus.Fatal("Missing 'endpoint' or 'bucketName' on s3 datastore")
- }
- if regionFound {
- return fmt.Sprintf("s3://%s/%s?region=%s", endpoint, bucket, region)
- } else {
- return fmt.Sprintf("s3://%s/%s", endpoint, bucket)
- }
- } else {
- sentry.CaptureException(errors.New("unknown datastore type: " + dsConf.Type))
- logrus.Fatal("Unknown datastore type: ", dsConf.Type)
- }
-
- return ""
-}
-
-func PickDatastore(forKind string, ctx rcontext.RequestContext) (*DatastoreRef, error) {
- ctx.Log.Info("Finding a suitable datastore to pick for " + forKind)
- confDatastores := ctx.Config.DataStores
- mediaStore := storage.GetDatabase().GetMediaStore(ctx)
-
- // Figure out which datastores are likely to be useful for us to check against first. This
- // helps speed up later checks which could require significant DB resources (estimating the
- // size of the datastore).
- var possibleDatastores = make([]config.DatastoreConfig, 0)
- for _, dsConf := range confDatastores {
- allowed := common.HasKind(dsConf.MediaKinds, common.Kind(forKind))
- if !allowed {
- continue
- }
-
- possibleDatastores = append(possibleDatastores, dsConf)
- }
-
- var targetDs *types.Datastore
- var targetDsConf config.DatastoreConfig
- var dsSize int64
- for _, dsConf := range possibleDatastores {
-
- ds, err := mediaStore.GetDatastoreByUri(GetUriForDatastore(dsConf))
- if err != nil {
- ctx.Log.Error("Error getting datastore: ", err.Error())
- sentry.CaptureException(err)
- continue
- }
-
- var size int64
-
- if len(possibleDatastores) > 1 {
- size, err = estimatedDatastoreSize(ds, ctx)
- if err != nil {
- ctx.Log.Error("Error estimating datastore size for ", ds.DatastoreId, ": ", err.Error())
- sentry.CaptureException(err)
- continue
- }
- }
-
- if targetDs == nil || size < dsSize {
- targetDs = ds
- targetDsConf = dsConf
- dsSize = size
- }
- }
-
- if targetDs != nil {
- ctx.Log.Info("Using ", targetDs.Uri)
- return newDatastoreRef(targetDs, targetDsConf), nil
- }
-
- return nil, errors.New("failed to pick a datastore: none available")
-}
-
-func estimatedDatastoreSize(ds *types.Datastore, ctx rcontext.RequestContext) (int64, error) {
- return storage.GetDatabase().GetMetadataStore(ctx).GetEstimatedSizeOfDatastore(ds.DatastoreId)
-}
diff --git a/storage/datastore/datastore_ref.go b/storage/datastore/datastore_ref.go
deleted file mode 100644
index 9d8e62dd2cb20e0c00d3e207c9632cc645676513..0000000000000000000000000000000000000000
--- a/storage/datastore/datastore_ref.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package datastore
-
-import (
- "errors"
- "io"
- "os"
- "path"
-
- "github.com/sirupsen/logrus"
- config2 "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/storage/datastore/ds_file"
- "github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type DatastoreRef struct {
- // TODO: Don't blindly copy properties from types.Datastore
- DatastoreId string
- Type string
- Uri string
-
- datastore *types.Datastore
- config config2.DatastoreConfig
-}
-
-func newDatastoreRef(ds *types.Datastore, config config2.DatastoreConfig) *DatastoreRef {
- return &DatastoreRef{
- DatastoreId: ds.DatastoreId,
- Type: ds.Type,
- Uri: ds.Uri,
- datastore: ds,
- config: config,
- }
-}
-
-func (d *DatastoreRef) UploadFile(file io.ReadCloser, expectedLength int64, ctx rcontext.RequestContext) (*types.ObjectInfo, error) {
- ctx = ctx.LogWithFields(logrus.Fields{"datastoreId": d.DatastoreId, "datastoreUri": d.Uri})
-
- if d.Type == "file" {
- return ds_file.PersistFile(d.Uri, file, ctx)
- } else if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return nil, err
- }
- return s3.UploadFile(file, expectedLength, ctx)
- } else {
- return nil, errors.New("unknown datastore type")
- }
-}
-
-func (d *DatastoreRef) DeleteObject(location string) error {
- if d.Type == "file" {
- return ds_file.DeletePersistedFile(d.Uri, location)
- } else if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return err
- }
- return s3.DeleteObject(location)
- } else {
- return errors.New("unknown datastore type")
- }
-}
-
-func (d *DatastoreRef) DownloadFile(location string) (io.ReadCloser, error) {
- if d.Type == "file" {
- return os.Open(path.Join(d.Uri, location))
- } else if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return nil, err
- }
- return s3.DownloadObject(location)
- } else {
- return nil, errors.New("unknown datastore type")
- }
-}
-
-func (d *DatastoreRef) ObjectExists(location string) bool {
- if d.Type == "file" {
- ok, err := util.FileExists(path.Join(d.Uri, location))
- if err != nil {
- return false
- }
- return ok
- } else if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return false
- }
- return s3.ObjectExists(location)
- } else {
- panic("unknown datastore type")
- }
-}
-
-func (d *DatastoreRef) OverwriteObject(location string, stream io.ReadCloser, ctx rcontext.RequestContext) error {
- if d.Type == "file" {
- _, _, err := ds_file.PersistFileAtLocation(path.Join(d.Uri, location), stream, ctx)
- return err
- } else if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return err
- }
- return s3.OverwriteObject(location, stream)
- } else {
- return errors.New("unknown datastore type")
- }
-}
-
-func (d *DatastoreRef) ListObjectIds(ctx rcontext.RequestContext) ([]string, error) {
- if d.Type == "s3" {
- s3, err := ds_s3.GetOrCreateS3Datastore(d.DatastoreId, d.config)
- if err != nil {
- return nil, err
- }
- return s3.ListObjects()
- } else {
- return nil, errors.New("cannot list objects in this datastore type")
- }
-}
diff --git a/storage/datastore/ds_file/file_store.go b/storage/datastore/ds_file/file_store.go
deleted file mode 100644
index 182d2349ecd7c25534d26fb8057459ae2bcf7ee1..0000000000000000000000000000000000000000
--- a/storage/datastore/ds_file/file_store.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package ds_file
-
-import (
- "errors"
- "io"
- "os"
- "path"
-
- "github.com/turt2live/matrix-media-repo/util/ids"
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-func PersistFile(basePath string, file io.ReadCloser, ctx rcontext.RequestContext) (*types.ObjectInfo, error) {
- defer stream_util.DumpAndCloseStream(file)
-
- exists := true
- var primaryContainer string
- var secondaryContainer string
- var fileName string
- var targetDir string
- var targetFile string
- attempts := 0
- for exists {
- fileId, err := ids.NewUniqueId()
- if err != nil {
- return nil, err
- }
-
- primaryContainer = fileId[0:2]
- secondaryContainer = fileId[2:4]
- fileName = fileId[4:]
- targetDir = path.Join(basePath, primaryContainer, secondaryContainer)
- targetFile = path.Join(targetDir, fileName)
-
- ctx.Log.Info("Checking if file exists: " + targetFile)
-
- exists, err = util.FileExists(targetFile)
- attempts++
-
- if err != nil {
- ctx.Log.Error("Error checking if the file exists: ", err)
- }
-
- // Infinite loop protection
- if attempts > 5 {
- return nil, errors.New("failed to find a suitable directory")
- }
- }
-
- err := os.MkdirAll(targetDir, 0755)
- if err != nil {
- return nil, err
- }
-
- sizeBytes, hash, err := PersistFileAtLocation(targetFile, file, ctx)
- if err != nil {
- return nil, err
- }
-
- locationPath := path.Join(primaryContainer, secondaryContainer, fileName)
- return &types.ObjectInfo{
- Location: locationPath,
- Sha256Hash: hash,
- SizeBytes: sizeBytes,
- }, nil
-}
-
-func PersistFileAtLocation(targetFile string, file io.ReadCloser, ctx rcontext.RequestContext) (int64, string, error) {
- defer stream_util.DumpAndCloseStream(file)
-
- f, err := os.OpenFile(targetFile, os.O_WRONLY|os.O_CREATE, 0644)
- if err != nil {
- return 0, "", err
- }
- defer stream_util.DumpAndCloseStream(f)
-
- rfile, wfile := io.Pipe()
- tr := io.TeeReader(file, wfile)
-
- done := make(chan bool)
- defer close(done)
-
- var hash string
- var sizeBytes int64
- var hashErr error
- var writeErr error
-
- go func() {
- defer wfile.Close()
- ctx.Log.Info("Calculating hash of stream...")
- hash, hashErr = stream_util.GetSha256HashOfStream(io.NopCloser(tr))
- ctx.Log.Info("Hash of file is ", hash)
- done <- true
- }()
-
- go func() {
- ctx.Log.Info("Writing file...")
- sizeBytes, writeErr = io.Copy(f, rfile)
- ctx.Log.Info("Wrote ", sizeBytes, " bytes to file")
- done <- true
- }()
-
- for c := 0; c < 2; c++ {
- <-done
- }
-
- if hashErr != nil {
- defer os.Remove(targetFile)
- return 0, "", hashErr
- }
-
- if writeErr != nil {
- return 0, "", writeErr
- }
-
- return sizeBytes, hash, nil
-}
-
-func DeletePersistedFile(basePath string, location string) error {
- err := os.Remove(path.Join(basePath, location))
- if err != nil && os.IsNotExist(err) {
- // It didn't exist, so pretend it was successful
- return nil
- }
- return err
-}
diff --git a/storage/datastore/ds_s3/s3_store.go b/storage/datastore/ds_s3/s3_store.go
deleted file mode 100644
index 3a3914e8b9a34cc620eb72cb38ad380e3a2edfa0..0000000000000000000000000000000000000000
--- a/storage/datastore/ds_s3/s3_store.go
+++ /dev/null
@@ -1,253 +0,0 @@
-package ds_s3
-
-import (
- "errors"
- "fmt"
- "io"
- "os"
- "strconv"
- "strings"
-
- "github.com/minio/minio-go/v6"
- "github.com/turt2live/matrix-media-repo/util/ids"
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/metrics"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-var stores = make(map[string]*s3Datastore)
-
-type s3Datastore struct {
- conf config.DatastoreConfig
- dsId string
- client *minio.Client
- bucket string
- region string
- tempPath string
- storageClass string
-}
-
-func GetOrCreateS3Datastore(dsId string, conf config.DatastoreConfig) (*s3Datastore, error) {
- if s, ok := stores[dsId]; ok {
- return s, nil
- }
-
- endpoint, epFound := conf.Options["endpoint"]
- bucket, bucketFound := conf.Options["bucketName"]
- accessKeyId, keyFound := conf.Options["accessKeyId"]
- accessSecret, secretFound := conf.Options["accessSecret"]
- region, regionFound := conf.Options["region"]
- tempPath, tempPathFound := conf.Options["tempPath"]
- storageClass, storageClassFound := conf.Options["storageClass"]
- if !epFound || !bucketFound || !keyFound || !secretFound {
- return nil, errors.New("invalid configuration: missing s3 options")
- }
- if !tempPathFound {
- logrus.Warn("Datastore ", dsId, " (s3) does not have a tempPath set - this could lead to excessive memory usage by the media repo")
- }
- if !storageClassFound {
- storageClass = "STANDARD"
- }
-
- useSsl := true
- useSslStr, sslFound := conf.Options["ssl"]
- if sslFound && useSslStr != "" {
- useSsl, _ = strconv.ParseBool(useSslStr)
- }
-
- var s3client *minio.Client
- var err error
-
- if regionFound {
- s3client, err = minio.NewWithRegion(endpoint, accessKeyId, accessSecret, useSsl, region)
- } else {
- s3client, err = minio.New(endpoint, accessKeyId, accessSecret, useSsl)
- }
- if err != nil {
- return nil, err
- }
-
- s3ds := &s3Datastore{
- conf: conf,
- dsId: dsId,
- client: s3client,
- bucket: bucket,
- region: region,
- tempPath: tempPath,
- storageClass: storageClass,
- }
- stores[dsId] = s3ds
- return s3ds, nil
-}
-
-func GetS3URL(datastoreId string, location string) (string, error) {
- var store *s3Datastore
- var ok bool
- if store, ok = stores[datastoreId]; !ok {
- return "", errors.New("s3 datastore not found")
- }
-
- // HACK: Surely there's a better way...
- return fmt.Sprintf("https://%s/%s/%s", store.conf.Options["endpoint"], store.bucket, location), nil
-}
-
-func ParseS3URL(s3url string) (string, string, string, error) {
- trimmed := s3url[8:] // trim off https
- parts := strings.Split(trimmed, "/")
- if len(parts) < 3 {
- return "", "", "", errors.New("invalid url")
- }
-
- endpoint := parts[0]
- location := parts[len(parts)-1]
- bucket := strings.Join(parts[1:len(parts)-1], "/")
-
- return endpoint, bucket, location, nil
-}
-
-func (s *s3Datastore) EnsureBucketExists() error {
- found, err := s.client.BucketExists(s.bucket)
- if err != nil {
- return err
- }
- if !found {
- return errors.New("bucket not found")
- }
- return nil
-}
-
-func (s *s3Datastore) EnsureTempPathExists() error {
- err := os.MkdirAll(s.tempPath, os.ModePerm)
- if err != os.ErrExist && err != nil {
- return err
- }
- return nil
-}
-
-func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx rcontext.RequestContext) (*types.ObjectInfo, error) {
- defer stream_util.DumpAndCloseStream(file)
-
- objectName, err := ids.NewUniqueId()
- if err != nil {
- return nil, err
- }
-
- var rs3 io.ReadCloser
- var ws3 io.WriteCloser
- rs3, ws3 = io.Pipe()
- tr := io.TeeReader(file, ws3)
-
- done := make(chan bool)
- defer close(done)
-
- var hash string
- var sizeBytes int64
- var hashErr error
- var uploadErr error
-
- go func() {
- defer ws3.Close()
- ctx.Log.Info("Calculating hash of stream...")
- hash, hashErr = stream_util.GetSha256HashOfStream(io.NopCloser(tr))
- ctx.Log.Info("Hash of file is ", hash)
- done <- true
- }()
-
- go func() {
- if expectedLength <= 0 {
- if s.tempPath != "" {
- ctx.Log.Info("Buffering file to temp path due to unknown file size")
- var f *os.File
- f, uploadErr = os.CreateTemp(s.tempPath, "mr*")
- if uploadErr != nil {
- io.Copy(io.Discard, rs3)
- done <- true
- return
- }
- defer os.Remove(f.Name())
- expectedLength, uploadErr = io.Copy(f, rs3)
- stream_util.DumpAndCloseStream(f)
- f, uploadErr = os.Open(f.Name())
- if uploadErr != nil {
- done <- true
- return
- }
- rs3 = f
- defer stream_util.DumpAndCloseStream(f)
- } else {
- ctx.Log.Warn("Uploading content of unknown length to s3 - this could result in high memory usage")
- expectedLength = -1
- }
- }
- ctx.Log.Info("Uploading file...")
- metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc()
- sizeBytes, uploadErr = s.client.PutObjectWithContext(ctx, s.bucket, objectName, rs3, expectedLength, minio.PutObjectOptions{StorageClass: s.storageClass})
- ctx.Log.Info("Uploaded ", sizeBytes, " bytes to s3")
- done <- true
- }()
-
- for c := 0; c < 2; c++ {
- <-done
- }
-
- obj := &types.ObjectInfo{
- Location: objectName,
- Sha256Hash: hash,
- SizeBytes: sizeBytes,
- }
-
- if hashErr != nil {
- s.DeleteObject(obj.Location)
- return nil, hashErr
- }
-
- if uploadErr != nil {
- return nil, uploadErr
- }
-
- return obj, nil
-}
-
-func (s *s3Datastore) DeleteObject(location string) error {
- logrus.Info("Deleting object from bucket ", s.bucket, ": ", location)
- metrics.S3Operations.With(prometheus.Labels{"operation": "RemoveObject"}).Inc()
- return s.client.RemoveObject(s.bucket, location)
-}
-
-func (s *s3Datastore) DownloadObject(location string) (io.ReadCloser, error) {
- logrus.Info("Downloading object from bucket ", s.bucket, ": ", location)
- metrics.S3Operations.With(prometheus.Labels{"operation": "GetObject"}).Inc()
- return s.client.GetObject(s.bucket, location, minio.GetObjectOptions{})
-}
-
-func (s *s3Datastore) ObjectExists(location string) bool {
- metrics.S3Operations.With(prometheus.Labels{"operation": "StatObject"}).Inc()
- stat, err := s.client.StatObject(s.bucket, location, minio.StatObjectOptions{})
- if err != nil {
- return false
- }
- return stat.Size > 0
-}
-
-func (s *s3Datastore) OverwriteObject(location string, stream io.ReadCloser) error {
- defer stream_util.DumpAndCloseStream(stream)
- metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc()
- _, err := s.client.PutObject(s.bucket, location, stream, -1, minio.PutObjectOptions{StorageClass: s.storageClass})
- return err
-}
-
-func (s *s3Datastore) ListObjects() ([]string, error) {
- doneCh := make(chan struct{})
- defer close(doneCh)
- list := make([]string, 0)
- metrics.S3Operations.With(prometheus.Labels{"operation": "ListObjectsV2"}).Inc()
- for message := range s.client.ListObjectsV2(s.bucket, "", true, doneCh) {
- list = append(list, message.Key)
- }
- return list, nil
-}
diff --git a/storage/ds_utils.go b/storage/ds_utils.go
deleted file mode 100644
index d925348f5ec754df4864c6f7ce89b9fc5223ac5d..0000000000000000000000000000000000000000
--- a/storage/ds_utils.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package storage
-
-import (
- "database/sql"
- "github.com/turt2live/matrix-media-repo/util/ids"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/storage/stores"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-func GetOrCreateDatastoreOfType(ctx rcontext.RequestContext, dsType string, dsUri string) (*types.Datastore, error) {
- mediaService := GetDatabase().GetMediaStore(ctx)
- datastore, err := mediaService.GetDatastoreByUri(dsUri)
- if err != nil && err == sql.ErrNoRows {
- id, err2 := ids.NewUniqueId()
- if err2 != nil {
- ctx.Log.Error("Error generating datastore ID for URI ", dsUri, ": ", err)
- return nil, err2
- }
- datastore = &types.Datastore{
- DatastoreId: id,
- Type: dsType,
- Uri: dsUri,
- }
- err2 = mediaService.InsertDatastore(datastore)
- if err2 != nil {
- ctx.Log.Error("Error creating datastore for URI ", dsUri, ": ", err)
- return nil, err2
- }
- }
- return datastore, nil
-}
-
-func getOrCreateDatastoreWithMediaService(mediaService *stores.MediaStore, basePath string) (*types.Datastore, error) {
- datastore, err := mediaService.GetDatastoreByUri(basePath)
- if err != nil && err == sql.ErrNoRows {
- id, err2 := ids.NewUniqueId()
- if err2 != nil {
- logrus.Error("Error generating datastore ID for base path ", basePath, ": ", err)
- return nil, err2
- }
- datastore = &types.Datastore{
- DatastoreId: id,
- Type: "file",
- Uri: basePath,
- }
- err2 = mediaService.InsertDatastore(datastore)
- if err2 != nil {
- logrus.Error("Error creating datastore for base path ", basePath, ": ", err)
- return nil, err2
- }
- } else if err != nil {
- logrus.Error("Error getting datastore for base path ", basePath, ": ", err)
- return nil, err
- }
-
- return datastore, nil
-}
diff --git a/storage/startup_migrations.go b/storage/startup_migrations.go
deleted file mode 100644
index f1dcb930a8a50efac1c8a383698bb0c05468381b..0000000000000000000000000000000000000000
--- a/storage/startup_migrations.go
+++ /dev/null
@@ -1,120 +0,0 @@
-package storage
-
-import (
- "github.com/getsentry/sentry-go"
- "path"
-
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-func populateThumbnailHashes(db *Database) error {
- svc := db.GetThumbnailStore(rcontext.Initial())
- mediaSvc := db.GetMediaStore(rcontext.Initial())
-
- thumbs, err := svc.GetAllWithoutHash()
- if err != nil {
- logrus.Error("Failed to get thumbnails that don't have hashes: ", err)
- return err
- }
-
- for _, thumb := range thumbs {
- datastore, err := mediaSvc.GetDatastore(thumb.DatastoreId)
- if err != nil {
- logrus.Error("Error getting datastore for thumbnail ", thumb.Origin, " ", thumb.MediaId, ": ", err)
- sentry.CaptureException(err)
- continue
- }
- if datastore.Type != "file" {
- logrus.Error("Unrecognized datastore type for thumbnail ", thumb.Origin, " ", thumb.MediaId)
- sentry.CaptureException(err)
- continue
- }
- location := path.Join(datastore.Uri, thumb.Location)
-
- hash, err := util.GetFileHash(location)
- if err != nil {
- logrus.Error("Failed to generate hash for location '", location, "': ", err)
- return err
- }
-
- thumb.Sha256Hash = hash
- err = svc.UpdateHash(thumb)
- if err != nil {
- logrus.Error("Failed to update hash for '", location, "': ", err)
- return err
- }
-
- logrus.Info("Updated hash for thumbnail at '", location, "' as ", hash)
- }
-
- return nil
-}
-
-func populateDatastores(db *Database) error {
- logrus.Info("Starting to populate datastores...")
-
- thumbService := db.GetThumbnailStore(rcontext.Initial())
- mediaService := db.GetMediaStore(rcontext.Initial())
-
- logrus.Info("Fetching thumbnails...")
- thumbs, err := thumbService.GetAllWithoutDatastore()
- if err != nil {
- logrus.Error("Failed to get thumbnails that don't have a datastore: ", err)
- return err
- }
-
- for _, thumb := range thumbs {
- basePath := path.Clean(path.Join(thumb.Location, "..", "..", ".."))
- datastore, err := getOrCreateDatastoreWithMediaService(mediaService, basePath)
- if err != nil {
- logrus.Error("Error getting datastore for thumbnail path ", basePath, ": ", err)
- sentry.CaptureException(err)
- continue
- }
-
- thumb.DatastoreId = datastore.DatastoreId
- thumb.Location = util.GetLastSegmentsOfPath(thumb.Location, 3)
-
- err = thumbService.UpdateDatastoreAndLocation(thumb)
- if err != nil {
- logrus.Error("Failed to update datastore for thumbnail ", thumb.Origin, " ", thumb.MediaId, ": ", err)
- sentry.CaptureException(err)
- continue
- }
-
- logrus.Info("Updated datastore for thumbnail ", thumb.Origin, " ", thumb.MediaId)
- }
-
- logrus.Info("Fetching media...")
- mediaRecords, err := mediaService.GetAllWithoutDatastore()
- if err != nil {
- logrus.Error("Failed to get media that doesn't have a datastore: ", err)
- return err
- }
-
- for _, media := range mediaRecords {
- basePath := path.Clean(path.Join(media.Location, "..", "..", ".."))
- datastore, err := getOrCreateDatastoreWithMediaService(mediaService, basePath)
- if err != nil {
- logrus.Error("Error getting datastore for media path ", basePath, ": ", err)
- sentry.CaptureException(err)
- continue
- }
-
- media.DatastoreId = datastore.DatastoreId
- media.Location = util.GetLastSegmentsOfPath(media.Location, 3)
-
- err = mediaService.UpdateDatastoreAndLocation(media)
- if err != nil {
- logrus.Error("Failed to update datastore for media ", media.Origin, " ", media.MediaId, ": ", err)
- sentry.CaptureException(err)
- continue
- }
-
- logrus.Info("Updated datastore for media ", media.Origin, " ", media.MediaId)
- }
-
- return nil
-}
diff --git a/storage/storage.go b/storage/storage.go
deleted file mode 100644
index 6a84272375c309758662ab49457939beee1e1feb..0000000000000000000000000000000000000000
--- a/storage/storage.go
+++ /dev/null
@@ -1,127 +0,0 @@
-package storage
-
-import (
- "database/sql"
- "sync"
-
- "github.com/DavidHuie/gomigrate"
- _ "github.com/lib/pq" // postgres driver
- "github.com/sirupsen/logrus"
- "github.com/turt2live/matrix-media-repo/common/config"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/storage/stores"
-)
-
-type Database struct {
- db *sql.DB
- repos repos
-}
-
-type repos struct {
- mediaStore *stores.MediaStoreFactory
- thumbnailStore *stores.ThumbnailStoreFactory
- urlStore *stores.UrlStoreFactory
- metadataStore *stores.MetadataStoreFactory
- exportStore *stores.ExportStoreFactory
- mediaAttributesStore *stores.MediaAttributesStoreFactory
-}
-
-var dbInstance *Database
-var singletonDbLock = &sync.Once{}
-
-func GetDatabase() *Database {
- if dbInstance == nil {
- singletonDbLock.Do(func() {
- err := OpenDatabase(
- config.Get().Database.Postgres,
- config.Get().Database.Pool.MaxConnections,
- config.Get().Database.Pool.MaxIdle)
- if err != nil {
- panic(err)
- }
- })
- }
- return dbInstance
-}
-
-func OpenDatabase(connectionString string, maxConns int, maxIdleConns int) error {
- d := &Database{}
- var err error
-
- if d.db, err = sql.Open("postgres", connectionString); err != nil {
- return err
- }
- d.db.SetMaxOpenConns(maxConns)
- d.db.SetMaxIdleConns(maxIdleConns)
-
- // Make sure the database is how we want it
- migrator, err := gomigrate.NewMigratorWithLogger(d.db, gomigrate.Postgres{}, config.Runtime.MigrationsPath, logrus.StandardLogger())
- if err != nil {
- return err
- }
- err = migrator.Migrate()
- if err != nil {
- return err
- }
-
- // New the repo factories
- logrus.Info("Setting up media DB store...")
- if d.repos.mediaStore, err = stores.InitMediaStore(d.db); err != nil {
- return err
- }
- logrus.Info("Setting up thumbnails DB store...")
- if d.repos.thumbnailStore, err = stores.InitThumbnailStore(d.db); err != nil {
- return err
- }
- logrus.Info("Setting up URL previews DB store...")
- if d.repos.urlStore, err = stores.InitUrlStore(d.db); err != nil {
- return err
- }
- logrus.Info("Setting up metadata DB store...")
- if d.repos.metadataStore, err = stores.InitMetadataStore(d.db); err != nil {
- return err
- }
- logrus.Info("Setting up export DB store...")
- if d.repos.exportStore, err = stores.InitExportStore(d.db); err != nil {
- return err
- }
- logrus.Info("Setting up media attributes DB store...")
- if d.repos.mediaAttributesStore, err = stores.InitMediaAttributesStore(d.db); err != nil {
- return err
- }
-
- // Run some tasks that should always be done on startup
- if err = populateDatastores(d); err != nil {
- return err
- }
- if err = populateThumbnailHashes(d); err != nil {
- return err
- }
-
- dbInstance = d
- return nil
-}
-
-func (d *Database) GetMediaStore(ctx rcontext.RequestContext) *stores.MediaStore {
- return d.repos.mediaStore.Create(ctx)
-}
-
-func (d *Database) GetThumbnailStore(ctx rcontext.RequestContext) *stores.ThumbnailStore {
- return d.repos.thumbnailStore.New(ctx)
-}
-
-func (d *Database) GetUrlStore(ctx rcontext.RequestContext) *stores.UrlStore {
- return d.repos.urlStore.Create(ctx)
-}
-
-func (d *Database) GetMetadataStore(ctx rcontext.RequestContext) *stores.MetadataStore {
- return d.repos.metadataStore.Create(ctx)
-}
-
-func (d *Database) GetExportStore(ctx rcontext.RequestContext) *stores.ExportStore {
- return d.repos.exportStore.Create(ctx)
-}
-
-func (d *Database) GetMediaAttributesStore(ctx rcontext.RequestContext) *stores.MediaAttributesStore {
- return d.repos.mediaAttributesStore.Create(ctx)
-}
diff --git a/storage/stores/export_store.go b/storage/stores/export_store.go
deleted file mode 100644
index 9fc233ae954049de41ea55cf9acc04f6bcfc4907..0000000000000000000000000000000000000000
--- a/storage/stores/export_store.go
+++ /dev/null
@@ -1,148 +0,0 @@
-package stores
-
-import (
- "database/sql"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-const insertExportMetadata = "INSERT INTO exports (export_id, entity) VALUES ($1, $2);"
-const insertExportPart = "INSERT INTO export_parts (export_id, index, size_bytes, file_name, datastore_id, location) VALUES ($1, $2, $3, $4, $5, $6);"
-const selectExportMetadata = "SELECT export_id, entity FROM exports WHERE export_id = $1;"
-const selectExportParts = "SELECT export_id, index, size_bytes, file_name, datastore_id, location FROM export_parts WHERE export_id = $1;"
-const selectExportPart = "SELECT export_id, index, size_bytes, file_name, datastore_id, location FROM export_parts WHERE export_id = $1 AND index = $2;"
-const deleteExportParts = "DELETE FROM export_parts WHERE export_id = $1;"
-const deleteExport = "DELETE FROM exports WHERE export_id = $1;"
-
-type exportStoreStatements struct {
- insertExportMetadata *sql.Stmt
- insertExportPart *sql.Stmt
- selectExportMetadata *sql.Stmt
- selectExportParts *sql.Stmt
- selectExportPart *sql.Stmt
- deleteExportParts *sql.Stmt
- deleteExport *sql.Stmt
-}
-
-type ExportStoreFactory struct {
- sqlDb *sql.DB
- stmts *exportStoreStatements
-}
-
-type ExportStore struct {
- factory *ExportStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *exportStoreStatements // copied from factory
-}
-
-func InitExportStore(sqlDb *sql.DB) (*ExportStoreFactory, error) {
- store := ExportStoreFactory{stmts: &exportStoreStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.insertExportMetadata, err = store.sqlDb.Prepare(insertExportMetadata); err != nil {
- return nil, err
- }
- if store.stmts.insertExportPart, err = store.sqlDb.Prepare(insertExportPart); err != nil {
- return nil, err
- }
- if store.stmts.selectExportMetadata, err = store.sqlDb.Prepare(selectExportMetadata); err != nil {
- return nil, err
- }
- if store.stmts.selectExportParts, err = store.sqlDb.Prepare(selectExportParts); err != nil {
- return nil, err
- }
- if store.stmts.selectExportPart, err = store.sqlDb.Prepare(selectExportPart); err != nil {
- return nil, err
- }
- if store.stmts.deleteExportParts, err = store.sqlDb.Prepare(deleteExportParts); err != nil {
- return nil, err
- }
- if store.stmts.deleteExport, err = store.sqlDb.Prepare(deleteExport); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *ExportStoreFactory) Create(ctx rcontext.RequestContext) *ExportStore {
- return &ExportStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *ExportStore) InsertExport(exportId string, entity string) error {
- _, err := s.statements.insertExportMetadata.ExecContext(s.ctx, exportId, entity)
- return err
-}
-
-func (s *ExportStore) InsertExportPart(exportId string, index int, size int64, name string, datastoreId string, location string) error {
- _, err := s.statements.insertExportPart.ExecContext(s.ctx, exportId, index, size, name, datastoreId, location)
- return err
-}
-
-func (s *ExportStore) GetExportMetadata(exportId string) (*types.ExportMetadata, error) {
- m := &types.ExportMetadata{}
- err := s.statements.selectExportMetadata.QueryRowContext(s.ctx, exportId).Scan(
- &m.ExportID,
- &m.Entity,
- )
- return m, err
-}
-
-func (s *ExportStore) GetExportParts(exportId string) ([]*types.ExportPart, error) {
- rows, err := s.statements.selectExportParts.QueryContext(s.ctx, exportId)
- if err != nil {
- return nil, err
- }
-
- var results []*types.ExportPart
- for rows.Next() {
- obj := &types.ExportPart{}
- err = rows.Scan(
- &obj.ExportID,
- &obj.Index,
- &obj.SizeBytes,
- &obj.FileName,
- &obj.DatastoreID,
- &obj.Location,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *ExportStore) GetExportPart(exportId string, index int) (*types.ExportPart, error) {
- m := &types.ExportPart{}
- err := s.statements.selectExportPart.QueryRowContext(s.ctx, exportId, index).Scan(
- &m.ExportID,
- &m.Index,
- &m.SizeBytes,
- &m.FileName,
- &m.DatastoreID,
- &m.Location,
- )
- return m, err
-}
-
-func (s *ExportStore) DeleteExportAndParts(exportId string) error {
- _, err := s.statements.deleteExportParts.ExecContext(s.ctx, exportId)
- if err != nil {
- return err
- }
-
- _, err = s.statements.deleteExport.ExecContext(s.ctx, exportId)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/storage/stores/media_attributes_store.go b/storage/stores/media_attributes_store.go
deleted file mode 100644
index ef4827a0bf86bae8f50b84489da21d19ab95968b..0000000000000000000000000000000000000000
--- a/storage/stores/media_attributes_store.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package stores
-
-import (
- "database/sql"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-const selectMediaAttributes = "SELECT origin, media_id, purpose FROM media_attributes WHERE origin = $1 AND media_id = $2;"
-const upsertMediaPurpose = "INSERT INTO media_attributes (origin, media_id, purpose) VALUES ($1, $2, $3) ON CONFLICT (origin, media_id) DO UPDATE SET purpose = $3;"
-
-type mediaAttributesStoreStatements struct {
- selectMediaAttributes *sql.Stmt
- upsertMediaPurpose *sql.Stmt
-}
-
-type MediaAttributesStoreFactory struct {
- sqlDb *sql.DB
- stmts *mediaAttributesStoreStatements
-}
-
-type MediaAttributesStore struct {
- factory *MediaAttributesStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *mediaAttributesStoreStatements // copied from factory
-}
-
-func InitMediaAttributesStore(sqlDb *sql.DB) (*MediaAttributesStoreFactory, error) {
- store := MediaAttributesStoreFactory{stmts: &mediaAttributesStoreStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.selectMediaAttributes, err = store.sqlDb.Prepare(selectMediaAttributes); err != nil {
- return nil, err
- }
- if store.stmts.upsertMediaPurpose, err = store.sqlDb.Prepare(upsertMediaPurpose); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *MediaAttributesStoreFactory) Create(ctx rcontext.RequestContext) *MediaAttributesStore {
- return &MediaAttributesStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *MediaAttributesStore) GetAttributes(origin string, mediaId string) (*types.MediaAttributes, error) {
- r := s.statements.selectMediaAttributes.QueryRowContext(s.ctx, origin, mediaId)
- obj := &types.MediaAttributes{}
- err := r.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.Purpose,
- )
- return obj, err
-}
-
-func (s *MediaAttributesStore) GetAttributesDefaulted(origin string, mediaId string) (*types.MediaAttributes, error) {
- attr, err := s.GetAttributes(origin, mediaId)
- if err == sql.ErrNoRows {
- return &types.MediaAttributes{
- Origin: origin,
- MediaId: mediaId,
- Purpose: types.PurposeNone,
- }, nil
- }
- return attr, err
-}
-
-func (s *MediaAttributesStore) UpsertPurpose(origin string, mediaId string, purpose string) error {
- _, err := s.statements.upsertMediaPurpose.ExecContext(s.ctx, origin, mediaId, purpose)
- return err
-}
diff --git a/storage/stores/media_store.go b/storage/stores/media_store.go
deleted file mode 100644
index 958f1b0a79447667b2ab54d1f515b770d23dc9a3..0000000000000000000000000000000000000000
--- a/storage/stores/media_store.go
+++ /dev/null
@@ -1,860 +0,0 @@
-package stores
-
-import (
- "database/sql"
- "errors"
- "fmt"
- "github.com/turt2live/matrix-media-repo/util"
- "strings"
- "sync"
-
- "github.com/lib/pq"
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-const selectMedia = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 and media_id = $2;"
-const selectMediaByHash = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE sha256_hash = $1;"
-const insertMedia = "INSERT INTO media (origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);"
-const selectOldMedia = "SELECT m.origin, m.media_id, m.upload_name, m.content_type, m.user_id, m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, quarantined FROM media AS m WHERE m.origin <> ANY($1) AND m.creation_ts < $2 AND (SELECT COUNT(*) FROM media AS d WHERE d.sha256_hash = m.sha256_hash AND d.creation_ts >= $2) = 0 AND (SELECT COUNT(*) FROM media AS d WHERE d.sha256_hash = m.sha256_hash AND d.origin = ANY($1)) = 0;"
-const selectOrigins = "SELECT DISTINCT origin FROM media;"
-const deleteMedia = "DELETE FROM media WHERE origin = $1 AND media_id = $2;"
-const updateQuarantined = "UPDATE media SET quarantined = $3 WHERE origin = $1 AND media_id = $2;"
-const selectDatastore = "SELECT datastore_id, ds_type, uri FROM datastores WHERE datastore_id = $1;"
-const selectDatastoreByUri = "SELECT datastore_id, ds_type, uri FROM datastores WHERE uri = $1;"
-const insertDatastore = "INSERT INTO datastores (datastore_id, ds_type, uri) VALUES ($1, $2, $3);"
-const selectMediaWithoutDatastore = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE datastore_id IS NULL OR datastore_id = '';"
-const updateMediaDatastoreAndLocation = "UPDATE media SET location = $4, datastore_id = $3 WHERE origin = $1 AND media_id = $2;"
-const selectAllDatastores = "SELECT datastore_id, ds_type, uri FROM datastores;"
-const selectAllMediaForServer = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1"
-const selectAllMediaForServerUsers = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND user_id = ANY($2)"
-const selectAllMediaForServerIds = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND media_id = ANY($2)"
-const selectQuarantinedMedia = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE quarantined = true;"
-const selectServerQuarantinedMedia = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE quarantined = true AND origin = $1;"
-const selectMediaByUser = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE user_id = $1"
-const selectMediaByUserBefore = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE user_id = $1 AND creation_ts <= $2"
-const selectMediaByDomainBefore = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND creation_ts <= $2"
-const selectMediaByLocation = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE datastore_id = $1 AND location = $2"
-const selectIfQuarantined = "SELECT 1 FROM media WHERE sha256_hash = $1 AND quarantined = $2 LIMIT 1;"
-const selectMediaLocationsForDatastore = "SELECT distinct location FROM media WHERE datastore_id = $1;"
-
-var UsersUsageStatsSorts = []string{"media_count", "media_length", "user_id"}
-
-var dsCacheByPath = sync.Map{} // [string] => Datastore
-var dsCacheById = sync.Map{} // [string] => Datastore
-
-type mediaStoreStatements struct {
- selectMedia *sql.Stmt
- selectMediaByHash *sql.Stmt
- insertMedia *sql.Stmt
- selectOldMedia *sql.Stmt
- selectOrigins *sql.Stmt
- deleteMedia *sql.Stmt
- updateQuarantined *sql.Stmt
- selectDatastore *sql.Stmt
- selectDatastoreByUri *sql.Stmt
- insertDatastore *sql.Stmt
- selectMediaWithoutDatastore *sql.Stmt
- updateMediaDatastoreAndLocation *sql.Stmt
- selectAllDatastores *sql.Stmt
- selectMediaInDatastoreOlderThan *sql.Stmt
- selectAllMediaForServer *sql.Stmt
- selectAllMediaForServerUsers *sql.Stmt
- selectAllMediaForServerIds *sql.Stmt
- selectQuarantinedMedia *sql.Stmt
- selectServerQuarantinedMedia *sql.Stmt
- selectMediaByUser *sql.Stmt
- selectMediaByUserBefore *sql.Stmt
- selectMediaByDomainBefore *sql.Stmt
- selectMediaByLocation *sql.Stmt
- selectIfQuarantined *sql.Stmt
- selectMediaLocationsForDatastore *sql.Stmt
-}
-
-type MediaStoreFactory struct {
- sqlDb *sql.DB
- stmts *mediaStoreStatements
-}
-
-type MediaStore struct {
- factory *MediaStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *mediaStoreStatements // copied from factory
-}
-
-func InitMediaStore(sqlDb *sql.DB) (*MediaStoreFactory, error) {
- store := MediaStoreFactory{stmts: &mediaStoreStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.selectMedia, err = store.sqlDb.Prepare(selectMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaByHash, err = store.sqlDb.Prepare(selectMediaByHash); err != nil {
- return nil, err
- }
- if store.stmts.insertMedia, err = store.sqlDb.Prepare(insertMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectOldMedia, err = store.sqlDb.Prepare(selectOldMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectOrigins, err = store.sqlDb.Prepare(selectOrigins); err != nil {
- return nil, err
- }
- if store.stmts.deleteMedia, err = store.sqlDb.Prepare(deleteMedia); err != nil {
- return nil, err
- }
- if store.stmts.updateQuarantined, err = store.sqlDb.Prepare(updateQuarantined); err != nil {
- return nil, err
- }
- if store.stmts.selectDatastore, err = store.sqlDb.Prepare(selectDatastore); err != nil {
- return nil, err
- }
- if store.stmts.selectDatastoreByUri, err = store.sqlDb.Prepare(selectDatastoreByUri); err != nil {
- return nil, err
- }
- if store.stmts.insertDatastore, err = store.sqlDb.Prepare(insertDatastore); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaWithoutDatastore, err = store.sqlDb.Prepare(selectMediaWithoutDatastore); err != nil {
- return nil, err
- }
- if store.stmts.updateMediaDatastoreAndLocation, err = store.sqlDb.Prepare(updateMediaDatastoreAndLocation); err != nil {
- return nil, err
- }
- if store.stmts.selectAllDatastores, err = store.sqlDb.Prepare(selectAllDatastores); err != nil {
- return nil, err
- }
- if store.stmts.selectAllMediaForServer, err = store.sqlDb.Prepare(selectAllMediaForServer); err != nil {
- return nil, err
- }
- if store.stmts.selectAllMediaForServerUsers, err = store.sqlDb.Prepare(selectAllMediaForServerUsers); err != nil {
- return nil, err
- }
- if store.stmts.selectAllMediaForServerIds, err = store.sqlDb.Prepare(selectAllMediaForServerIds); err != nil {
- return nil, err
- }
- if store.stmts.selectQuarantinedMedia, err = store.sqlDb.Prepare(selectQuarantinedMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectServerQuarantinedMedia, err = store.sqlDb.Prepare(selectServerQuarantinedMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaByUser, err = store.sqlDb.Prepare(selectMediaByUser); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaByUserBefore, err = store.sqlDb.Prepare(selectMediaByUserBefore); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaByDomainBefore, err = store.sqlDb.Prepare(selectMediaByDomainBefore); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaByLocation, err = store.sqlDb.Prepare(selectMediaByLocation); err != nil {
- return nil, err
- }
- if store.stmts.selectIfQuarantined, err = store.sqlDb.Prepare(selectIfQuarantined); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaLocationsForDatastore, err = store.sqlDb.Prepare(selectMediaLocationsForDatastore); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *MediaStoreFactory) Create(ctx rcontext.RequestContext) *MediaStore {
- return &MediaStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *MediaStore) Insert(media *types.Media) error {
- _, err := s.statements.insertMedia.ExecContext(
- s.ctx,
- media.Origin,
- media.MediaId,
- media.UploadName,
- media.ContentType,
- media.UserId,
- media.Sha256Hash,
- media.SizeBytes,
- media.DatastoreId,
- media.Location,
- media.CreationTs,
- media.Quarantined,
- )
- return err
-}
-
-func (s *MediaStore) GetByHash(hash string) ([]*types.Media, error) {
- rows, err := s.statements.selectMediaByHash.QueryContext(s.ctx, hash)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) Get(origin string, mediaId string) (*types.Media, error) {
- m := &types.Media{}
- err := s.statements.selectMedia.QueryRowContext(s.ctx, origin, mediaId).Scan(
- &m.Origin,
- &m.MediaId,
- &m.UploadName,
- &m.ContentType,
- &m.UserId,
- &m.Sha256Hash,
- &m.SizeBytes,
- &m.DatastoreId,
- &m.Location,
- &m.CreationTs,
- &m.Quarantined,
- )
- return m, err
-}
-
-func (s *MediaStore) GetOldMedia(exceptOrigins []string, beforeTs int64) ([]*types.Media, error) {
- rows, err := s.statements.selectOldMedia.QueryContext(s.ctx, pq.Array(exceptOrigins), beforeTs)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetOrigins() ([]string, error) {
- rows, err := s.statements.selectOrigins.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []string
- for rows.Next() {
- obj := ""
- err = rows.Scan(&obj)
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) Delete(origin string, mediaId string) error {
- _, err := s.statements.deleteMedia.ExecContext(s.ctx, origin, mediaId)
- return err
-}
-
-func (s *MediaStore) SetQuarantined(origin string, mediaId string, isQuarantined bool) error {
- _, err := s.statements.updateQuarantined.ExecContext(s.ctx, origin, mediaId, isQuarantined)
- return err
-}
-
-func (s *MediaStore) UpdateDatastoreAndLocation(media *types.Media) error {
- _, err := s.statements.updateMediaDatastoreAndLocation.ExecContext(
- s.ctx,
- media.Origin,
- media.MediaId,
- media.DatastoreId,
- media.Location,
- )
-
- return err
-}
-
-func (s *MediaStore) GetDatastore(id string) (*types.Datastore, error) {
- if v, ok := dsCacheById.Load(id); ok {
- ds := v.(*types.Datastore)
- return &types.Datastore{
- DatastoreId: ds.DatastoreId,
- Type: ds.Type,
- Uri: ds.Uri,
- }, nil
- }
-
- d := &types.Datastore{}
- err := s.statements.selectDatastore.QueryRowContext(s.ctx, id).Scan(
- &d.DatastoreId,
- &d.Type,
- &d.Uri,
- )
- if err != nil {
- dsCacheById.Store(d.Uri, d)
- dsCacheByPath.Store(d.Uri, d)
- }
-
- return &types.Datastore{
- DatastoreId: d.DatastoreId,
- Type: d.Type,
- Uri: d.Uri,
- }, err
-}
-
-func (s *MediaStore) InsertDatastore(datastore *types.Datastore) error {
- _, err := s.statements.insertDatastore.ExecContext(
- s.ctx,
- datastore.DatastoreId,
- datastore.Type,
- datastore.Uri,
- )
- if err != nil {
- d := &types.Datastore{
- DatastoreId: datastore.DatastoreId,
- Type: datastore.Type,
- Uri: datastore.Uri,
- }
- dsCacheById.Store(d.Uri, d)
- dsCacheByPath.Store(d.Uri, d)
- }
- return err
-}
-
-func (s *MediaStore) GetDatastoreByUri(uri string) (*types.Datastore, error) {
- if v, ok := dsCacheByPath.Load(uri); ok {
- ds := v.(*types.Datastore)
- return &types.Datastore{
- DatastoreId: ds.DatastoreId,
- Type: ds.Type,
- Uri: ds.Uri,
- }, nil
- }
-
- d := &types.Datastore{}
- err := s.statements.selectDatastoreByUri.QueryRowContext(s.ctx, uri).Scan(
- &d.DatastoreId,
- &d.Type,
- &d.Uri,
- )
- if err != nil {
- dsCacheById.Store(d.Uri, d)
- dsCacheByPath.Store(d.Uri, d)
- }
-
- return &types.Datastore{
- DatastoreId: d.DatastoreId,
- Type: d.Type,
- Uri: d.Uri,
- }, err
-}
-
-func (s *MediaStore) GetAllWithoutDatastore() ([]*types.Media, error) {
- rows, err := s.statements.selectMediaWithoutDatastore.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetAllDatastores() ([]*types.Datastore, error) {
- rows, err := s.statements.selectAllDatastores.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Datastore
- for rows.Next() {
- obj := &types.Datastore{}
- err = rows.Scan(
- &obj.DatastoreId,
- &obj.Type,
- &obj.Uri,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetAllMediaForServer(serverName string) ([]*types.Media, error) {
- rows, err := s.statements.selectAllMediaForServer.QueryContext(s.ctx, serverName)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetUsersUsageStatsForServer(
- serverName string,
- orderBy string,
- start int64,
- limit int64,
- fromTS int64,
- untilTS int64,
- searchTerm string,
- isAscendingOrder bool,
-) ([]*types.UserUsageStats, int64, error) {
- if !util.ArrayContains(UsersUsageStatsSorts, orderBy) {
- return nil, 0, errors.New("invalid orderBy")
- }
- if start < 0 {
- return nil, 0, errors.New("invalid start")
- }
- if limit < 0 {
- return nil, 0, errors.New("invalid limit")
- }
-
- orderDirection := "DESC"
- if isAscendingOrder {
- orderDirection = "ASC"
- }
-
- var queryParamIdx int64 = 1
- var commonQueryParams = []interface{}{serverName}
-
- var filters = []string{fmt.Sprintf("origin = $%d", queryParamIdx)}
- queryParamIdx++
- if fromTS >= 0 {
- filters = append(filters, fmt.Sprintf("creation_ts >= $%d", queryParamIdx))
- queryParamIdx++
- commonQueryParams = append(commonQueryParams, fromTS)
- }
- if untilTS >= 0 {
- filters = append(filters, fmt.Sprintf("creation_ts <= $%d", queryParamIdx))
- queryParamIdx++
- commonQueryParams = append(commonQueryParams, untilTS)
- }
- if searchTerm != "" {
- filters = append(filters, fmt.Sprintf("user_id LIKE $%d", queryParamIdx))
- queryParamIdx++
- commonQueryParams = append(commonQueryParams, fmt.Sprintf("@%%%s%%:%%", searchTerm))
- }
-
- var otherPaginationParams []interface{}
- limitClause := fmt.Sprintf("LIMIT $%d", queryParamIdx)
- queryParamIdx++
- otherPaginationParams = append(otherPaginationParams, limit)
-
- offsetClause := fmt.Sprintf("OFFSET $%d", queryParamIdx)
- queryParamIdx++
- otherPaginationParams = append(otherPaginationParams, start)
-
- commonQueryPortion := fmt.Sprintf(
- "FROM media "+
- "WHERE %s "+
- "GROUP BY user_id", strings.Join(filters, " AND "))
-
- paginationQuery := fmt.Sprintf(
- "SELECT COUNT(user_id) AS media_count, SUM(size_bytes) AS media_length, user_id "+
- "%s "+
- "ORDER BY %s %s "+
- "%s "+
- "%s;",
- commonQueryPortion,
- orderBy,
- orderDirection,
- limitClause,
- offsetClause)
- rows, err := s.factory.sqlDb.QueryContext(
- s.ctx,
- paginationQuery,
- append(commonQueryParams, otherPaginationParams...)...)
- if err != nil {
- return nil, 0, err
- }
-
- var results []*types.UserUsageStats
- for rows.Next() {
- obj := &types.UserUsageStats{}
- err = rows.Scan(
- &obj.MediaCount,
- &obj.MediaLength,
- &obj.UserId,
- )
- if err != nil {
- return nil, 0, err
- }
- results = append(results, obj)
- }
-
- totalQuery := fmt.Sprintf(
- "SELECT COUNT(*) "+
- "FROM ("+
- " SELECT user_id "+
- " %s "+
- ") as count_user_ids; ",
- commonQueryPortion)
- var totalNumRows int64 = 0
- err = s.factory.sqlDb.QueryRowContext(s.ctx, totalQuery, commonQueryParams...).Scan(&totalNumRows)
- if err != nil {
- return nil, 0, err
- }
-
- return results, totalNumRows, nil
-}
-
-func (s *MediaStore) GetAllMediaForServerUsers(serverName string, userIds []string) ([]*types.Media, error) {
- rows, err := s.statements.selectAllMediaForServerUsers.QueryContext(s.ctx, serverName, pq.Array(userIds))
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetAllMediaInIds(serverName string, mediaIds []string) ([]*types.Media, error) {
- rows, err := s.statements.selectAllMediaForServerIds.QueryContext(s.ctx, serverName, pq.Array(mediaIds))
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetAllQuarantinedMedia() ([]*types.Media, error) {
- rows, err := s.statements.selectQuarantinedMedia.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetQuarantinedMediaFor(serverName string) ([]*types.Media, error) {
- rows, err := s.statements.selectServerQuarantinedMedia.QueryContext(s.ctx, serverName)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetMediaByUser(userId string) ([]*types.Media, error) {
- rows, err := s.statements.selectMediaByUser.QueryContext(s.ctx, userId)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetMediaByUserBefore(userId string, beforeTs int64) ([]*types.Media, error) {
- rows, err := s.statements.selectMediaByUserBefore.QueryContext(s.ctx, userId, beforeTs)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetMediaByDomainBefore(serverName string, beforeTs int64) ([]*types.Media, error) {
- rows, err := s.statements.selectMediaByDomainBefore.QueryContext(s.ctx, serverName, beforeTs)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) GetMediaByLocation(datastoreId string, location string) ([]*types.Media, error) {
- rows, err := s.statements.selectMediaByLocation.QueryContext(s.ctx, datastoreId, location)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Media
- for rows.Next() {
- obj := &types.Media{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.UploadName,
- &obj.ContentType,
- &obj.UserId,
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Quarantined,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MediaStore) IsQuarantined(sha256hash string) (bool, error) {
- r := s.statements.selectIfQuarantined.QueryRow(sha256hash, true)
- var i int
- err := r.Scan(&i)
- if err == sql.ErrNoRows {
- return false, nil
- } else if err != nil {
- return false, err
- }
- return true, nil
-}
-
-func (s *MediaStore) GetDistinctLocationsForDatastore(datastoreId string) ([]string, error) {
- rows, err := s.statements.selectMediaLocationsForDatastore.QueryContext(s.ctx, datastoreId)
- if err != nil {
- return nil, err
- }
-
- locations := make([]string, 0)
- for rows.Next() {
- s := ""
- err = rows.Scan(&s)
- if err != nil {
- return nil, err
- }
- locations = append(locations, s)
- }
-
- return locations, nil
-}
diff --git a/storage/stores/metadata_store.go b/storage/stores/metadata_store.go
deleted file mode 100644
index 82a4c22f11b2be3d5efb7d8a3e5a890f45584e5e..0000000000000000000000000000000000000000
--- a/storage/stores/metadata_store.go
+++ /dev/null
@@ -1,429 +0,0 @@
-package stores
-
-import (
- "database/sql"
- "encoding/json"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-type folderSize struct {
- Size int64
-}
-
-const selectSizeOfDatastore = "SELECT COALESCE(SUM(size_bytes), 0) + COALESCE((SELECT SUM(size_bytes) FROM thumbnails WHERE datastore_id = $1), 0) AS size_total FROM media WHERE datastore_id = $1;"
-const upsertLastAccessed = "INSERT INTO last_access (sha256_hash, last_access_ts) VALUES ($1, $2) ON CONFLICT (sha256_hash) DO UPDATE SET last_access_ts = $2"
-const selectMediaLastAccessedBeforeInDatastore = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM media AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1 AND m.datastore_id = $2"
-const selectThumbnailsLastAccessedBeforeInDatastore = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM thumbnails AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1 AND m.datastore_id = $2"
-const changeDatastoreOfMediaHash = "UPDATE media SET datastore_id = $1, location = $2 WHERE sha256_hash = $3"
-const changeDatastoreOfThumbnailHash = "UPDATE thumbnails SET datastore_id = $1, location = $2 WHERE sha256_hash = $3"
-const selectUploadCountsForServer = "SELECT COALESCE((SELECT COUNT(origin) FROM media WHERE origin = $1), 0) AS media, COALESCE((SELECT COUNT(origin) FROM thumbnails WHERE origin = $1), 0) AS thumbnails"
-const selectUploadSizesForServer = "SELECT COALESCE((SELECT SUM(size_bytes) FROM media WHERE origin = $1), 0) AS media, COALESCE((SELECT SUM(size_bytes) FROM thumbnails WHERE origin = $1), 0) AS thumbnails"
-const selectUsersForServer = "SELECT DISTINCT user_id FROM media WHERE origin = $1 AND user_id IS NOT NULL AND LENGTH(user_id) > 0"
-const insertNewBackgroundTask = "INSERT INTO background_tasks (task, params, start_ts) VALUES ($1, $2, $3) RETURNING id;"
-const selectBackgroundTask = "SELECT id, task, params, start_ts, end_ts FROM background_tasks WHERE id = $1"
-const updateBackgroundTask = "UPDATE background_tasks SET end_ts = $2 WHERE id = $1"
-const selectAllBackgroundTasks = "SELECT id, task, params, start_ts, end_ts FROM background_tasks"
-const insertReservation = "INSERT INTO reserved_media (origin, media_id, reason) VALUES ($1, $2, $3);"
-const selectReservation = "SELECT origin, media_id, reason FROM reserved_media WHERE origin = $1 AND media_id = $2;"
-const selectMediaLastAccessed = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM media AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1;"
-const insertBlurhash = "INSERT INTO blurhashes (sha256_hash, blurhash) VALUES ($1, $2);"
-const selectBlurhash = "SELECT blurhash FROM blurhashes WHERE sha256_hash = $1;"
-const selectUserStats = "SELECT user_id, uploaded_bytes FROM user_stats WHERE user_id = $1;"
-
-type metadataStoreStatements struct {
- upsertLastAccessed *sql.Stmt
- selectSizeOfDatastore *sql.Stmt
- selectMediaLastAccessedBeforeInDatastore *sql.Stmt
- selectThumbnailsLastAccessedBeforeInDatastore *sql.Stmt
- changeDatastoreOfMediaHash *sql.Stmt
- changeDatastoreOfThumbnailHash *sql.Stmt
- selectUploadCountsForServer *sql.Stmt
- selectUploadSizesForServer *sql.Stmt
- selectUsersForServer *sql.Stmt
- insertNewBackgroundTask *sql.Stmt
- selectBackgroundTask *sql.Stmt
- updateBackgroundTask *sql.Stmt
- selectAllBackgroundTasks *sql.Stmt
- insertReservation *sql.Stmt
- selectReservation *sql.Stmt
- selectMediaLastAccessed *sql.Stmt
- insertBlurhash *sql.Stmt
- selectBlurhash *sql.Stmt
- selectUserStats *sql.Stmt
-}
-
-type MetadataStoreFactory struct {
- sqlDb *sql.DB
- stmts *metadataStoreStatements
-}
-
-type MetadataStore struct {
- factory *MetadataStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *metadataStoreStatements // copied from factory
-}
-
-func InitMetadataStore(sqlDb *sql.DB) (*MetadataStoreFactory, error) {
- store := MetadataStoreFactory{stmts: &metadataStoreStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.upsertLastAccessed, err = store.sqlDb.Prepare(upsertLastAccessed); err != nil {
- return nil, err
- }
- if store.stmts.selectSizeOfDatastore, err = store.sqlDb.Prepare(selectSizeOfDatastore); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaLastAccessedBeforeInDatastore, err = store.sqlDb.Prepare(selectMediaLastAccessedBeforeInDatastore); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailsLastAccessedBeforeInDatastore, err = store.sqlDb.Prepare(selectThumbnailsLastAccessedBeforeInDatastore); err != nil {
- return nil, err
- }
- if store.stmts.changeDatastoreOfMediaHash, err = store.sqlDb.Prepare(changeDatastoreOfMediaHash); err != nil {
- return nil, err
- }
- if store.stmts.changeDatastoreOfThumbnailHash, err = store.sqlDb.Prepare(changeDatastoreOfThumbnailHash); err != nil {
- return nil, err
- }
- if store.stmts.selectUsersForServer, err = store.sqlDb.Prepare(selectUsersForServer); err != nil {
- return nil, err
- }
- if store.stmts.selectUploadSizesForServer, err = store.sqlDb.Prepare(selectUploadSizesForServer); err != nil {
- return nil, err
- }
- if store.stmts.selectUploadCountsForServer, err = store.sqlDb.Prepare(selectUploadCountsForServer); err != nil {
- return nil, err
- }
- if store.stmts.insertNewBackgroundTask, err = store.sqlDb.Prepare(insertNewBackgroundTask); err != nil {
- return nil, err
- }
- if store.stmts.selectBackgroundTask, err = store.sqlDb.Prepare(selectBackgroundTask); err != nil {
- return nil, err
- }
- if store.stmts.updateBackgroundTask, err = store.sqlDb.Prepare(updateBackgroundTask); err != nil {
- return nil, err
- }
- if store.stmts.selectAllBackgroundTasks, err = store.sqlDb.Prepare(selectAllBackgroundTasks); err != nil {
- return nil, err
- }
- if store.stmts.insertReservation, err = store.sqlDb.Prepare(insertReservation); err != nil {
- return nil, err
- }
- if store.stmts.selectReservation, err = store.sqlDb.Prepare(selectReservation); err != nil {
- return nil, err
- }
- if store.stmts.selectMediaLastAccessed, err = store.sqlDb.Prepare(selectMediaLastAccessed); err != nil {
- return nil, err
- }
- if store.stmts.insertBlurhash, err = store.sqlDb.Prepare(insertBlurhash); err != nil {
- return nil, err
- }
- if store.stmts.selectBlurhash, err = store.sqlDb.Prepare(selectBlurhash); err != nil {
- return nil, err
- }
- if store.stmts.selectUserStats, err = store.sqlDb.Prepare(selectUserStats); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *MetadataStoreFactory) Create(ctx rcontext.RequestContext) *MetadataStore {
- return &MetadataStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *MetadataStore) UpsertLastAccess(sha256Hash string, timestamp int64) error {
- _, err := s.statements.upsertLastAccessed.ExecContext(s.ctx, sha256Hash, timestamp)
- return err
-}
-
-func (s *MetadataStore) ChangeDatastoreOfHash(datastoreId string, location string, sha256hash string) error {
- _, err1 := s.statements.changeDatastoreOfMediaHash.ExecContext(s.ctx, datastoreId, location, sha256hash)
- if err1 != nil {
- return err1
- }
- _, err2 := s.statements.changeDatastoreOfThumbnailHash.ExecContext(s.ctx, datastoreId, location, sha256hash)
- if err2 != nil {
- return err2
- }
- return nil
-}
-
-func (s *MetadataStore) GetEstimatedSizeOfDatastore(datastoreId string) (int64, error) {
- r := &folderSize{}
- err := s.statements.selectSizeOfDatastore.QueryRowContext(s.ctx, datastoreId).Scan(&r.Size)
- return r.Size, err
-}
-
-func (s *MetadataStore) GetOldMedia(beforeTs int64) ([]*types.MinimalMediaMetadata, error) {
- rows, err := s.statements.selectMediaLastAccessed.QueryContext(s.ctx, beforeTs)
- if err != nil {
- return nil, err
- }
-
- var results []*types.MinimalMediaMetadata
- for rows.Next() {
- obj := &types.MinimalMediaMetadata{}
- err = rows.Scan(
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.LastAccessTs,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MetadataStore) GetOldMediaInDatastore(datastoreId string, beforeTs int64) ([]*types.MinimalMediaMetadata, error) {
- rows, err := s.statements.selectMediaLastAccessedBeforeInDatastore.QueryContext(s.ctx, beforeTs, datastoreId)
- if err != nil {
- return nil, err
- }
-
- var results []*types.MinimalMediaMetadata
- for rows.Next() {
- obj := &types.MinimalMediaMetadata{}
- err = rows.Scan(
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.LastAccessTs,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MetadataStore) GetOldThumbnailsInDatastore(datastoreId string, beforeTs int64) ([]*types.MinimalMediaMetadata, error) {
- rows, err := s.statements.selectThumbnailsLastAccessedBeforeInDatastore.QueryContext(s.ctx, beforeTs, datastoreId)
- if err != nil {
- return nil, err
- }
-
- var results []*types.MinimalMediaMetadata
- for rows.Next() {
- obj := &types.MinimalMediaMetadata{}
- err = rows.Scan(
- &obj.Sha256Hash,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.LastAccessTs,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *MetadataStore) GetUsersForServer(serverName string) ([]string, error) {
- rows, err := s.statements.selectUsersForServer.QueryContext(s.ctx, serverName)
- if err != nil {
- return nil, err
- }
-
- results := make([]string, 0)
- for rows.Next() {
- v := ""
- err = rows.Scan(&v)
- if err != nil {
- return nil, err
- }
- results = append(results, v)
- }
-
- return results, nil
-}
-
-func (s *MetadataStore) GetByteUsageForServer(serverName string) (int64, int64, error) {
- row := s.statements.selectUploadSizesForServer.QueryRowContext(s.ctx, serverName)
-
- media := int64(0)
- thumbs := int64(0)
- err := row.Scan(&media, &thumbs)
- if err != nil {
- return 0, 0, err
- }
-
- return media, thumbs, nil
-}
-
-func (s *MetadataStore) GetCountUsageForServer(serverName string) (int64, int64, error) {
- row := s.statements.selectUploadCountsForServer.QueryRowContext(s.ctx, serverName)
-
- media := int64(0)
- thumbs := int64(0)
- err := row.Scan(&media, &thumbs)
- if err != nil {
- return 0, 0, err
- }
-
- return media, thumbs, nil
-}
-
-func (s *MetadataStore) CreateBackgroundTask(name string, params map[string]interface{}) (*types.BackgroundTask, error) {
- now := util.NowMillis()
- b, err := json.Marshal(params)
- if err != nil {
- return nil, err
- }
- r := s.statements.insertNewBackgroundTask.QueryRowContext(s.ctx, name, string(b), now)
- var id int
- err = r.Scan(&id)
- if err != nil {
- return nil, err
- }
- return &types.BackgroundTask{
- ID: id,
- Name: name,
- StartTs: now,
- EndTs: 0,
- }, nil
-}
-
-func (s *MetadataStore) FinishedBackgroundTask(id int) error {
- now := util.NowMillis()
- _, err := s.statements.updateBackgroundTask.ExecContext(s.ctx, id, now)
- return err
-}
-
-func (s *MetadataStore) GetBackgroundTask(id int) (*types.BackgroundTask, error) {
- r := s.statements.selectBackgroundTask.QueryRowContext(s.ctx, id)
- task := &types.BackgroundTask{}
- var paramsStr string
- var endTs sql.NullInt64
-
- err := r.Scan(&task.ID, &task.Name, ¶msStr, &task.StartTs, &endTs)
- if err != nil {
- return nil, err
- }
-
- err = json.Unmarshal([]byte(paramsStr), &task.Params)
- if err != nil {
- return nil, err
- }
-
- if endTs.Valid {
- task.EndTs = endTs.Int64
- }
-
- return task, nil
-}
-
-func (s *MetadataStore) GetAllBackgroundTasks() ([]*types.BackgroundTask, error) {
- rows, err := s.statements.selectAllBackgroundTasks.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- results := make([]*types.BackgroundTask, 0)
- for rows.Next() {
- task := &types.BackgroundTask{}
- var paramsStr string
- var endTs sql.NullInt64
-
- err := rows.Scan(&task.ID, &task.Name, ¶msStr, &task.StartTs, &endTs)
- if err != nil {
- return nil, err
- }
-
- err = json.Unmarshal([]byte(paramsStr), &task.Params)
- if err != nil {
- return nil, err
- }
-
- if endTs.Valid {
- task.EndTs = endTs.Int64
- }
-
- results = append(results, task)
- }
-
- return results, nil
-}
-
-func (s *MetadataStore) ReserveMediaId(origin string, mediaId string, reason string) error {
- _, err := s.statements.insertReservation.ExecContext(s.ctx, origin, mediaId, reason)
- if err != nil {
- return err
- }
- return nil
-}
-
-func (s *MetadataStore) IsReserved(origin string, mediaId string) (bool, error) {
- r := s.statements.selectReservation.QueryRowContext(s.ctx, origin, mediaId)
- var dbOrigin string
- var dbMediaId string
- var dbReason string
-
- err := r.Scan(&dbOrigin, &dbMediaId, &dbReason)
- if err == sql.ErrNoRows {
- return false, nil
- }
- if err != nil {
- return true, err
- }
- return true, nil
-}
-
-func (s *MetadataStore) InsertBlurhash(sha256Hash string, blurhash string) error {
- _, err := s.statements.insertBlurhash.ExecContext(s.ctx, sha256Hash, blurhash)
- if err != nil {
- return err
- }
- return nil
-}
-
-func (s *MetadataStore) GetBlurhash(sha256Hash string) (string, error) {
- r := s.statements.selectBlurhash.QueryRowContext(s.ctx, sha256Hash)
- var blurhash string
-
- err := r.Scan(&blurhash)
- if err == sql.ErrNoRows {
- return "", nil
- }
- if err != nil {
- return "", err
- }
- return blurhash, nil
-}
-
-func (s *MetadataStore) GetUserStats(userId string) (*types.UserStats, error) {
- r := s.statements.selectUserStats.QueryRowContext(s.ctx, userId)
-
- stat := &types.UserStats{}
- err := r.Scan(
- &stat.UserId,
- &stat.UploadedBytes,
- )
- if err != nil {
- return nil, err
- }
- return stat, nil
-}
diff --git a/storage/stores/thumbnail_store.go b/storage/stores/thumbnail_store.go
deleted file mode 100644
index 65f489b15b2f31da51f1b8a1bd59449bebc525c4..0000000000000000000000000000000000000000
--- a/storage/stores/thumbnail_store.go
+++ /dev/null
@@ -1,329 +0,0 @@
-package stores
-
-import (
- "database/sql"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
-)
-
-const selectThumbnail = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE origin = $1 and media_id = $2 and width = $3 and height = $4 and method = $5 and animated = $6;"
-const insertThumbnail = "INSERT INTO thumbnails (origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12);"
-const updateThumbnailHash = "UPDATE thumbnails SET sha256_hash = $7 WHERE origin = $1 and media_id = $2 and width = $3 and height = $4 and method = $5 and animated = $6;"
-const selectThumbnailsWithoutHash = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE sha256_hash IS NULL OR sha256_hash = '';"
-const selectThumbnailsWithoutDatastore = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE datastore_id IS NULL OR datastore_id = '';"
-const updateThumbnailDatastoreAndLocation = "UPDATE thumbnails SET location = $8, datastore_id = $7 WHERE origin = $1 and media_id = $2 and width = $3 and height = $4 and method = $5 and animated = $6;"
-const selectThumbnailsForMedia = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE origin = $1 AND media_id = $2;"
-const deleteThumbnailsForMedia = "DELETE FROM thumbnails WHERE origin = $1 AND media_id = $2;"
-const selectThumbnailsCreatedBefore = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE creation_ts < $1;"
-const deleteThumbnailsWithHash = "DELETE FROM thumbnails WHERE sha256_hash = $1;"
-const selectThumbnailLocationsForDatastore = "SELECT distinct location FROM thumbnails WHERE datastore_id = $1;"
-
-type thumbnailStatements struct {
- selectThumbnail *sql.Stmt
- insertThumbnail *sql.Stmt
- updateThumbnailHash *sql.Stmt
- selectThumbnailsWithoutHash *sql.Stmt
- selectThumbnailsWithoutDatastore *sql.Stmt
- updateThumbnailDatastoreAndLocation *sql.Stmt
- selectThumbnailsForMedia *sql.Stmt
- deleteThumbnailsForMedia *sql.Stmt
- selectThumbnailsCreatedBefore *sql.Stmt
- deleteThumbnailsWithHash *sql.Stmt
- selectThumbnailLocationsForDatastore *sql.Stmt
-}
-
-type ThumbnailStoreFactory struct {
- sqlDb *sql.DB
- stmts *thumbnailStatements
-}
-
-type ThumbnailStore struct {
- factory *ThumbnailStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *thumbnailStatements // copied from factory
-}
-
-func InitThumbnailStore(sqlDb *sql.DB) (*ThumbnailStoreFactory, error) {
- store := ThumbnailStoreFactory{stmts: &thumbnailStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.selectThumbnail, err = store.sqlDb.Prepare(selectThumbnail); err != nil {
- return nil, err
- }
- if store.stmts.insertThumbnail, err = store.sqlDb.Prepare(insertThumbnail); err != nil {
- return nil, err
- }
- if store.stmts.updateThumbnailHash, err = store.sqlDb.Prepare(updateThumbnailHash); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailsWithoutHash, err = store.sqlDb.Prepare(selectThumbnailsWithoutHash); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailsWithoutDatastore, err = store.sqlDb.Prepare(selectThumbnailsWithoutDatastore); err != nil {
- return nil, err
- }
- if store.stmts.updateThumbnailDatastoreAndLocation, err = store.sqlDb.Prepare(updateThumbnailDatastoreAndLocation); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailsForMedia, err = store.sqlDb.Prepare(selectThumbnailsForMedia); err != nil {
- return nil, err
- }
- if store.stmts.deleteThumbnailsForMedia, err = store.sqlDb.Prepare(deleteThumbnailsForMedia); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailsCreatedBefore, err = store.sqlDb.Prepare(selectThumbnailsCreatedBefore); err != nil {
- return nil, err
- }
- if store.stmts.deleteThumbnailsWithHash, err = store.sqlDb.Prepare(deleteThumbnailsWithHash); err != nil {
- return nil, err
- }
- if store.stmts.selectThumbnailLocationsForDatastore, err = store.sqlDb.Prepare(selectThumbnailLocationsForDatastore); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *ThumbnailStoreFactory) New(ctx rcontext.RequestContext) *ThumbnailStore {
- return &ThumbnailStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *ThumbnailStore) Insert(thumbnail *types.Thumbnail) error {
- _, err := s.statements.insertThumbnail.ExecContext(
- s.ctx,
- thumbnail.Origin,
- thumbnail.MediaId,
- thumbnail.Width,
- thumbnail.Height,
- thumbnail.Method,
- thumbnail.Animated,
- thumbnail.ContentType,
- thumbnail.SizeBytes,
- thumbnail.DatastoreId,
- thumbnail.Location,
- thumbnail.CreationTs,
- thumbnail.Sha256Hash,
- )
-
- return err
-}
-
-func (s *ThumbnailStore) Get(origin string, mediaId string, width int, height int, method string, animated bool) (*types.Thumbnail, error) {
- t := &types.Thumbnail{}
- err := s.statements.selectThumbnail.QueryRowContext(s.ctx, origin, mediaId, width, height, method, animated).Scan(
- &t.Origin,
- &t.MediaId,
- &t.Width,
- &t.Height,
- &t.Method,
- &t.Animated,
- &t.ContentType,
- &t.SizeBytes,
- &t.DatastoreId,
- &t.Location,
- &t.CreationTs,
- &t.Sha256Hash,
- )
- return t, err
-}
-
-func (s *ThumbnailStore) UpdateHash(thumbnail *types.Thumbnail) error {
- _, err := s.statements.updateThumbnailHash.ExecContext(
- s.ctx,
- thumbnail.Origin,
- thumbnail.MediaId,
- thumbnail.Width,
- thumbnail.Height,
- thumbnail.Method,
- thumbnail.Animated,
- thumbnail.Sha256Hash,
- )
-
- return err
-}
-
-func (s *ThumbnailStore) UpdateDatastoreAndLocation(thumbnail *types.Thumbnail) error {
- _, err := s.statements.updateThumbnailDatastoreAndLocation.ExecContext(
- s.ctx,
- thumbnail.Origin,
- thumbnail.MediaId,
- thumbnail.Width,
- thumbnail.Height,
- thumbnail.Method,
- thumbnail.Animated,
- thumbnail.DatastoreId,
- thumbnail.Location,
- )
-
- return err
-}
-
-func (s *ThumbnailStore) GetAllWithoutHash() ([]*types.Thumbnail, error) {
- rows, err := s.statements.selectThumbnailsWithoutHash.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Thumbnail
- for rows.Next() {
- obj := &types.Thumbnail{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.Width,
- &obj.Height,
- &obj.Method,
- &obj.Animated,
- &obj.ContentType,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Sha256Hash,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *ThumbnailStore) GetAllWithoutDatastore() ([]*types.Thumbnail, error) {
- rows, err := s.statements.selectThumbnailsWithoutDatastore.QueryContext(s.ctx)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Thumbnail
- for rows.Next() {
- obj := &types.Thumbnail{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.Width,
- &obj.Height,
- &obj.Method,
- &obj.Animated,
- &obj.ContentType,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Sha256Hash,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *ThumbnailStore) GetAllForMedia(origin string, mediaId string) ([]*types.Thumbnail, error) {
- rows, err := s.statements.selectThumbnailsForMedia.QueryContext(s.ctx, origin, mediaId)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Thumbnail
- for rows.Next() {
- obj := &types.Thumbnail{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.Width,
- &obj.Height,
- &obj.Method,
- &obj.Animated,
- &obj.ContentType,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Sha256Hash,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *ThumbnailStore) DeleteAllForMedia(origin string, mediaId string) error {
- _, err := s.statements.deleteThumbnailsForMedia.ExecContext(s.ctx, origin, mediaId)
- if err != nil {
- return err
- }
- return nil
-}
-
-func (s *ThumbnailStore) GetOldThumbnails(beforeTs int64) ([]*types.Thumbnail, error) {
- rows, err := s.statements.selectThumbnailsCreatedBefore.QueryContext(s.ctx, beforeTs)
- if err != nil {
- return nil, err
- }
-
- var results []*types.Thumbnail
- for rows.Next() {
- obj := &types.Thumbnail{}
- err = rows.Scan(
- &obj.Origin,
- &obj.MediaId,
- &obj.Width,
- &obj.Height,
- &obj.Method,
- &obj.Animated,
- &obj.ContentType,
- &obj.SizeBytes,
- &obj.DatastoreId,
- &obj.Location,
- &obj.CreationTs,
- &obj.Sha256Hash,
- )
- if err != nil {
- return nil, err
- }
- results = append(results, obj)
- }
-
- return results, nil
-}
-
-func (s *ThumbnailStore) DeleteWithHash(sha256hash string) error {
- _, err := s.statements.deleteThumbnailsWithHash.ExecContext(s.ctx, sha256hash)
- if err != nil {
- return err
- }
- return nil
-}
-
-func (s *ThumbnailStore) GetDistinctLocationsForDatastore(datastoreId string) ([]string, error) {
- rows, err := s.statements.selectThumbnailLocationsForDatastore.QueryContext(s.ctx, datastoreId)
- if err != nil {
- return nil, err
- }
-
- locations := make([]string, 0)
- for rows.Next() {
- s := ""
- err = rows.Scan(&s)
- if err != nil {
- return nil, err
- }
- locations = append(locations, s)
- }
-
- return locations, nil
-}
diff --git a/storage/stores/url_store.go b/storage/stores/url_store.go
deleted file mode 100644
index b97cceb6d62d9cd92b84ee42d1e499dcab41f3fc..0000000000000000000000000000000000000000
--- a/storage/stores/url_store.go
+++ /dev/null
@@ -1,122 +0,0 @@
-package stores
-
-import (
- "database/sql"
-
- "github.com/turt2live/matrix-media-repo/common/rcontext"
- "github.com/turt2live/matrix-media-repo/types"
- "github.com/turt2live/matrix-media-repo/util"
-)
-
-const selectUrlPreview = "SELECT url, error_code, bucket_ts, site_url, site_name, resource_type, description, title, image_mxc, image_type, image_size, image_width, image_height, language_header FROM url_previews WHERE url = $1 AND bucket_ts = $2 AND language_header = $3;"
-const insertUrlPreview = "INSERT INTO url_previews (url, error_code, bucket_ts, site_url, site_name, resource_type, description, title, image_mxc, image_type, image_size, image_width, image_height, language_header) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14);"
-const deletePreviewsOlderThan = "DELETE FROM url_previews WHERE bucket_ts <= $1;"
-
-type urlStatements struct {
- selectUrlPreview *sql.Stmt
- insertUrlPreview *sql.Stmt
- deletePreviewsOlderThan *sql.Stmt
-}
-
-type UrlStoreFactory struct {
- sqlDb *sql.DB
- stmts *urlStatements
-}
-
-type UrlStore struct {
- factory *UrlStoreFactory // just for reference
- ctx rcontext.RequestContext
- statements *urlStatements // copied from factory
-}
-
-func InitUrlStore(sqlDb *sql.DB) (*UrlStoreFactory, error) {
- store := UrlStoreFactory{stmts: &urlStatements{}}
- var err error
-
- store.sqlDb = sqlDb
-
- if store.stmts.selectUrlPreview, err = store.sqlDb.Prepare(selectUrlPreview); err != nil {
- return nil, err
- }
- if store.stmts.insertUrlPreview, err = store.sqlDb.Prepare(insertUrlPreview); err != nil {
- return nil, err
- }
- if store.stmts.deletePreviewsOlderThan, err = store.sqlDb.Prepare(deletePreviewsOlderThan); err != nil {
- return nil, err
- }
-
- return &store, nil
-}
-
-func (f *UrlStoreFactory) Create(ctx rcontext.RequestContext) *UrlStore {
- return &UrlStore{
- factory: f,
- ctx: ctx,
- statements: f.stmts, // we copy this intentionally
- }
-}
-
-func (s *UrlStore) GetPreview(url string, ts int64, languageHeader string) (*types.CachedUrlPreview, error) {
- r := &types.CachedUrlPreview{
- Preview: &types.UrlPreview{},
- }
- err := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, GetBucketTs(ts), languageHeader).Scan(
- &r.SearchUrl,
- &r.ErrorCode,
- &r.FetchedTs,
- &r.Preview.Url,
- &r.Preview.SiteName,
- &r.Preview.Type,
- &r.Preview.Description,
- &r.Preview.Title,
- &r.Preview.ImageMxc,
- &r.Preview.ImageType,
- &r.Preview.ImageSize,
- &r.Preview.ImageWidth,
- &r.Preview.ImageHeight,
- &r.Preview.LanguageHeader,
- )
-
- return r, err
-}
-
-func (s *UrlStore) InsertPreview(record *types.CachedUrlPreview) error {
- _, err := s.statements.insertUrlPreview.ExecContext(
- s.ctx,
- record.SearchUrl,
- record.ErrorCode,
- GetBucketTs(record.FetchedTs),
- record.Preview.Url,
- record.Preview.SiteName,
- record.Preview.Type,
- record.Preview.Description,
- record.Preview.Title,
- record.Preview.ImageMxc,
- record.Preview.ImageType,
- record.Preview.ImageSize,
- record.Preview.ImageWidth,
- record.Preview.ImageHeight,
- record.Preview.LanguageHeader,
- )
-
- return err
-}
-
-func (s *UrlStore) InsertPreviewError(url string, errorCode string) error {
- return s.InsertPreview(&types.CachedUrlPreview{
- Preview: &types.UrlPreview{},
- SearchUrl: url,
- ErrorCode: errorCode,
- FetchedTs: util.NowMillis(),
- })
-}
-
-func (s *UrlStore) DeleteOlderThan(beforeTs int64) error {
- _, err := s.statements.deletePreviewsOlderThan.ExecContext(s.ctx, beforeTs)
- return err
-}
-
-func GetBucketTs(ts int64) int64 {
- // 1 hour buckets
- return (ts / 3600000) * 3600000
-}
diff --git a/tasks/task_runner/import_data.go b/tasks/task_runner/import_data.go
index 26a9092ec4650f24059e84b0bd31456701e02a2d..236fa38aff7de90b9b3120c50573de29ac66725e 100644
--- a/tasks/task_runner/import_data.go
+++ b/tasks/task_runner/import_data.go
@@ -173,7 +173,7 @@ func AppendImportFile(ctx rcontext.RequestContext, importId string, data io.Read
// caller *just* started the import job and is already trying to append.
for i := 0; i < 5; i++ {
err := try()
- if err == common.ErrMediaNotFound {
+ if errors.Is(err, common.ErrMediaNotFound) {
time.Sleep(100 * time.Millisecond)
} else {
return err
diff --git a/templates/export_index.html b/templates/export_index.html
index bf110445a12129af2915c814ab716cc62c0cd766..5be76eb4f08954a7b57d4c22db80184175a259a8 100644
--- a/templates/export_index.html
+++ b/templates/export_index.html
@@ -2,7 +2,7 @@
<html lang="en">
<head>
<title>{{.Entity}} media export</title>
- <style type="text/css">
+ <style>
body, html {
margin: 0;
padding: 0;
@@ -63,6 +63,7 @@
{{range .Media}}
<tr>
<td>
+ <!--suppress HtmlUnknownTarget -->
<a href="{{.ArchivedName}}"><code>{{if eq .FileName ""}}<no name>{{else}}{{.FileName}}{{end}}</code></a>
<span class="muted"><code>mxc://{{.Origin}}/{{.MediaID}}</code></span>
</td>
diff --git a/templates/view_export.html b/templates/view_export.html
index 380ddb1b3e880b5e5ac9ac0f055234569d3ed7fa..b2b6cff023bf34fb44f7349e4e87743405496aa2 100644
--- a/templates/view_export.html
+++ b/templates/view_export.html
@@ -2,7 +2,7 @@
<html lang="en">
<head>
<title>{{.Entity}} media export</title>
- <style type="text/css">
+ <style>
body, html {
margin: 0;
padding: 0;
@@ -24,7 +24,7 @@
}
</style>
<noscript>
- <style type="text/css">
+ <style>
#delete-option {
display: none;
}
@@ -33,12 +33,12 @@
<script type="text/javascript">
<!--
function deleteExport() {
- var url = "/_matrix/media/unstable/admin/export/{{.ExportID}}/delete";
- var xhr = new XMLHttpRequest();
+ const url = "/_matrix/media/unstable/admin/export/{{.ExportID}}/delete";
+ const xhr = new XMLHttpRequest();
xhr.open("DELETE", url, true);
xhr.onreadystatechange = function() {
if (this.readyState !== 4) return;
- var element = document.getElementById("main");
+ const element = document.getElementById("main");
if (!element) return;
if (this.status === 200) {
@@ -62,7 +62,7 @@
</p>
<ul>
{{range .ExportParts}}
- <li><a href="/_matrix/media/unstable/admin/export/{{.ExportID}}/part/{{.Index}}" download>{{.FileName}}</a> ({{.SizeBytesHuman}})</li>
+ <li><!--suppress HtmlUnknownTarget --><a href="/_matrix/media/unstable/admin/export/{{.ExportID}}/part/{{.Index}}" download>{{.FileName}}</a> ({{.SizeBytesHuman}})</li>
{{end}}
</ul>
<p id="delete-option">Downloaded all your data? <a href="javascript:deleteExport()">Delete your export</a></p>
diff --git a/thumbnailing/i/01-factories.go b/thumbnailing/i/01-factories.go
index 2779175c83a0dc01e7522c8e3e8fd6281792c537..e8410865c51bc9b1e0fa54cb6a262d8abd88a2b1 100644
--- a/thumbnailing/i/01-factories.go
+++ b/thumbnailing/i/01-factories.go
@@ -49,16 +49,3 @@ func GetSupportedContentTypes() []string {
}
return a
}
-
-func GetSupportedAnimationTypes() []string {
- a := make([]string, 0)
- for _, d := range generators {
- if !d.supportsAnimation() {
- continue
- }
- for _, c := range d.supportedContentTypes() {
- a = append(a, c)
- }
- }
- return a
-}
diff --git a/types/MAYBE_LEGACY.txt b/types/MAYBE_LEGACY.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/types/background_task.go b/types/background_task.go
deleted file mode 100644
index 9f7111dd92cfdd125830fff1a57ec1bf5e88b7f1..0000000000000000000000000000000000000000
--- a/types/background_task.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package types
-
-type BackgroundTask struct {
- ID int
- Name string
- Params map[string]interface{}
- StartTs int64
- EndTs int64
-}
diff --git a/types/datastore.go b/types/datastore.go
deleted file mode 100644
index 6cc13167fa163607711c451a47dc70c501b64fb4..0000000000000000000000000000000000000000
--- a/types/datastore.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package types
-
-type Datastore struct {
- DatastoreId string
- Type string
- Uri string
-}
-
-type DatastoreMigrationEstimate struct {
- ThumbnailsAffected int64 `json:"thumbnails_affected"`
- ThumbnailHashesAffected int64 `json:"thumbnail_hashes_affected"`
- ThumbnailBytes int64 `json:"thumbnail_bytes"`
- MediaAffected int64 `json:"media_affected"`
- MediaHashesAffected int64 `json:"media_hashes_affected"`
- MediaBytes int64 `json:"media_bytes"`
- TotalHashesAffected int64 `json:"total_hashes_affected"`
- TotalBytes int64 `json:"total_bytes"`
-}
diff --git a/types/exports.go b/types/exports.go
deleted file mode 100644
index 17006fb79b59de5dd7f6fcd64f79b2709b3296d4..0000000000000000000000000000000000000000
--- a/types/exports.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package types
-
-type ExportMetadata struct {
- ExportID string
- Entity string
-}
-
-type ExportPart struct {
- ExportID string
- Index int
- FileName string
- SizeBytes int64
- DatastoreID string
- Location string
-}
diff --git a/types/media.go b/types/media.go
deleted file mode 100644
index 873da958825193c03c6152b283ff46bdcadf234d..0000000000000000000000000000000000000000
--- a/types/media.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package types
-
-import "io"
-
-type Media struct {
- Origin string
- MediaId string
- UploadName string
- ContentType string
- UserId string
- Sha256Hash string
- SizeBytes int64
- DatastoreId string
- Location string
- CreationTs int64
- Quarantined bool
-}
-
-type MinimalMedia struct {
- Origin string
- MediaId string
- Stream io.ReadCloser
- UploadName string
- ContentType string
- SizeBytes int64
- KnownMedia *Media
-}
-
-type MinimalMediaMetadata struct {
- SizeBytes int64
- Sha256Hash string
- Location string
- CreationTs int64
- LastAccessTs int64
- DatastoreId string
-}
-
-type UserUsageStats struct {
- MediaCount int64
- MediaLength int64
- UserId string
-}
-
-func (m *Media) MxcUri() string {
- return "mxc://" + m.Origin + "/" + m.MediaId
-}
diff --git a/types/media_attributes.go b/types/media_attributes.go
deleted file mode 100644
index c938e0e95d7b64f4d5c08a1c2cc5611ee260cde2..0000000000000000000000000000000000000000
--- a/types/media_attributes.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package types
-
-type MediaAttributes struct {
- Origin string
- MediaId string
- Purpose string
-}
-
-const PurposeNone = "none"
-const PurposePinned = "pinned"
-
-var AllPurposes = []string{PurposeNone, PurposePinned}
diff --git a/types/object_info.go b/types/object_info.go
deleted file mode 100644
index 11fe6a40bc6c29edfc21ad8dee539c6658583b7d..0000000000000000000000000000000000000000
--- a/types/object_info.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package types
-
-type ObjectInfo struct {
- Location string
- Sha256Hash string
- SizeBytes int64
-}
diff --git a/types/stats.go b/types/stats.go
deleted file mode 100644
index 01188c9de64fa1e8ef0a4c28d1e3fc9060cb05f9..0000000000000000000000000000000000000000
--- a/types/stats.go
+++ /dev/null
@@ -1,6 +0,0 @@
-package types
-
-type UserStats struct {
- UserId string
- UploadedBytes int64
-}
diff --git a/types/thumbnail.go b/types/thumbnail.go
deleted file mode 100644
index de71be3445d6be63af0bc25d4db791307cff8031..0000000000000000000000000000000000000000
--- a/types/thumbnail.go
+++ /dev/null
@@ -1,25 +0,0 @@
-package types
-
-import (
- "io"
-)
-
-type Thumbnail struct {
- Origin string
- MediaId string
- Width int
- Height int
- Method string // "crop" or "scale"
- Animated bool
- ContentType string
- SizeBytes int64
- DatastoreId string
- Location string
- CreationTs int64
- Sha256Hash string
-}
-
-type StreamedThumbnail struct {
- Thumbnail *Thumbnail
- Stream io.ReadCloser
-}
diff --git a/types/url_preview.go b/types/url_preview.go
deleted file mode 100644
index 6dd1a8dde12e2d68eaa1226e14f1ebffdf7fc5d9..0000000000000000000000000000000000000000
--- a/types/url_preview.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package types
-
-type UrlPreview struct {
- Url string
- SiteName string
- Type string
- Description string
- Title string
- ImageMxc string
- ImageType string
- ImageSize int64
- ImageWidth int
- ImageHeight int
- LanguageHeader string
-}
-
-type CachedUrlPreview struct {
- Preview *UrlPreview
- SearchUrl string
- ErrorCode string
- FetchedTs int64
-}
diff --git a/url_previewing/p/calculated.go b/url_previewing/p/calculated.go
index ac2b41845607bfd970dffac6b279087efff72726..60421d1672cdca7f6ab1eea4a2cb7411edd701cc 100644
--- a/url_previewing/p/calculated.go
+++ b/url_previewing/p/calculated.go
@@ -1,6 +1,8 @@
package p
import (
+ "errors"
+
"github.com/prometheus/client_golang/prometheus"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
@@ -16,7 +18,7 @@ func GenerateCalculatedPreview(urlPayload *m.UrlPayload, languageHeader string,
ctx.Log.Warn("Error downloading content: ", err)
// Make sure the unsupported error gets passed through
- if err == m.ErrPreviewUnsupported {
+ if errors.Is(err, m.ErrPreviewUnsupported) {
return m.PreviewResult{}, m.ErrPreviewUnsupported
}
diff --git a/url_previewing/p/opengraph.go b/url_previewing/p/opengraph.go
index cb510a766270aad81331733c03f56fcc28e2f633..55a7bd78b8ca6957acab838eeb232c6906ee45bc 100644
--- a/url_previewing/p/opengraph.go
+++ b/url_previewing/p/opengraph.go
@@ -1,6 +1,7 @@
package p
import (
+ "errors"
"net/url"
"strconv"
"strings"
@@ -26,7 +27,7 @@ func GenerateOpenGraphPreview(urlPayload *m.UrlPayload, languageHeader string, c
ctx.Log.Error("Error downloading content: ", err)
// Make sure the unsupported error gets passed through
- if err == m.ErrPreviewUnsupported {
+ if errors.Is(err, m.ErrPreviewUnsupported) {
return m.PreviewResult{}, m.ErrPreviewUnsupported
}
diff --git a/util/LEGACY_files.go b/util/LEGACY_files.go
deleted file mode 100644
index 178ae1d8815cd4be638504bc2640705bf043c360..0000000000000000000000000000000000000000
--- a/util/LEGACY_files.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package util
-
-import (
- "os"
- "path"
-
- "github.com/turt2live/matrix-media-repo/util/stream_util"
-)
-
-func FileExists(path string) (bool, error) {
- _, err := os.Stat(path)
- if err == nil {
- return true, nil
- }
- if os.IsNotExist(err) {
- return false, nil
- }
- return true, err
-}
-
-func GetLastSegmentsOfPath(strPath string, segments int) string {
- combined := ""
- for i := 1; i <= segments; i++ {
- d, p := path.Split(strPath)
- strPath = path.Clean(d)
- combined = path.Join(p, combined)
- }
- return combined
-}
-
-func GetFileHash(filePath string) (string, error) {
- f, err := os.Open(filePath)
- if err != nil {
- return "", err
- }
- defer stream_util.DumpAndCloseStream(f)
-
- return stream_util.GetSha256HashOfStream(f)
-}
diff --git a/util/resource_handler/LEGACY_handler.go b/util/resource_handler/LEGACY_handler.go
deleted file mode 100644
index 66c8d05d286619eb5c74415ec42dcac5412813d4..0000000000000000000000000000000000000000
--- a/util/resource_handler/LEGACY_handler.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package resource_handler
-
-import (
- "reflect"
- "time"
-
- "github.com/Jeffail/tunny"
- "github.com/olebedev/emitter"
- "github.com/patrickmn/go-cache"
- "github.com/sirupsen/logrus"
-)
-
-type ResourceHandler struct {
- pool *tunny.Pool
- eventBus *emitter.Emitter
- itemCache *cache.Cache
-}
-
-type resource struct {
- isComplete bool
- result interface{}
-}
-
-type WorkRequest struct {
- Id string
- Metadata interface{}
-}
-
-func New(workers int, fetchFn func(object *WorkRequest) interface{}) (*ResourceHandler, error) {
- workFn := func(i interface{}) interface{} { return fetchFn(i.(*WorkRequest)) }
- pool := tunny.NewFunc(workers, workFn)
-
- bus := &emitter.Emitter{}
- itemCache := cache.New(30*time.Second, 1*time.Minute) // cache work for 30ish seconds
-
- handler := &ResourceHandler{pool, bus, itemCache}
- return handler, nil
-}
-
-func (h *ResourceHandler) Close() {
- logrus.Warn("Closing resource handler: " + reflect.TypeOf(h).Name())
- h.pool.Close()
-}
-
-func (h *ResourceHandler) GetResource(id string, metadata interface{}) chan interface{} {
- resultChan := make(chan interface{})
-
- // First see if we have already cached this request
- cachedResource, found := h.itemCache.Get(id)
- if found {
- res := cachedResource.(*resource)
-
- // If the request has already been completed, return that result
- if res.isComplete {
- // This is a goroutine to avoid a problem where the sending and return can race
- go func() {
- logrus.Warn("Returning cached reply from resource handler for resource ID " + id)
- resultChan <- res.result
- }()
- return resultChan
- }
-
- // Otherwise queue a wait function to handle the resource when it is complete
- go func() {
- result := <-h.eventBus.Once("complete_" + id)
- resultChan <- result.Args[0]
- }()
-
- return resultChan
- }
-
- // Cache that we're starting the request (never expire)
- h.itemCache.Set(id, &resource{false, nil}, cache.NoExpiration)
-
- go func() {
- // Queue the work (ignore errors)
- result := h.pool.Process(&WorkRequest{id, metadata})
- h.eventBus.Emit("complete_"+id, result)
-
- // Cache the result for future callers
- newResource := &resource{
- isComplete: true,
- result: result,
- }
- h.itemCache.Set(id, newResource, cache.DefaultExpiration)
-
- // and finally feed it back to the caller
- resultChan <- result
- }()
-
- return resultChan
-}
diff --git a/util/singleflight-counter/LEGACY_singleflight.go b/util/singleflight-counter/LEGACY_singleflight.go
deleted file mode 100644
index b32095c95cd40ff1e12b6fdb68c1fdb98e128a90..0000000000000000000000000000000000000000
--- a/util/singleflight-counter/LEGACY_singleflight.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package singleflight_counter
-
-import (
- "sync"
-)
-
-// Largely inspired by Go's singleflight package.
-// https://github.com/golang/sync/blob/112230192c580c3556b8cee6403af37a4fc5f28c/singleflight/singleflight.go
-
-type call struct {
- wg sync.WaitGroup
-
- valsMu sync.Mutex
- nextIndex int
- vals []interface{}
-
- val interface{}
- err error
- count int
-}
-
-type Group struct {
- mu sync.Mutex
- m map[string]*call
-}
-
-func (c *call) NextVal() interface{} {
- c.valsMu.Lock()
- val := c.val
- if c.vals != nil && len(c.vals) >= c.count {
- val = c.vals[c.nextIndex]
- c.nextIndex++
- }
- c.valsMu.Unlock()
- return val
-}
-
-func (g *Group) DoWithoutPost(key string, fn func() (interface{}, error)) (interface{}, int, error) {
- return g.Do(key, fn, func(v interface{}, total int, e error) []interface{} {
- return nil
- })
-}
-
-func (g *Group) Do(key string, fn func() (interface{}, error), postprocess func(v interface{}, total int, e error) []interface{}) (interface{}, int, error) {
- g.mu.Lock()
- if g.m == nil {
- g.m = make(map[string]*call)
- }
- if c, ok := g.m[key]; ok {
- c.count++
- g.mu.Unlock()
- c.wg.Wait()
-
- return c.NextVal(), c.count, c.err
- }
-
- c := new(call)
- c.count = 1 // Always start at 1 (for us)
- c.nextIndex = 0
- c.wg.Add(1)
- g.m[key] = c
- g.mu.Unlock()
-
- c.val, c.err = fn()
-
- g.mu.Lock()
- delete(g.m, key)
- g.mu.Unlock()
-
- c.vals = nil
- if postprocess != nil {
- c.vals = postprocess(c.val, c.count, c.err)
- }
-
- c.wg.Done()
-
- return c.NextVal(), c.count, c.err
-}
diff --git a/util/stream_util/LEGACY.txt b/util/stream_util/LEGACY.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/util/stream_util/streams.go b/util/stream_util/streams.go
deleted file mode 100644
index 3e921e5d218a5323198eb2ae6f816195e93af648..0000000000000000000000000000000000000000
--- a/util/stream_util/streams.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package stream_util
-
-import (
- "bytes"
- "crypto/sha256"
- "encoding/hex"
- "errors"
- "fmt"
- "io"
- "math"
-)
-
-func BufferToStream(buf *bytes.Buffer) io.ReadCloser {
- newBuf := bytes.NewReader(buf.Bytes())
- return io.NopCloser(newBuf)
-}
-
-func BytesToStream(b []byte) io.ReadCloser {
- return io.NopCloser(bytes.NewBuffer(b))
-}
-
-func GetSha256HashOfStream(r io.ReadCloser) (string, error) {
- defer DumpAndCloseStream(r)
-
- hasher := sha256.New()
-
- if _, err := io.Copy(hasher, r); err != nil {
- return "", err
- }
-
- return hex.EncodeToString(hasher.Sum(nil)), nil
-}
-
-func ForceDiscard(r io.Reader, nBytes int64) error {
- if nBytes == 0 {
- return nil // weird call, but ok
- }
-
- buf := make([]byte, 128)
-
- if nBytes < 0 {
- for true {
- _, err := r.Read(buf)
- if err == io.EOF {
- break
- } else if err != nil {
- return err
- }
- }
- return nil
- }
-
- read := int64(0)
- for (nBytes - read) > 0 {
- toRead := int(math.Min(float64(len(buf)), float64(nBytes-read)))
- if toRead != len(buf) {
- buf = make([]byte, toRead)
- }
- actuallyRead, err := r.Read(buf)
- if err != nil {
- return err
- }
- read += int64(actuallyRead)
- if (nBytes - read) < 0 {
- return errors.New(fmt.Sprintf("over-discarded from stream by %d bytes", nBytes-read))
- }
- }
-
- return nil
-}
-
-func DumpAndCloseStream(r io.ReadCloser) {
- if r == nil {
- return // nothing to dump or close
- }
- _ = ForceDiscard(r, -1)
- _ = r.Close()
-}
diff --git a/util/urls.go b/util/urls.go
index 7b02ede39041d4ecb7e30f20501de08f65b4de57..59f93bd846fb64097154a416de5e06a2fcb44f47 100644
--- a/util/urls.go
+++ b/util/urls.go
@@ -12,4 +12,4 @@ func MakeUrl(parts ...string) string {
}
}
return res
-}
\ No newline at end of file
+}
diff --git a/util/util_byte_seeker/LEGACY.txt b/util/util_byte_seeker/LEGACY.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/util/util_byte_seeker/seekable.go b/util/util_byte_seeker/seekable.go
deleted file mode 100644
index 3011677e36f766df3ecb3f76f9cd01a82f09463e..0000000000000000000000000000000000000000
--- a/util/util_byte_seeker/seekable.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package util_byte_seeker
-
-import (
- "bytes"
- "io"
-)
-
-type ByteSeeker struct {
- io.ReadSeekCloser
- s *bytes.Reader
-}
-
-func NewByteSeeker(b []byte) ByteSeeker {
- return ByteSeeker{s: bytes.NewReader(b)}
-}
-
-func (s ByteSeeker) Close() error {
- return nil // no-op
-}
-
-func (s ByteSeeker) Read(p []byte) (n int, err error) {
- return s.s.Read(p)
-}
-
-func (s ByteSeeker) Seek(offset int64, whence int) (int64, error) {
- return s.s.Seek(offset, whence)
-}