diff --git a/api/custom/purge.go b/api/custom/purge.go
index a5be8e0a547f131b23eb24751bbd4b70a884893b..9b5f40a5dd65802da42b1af0628dc98a250a3856 100644
--- a/api/custom/purge.go
+++ b/api/custom/purge.go
@@ -4,6 +4,7 @@ import (
"net/http"
"strconv"
+ "github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
@@ -36,3 +37,42 @@ func PurgeRemoteMedia(r *http.Request, log *logrus.Entry, user api.UserInfo) int
return &api.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}}
}
+
+func PurgeIndividualRecord(r *http.Request, log *logrus.Entry, user api.UserInfo) interface{} {
+ // TODO: Allow non-repo-admins to delete things
+
+ params := mux.Vars(r)
+
+ server := params["server"]
+ mediaId := params["mediaId"]
+
+ log = log.WithFields(logrus.Fields{
+ "server": server,
+ "mediaId": mediaId,
+ })
+
+ err := maintenance_controller.PurgeMedia(server, mediaId, r.Context(), log)
+ if err != nil {
+ log.Error("Error purging media: " + err.Error())
+ return api.InternalServerError("error purging media")
+ }
+
+ return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}}
+}
+
+func PurgeQurantined(r *http.Request, log *logrus.Entry, user api.UserInfo) interface{} {
+ // TODO: Allow non-repo-admins to delete things
+
+ affected, err := maintenance_controller.PurgeQuarantined(r.Context(), log)
+ if err != nil {
+ log.Error("Error purging media: " + err.Error())
+ return api.InternalServerError("error purging media")
+ }
+
+ mxcs := make([]string, 0)
+ for _, a := range affected {
+ mxcs = append(mxcs, a.MxcUri())
+ }
+
+ return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+}
diff --git a/api/webserver/webserver.go b/api/webserver/webserver.go
index 8478501d0bcc7e81d085fa21de362c182903fb16..7274ee27f66c34d66ac7e17d17c277de90324982 100644
--- a/api/webserver/webserver.go
+++ b/api/webserver/webserver.go
@@ -31,7 +31,9 @@ func Init() {
thumbnailHandler := handler{api.AccessTokenOptionalRoute(r0.ThumbnailMedia), "thumbnail", counter, false}
previewUrlHandler := handler{api.AccessTokenRequiredRoute(r0.PreviewUrl), "url_preview", counter, false}
identiconHandler := handler{api.AccessTokenOptionalRoute(r0.Identicon), "identicon", counter, false}
- purgeHandler := handler{api.RepoAdminRoute(custom.PurgeRemoteMedia), "purge_remote_media", counter, false}
+ purgeRemote := handler{api.RepoAdminRoute(custom.PurgeRemoteMedia), "purge_remote_media", counter, false}
+ purgeOneHandler := handler{api.RepoAdminRoute(custom.PurgeIndividualRecord), "purge_individual_media", counter, false}
+ purgeQuarantinedHandler := handler{api.RepoAdminRoute(custom.PurgeQurantined), "purge_quarantined", counter, false}
quarantineHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineMedia), "quarantine_media", counter, false}
quarantineRoomHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineRoomMedia), "quarantine_room", counter, false}
localCopyHandler := handler{api.AccessTokenRequiredRoute(unstable.LocalCopy), "local_copy", counter, false}
@@ -63,7 +65,10 @@ func Init() {
routes["/_matrix/media/"+version+"/config"] = route{"GET", configHandler}
// Routes that we define but are not part of the spec (management)
- routes["/_matrix/media/"+version+"/admin/purge_remote"] = route{"POST", purgeHandler}
+ routes["/_matrix/media/"+version+"/admin/purge_remote"] = route{"POST", purgeRemote}
+ routes["/_matrix/media/"+version+"/admin/purge/remote"] = route{"POST", purgeRemote}
+ routes["/_matrix/media/"+version+"/admin/purge/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"POST", purgeOneHandler}
+ routes["/_matrix/media/"+version+"/admin/purge/quarantined"] = route{"POST", purgeQuarantinedHandler}
routes["/_matrix/media/"+version+"/admin/quarantine/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"POST", quarantineHandler}
routes["/_matrix/media/"+version+"/admin/room/{roomId:[^/]+}/quarantine"] = route{"POST", quarantineRoomHandler}
routes["/_matrix/media/"+version+"/admin/datastores/{datastoreId:[^/]+}/size_estimate"] = route{"GET", storageEstimateHandler}
@@ -78,7 +83,7 @@ func Init() {
routes["/_matrix/media/"+version+"/admin/tasks/unfinished"] = route{"GET", listUnfinishedBackgroundTasksHandler}
// Routes that we should handle but aren't in the media namespace (synapse compat)
- routes["/_matrix/client/"+version+"/admin/purge_media_cache"] = route{"POST", purgeHandler}
+ routes["/_matrix/client/"+version+"/admin/purge_media_cache"] = route{"POST", purgeRemote}
routes["/_matrix/client/"+version+"/admin/quarantine_media/{roomId:[^/]+}"] = route{"POST", quarantineRoomHandler}
if version == "unstable" {
diff --git a/controllers/maintenance_controller/maintainance_controller.go b/controllers/maintenance_controller/maintainance_controller.go
index 8bc56af5f2f570c93ecbbba04aa8cca2150f1ed9..03f6cf6051feb87e18ab05b7f5c332c6d867b590 100644
--- a/controllers/maintenance_controller/maintainance_controller.go
+++ b/controllers/maintenance_controller/maintainance_controller.go
@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/sirupsen/logrus"
+ "github.com/turt2live/matrix-media-repo/controllers/download_controller"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
@@ -19,7 +20,7 @@ func StartStorageMigration(sourceDs *datastore.DatastoreRef, targetDs *datastore
task, err := db.CreateBackgroundTask("storage_migration", map[string]interface{}{
"source_datastore_id": sourceDs.DatastoreId,
"target_datastore_id": targetDs.DatastoreId,
- "before_ts": beforeTs,
+ "before_ts": beforeTs,
})
if err != nil {
return nil, err
@@ -149,6 +150,7 @@ func EstimateDatastoreSizeWithAge(beforeTs int64, datastoreId string, ctx contex
func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Entry) (int, error) {
db := storage.GetDatabase().GetMediaStore(ctx, log)
+ thumbsDb := storage.GetDatabase().GetThumbnailStore(ctx, log)
origins, err := db.GetOrigins()
if err != nil {
@@ -196,7 +198,101 @@ func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Ent
if err != nil {
log.Warn("Error removing media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
}
+
+ // Delete the thumbnails too
+ thumbs, err := thumbsDb.GetAllForMedia(media.Origin, media.MediaId)
+ if err != nil {
+ log.Warn("Error getting thumbnails for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+ continue
+ }
+ for _, thumb := range thumbs {
+ log.Info("Deleting thumbnail with hash: ", thumb.Sha256Hash)
+ ds, err := datastore.LocateDatastore(ctx, log, thumb.DatastoreId)
+ if err != nil {
+ log.Warn("Error removing thumbnail for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+ continue
+ }
+
+ err = ds.DeleteObject(thumb.Location)
+ if err != nil {
+ log.Warn("Error removing thumbnail for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+ continue
+ }
+ }
+ err = thumbsDb.DeleteAllForMedia(media.Origin, media.MediaId)
+ if err != nil {
+ log.Warn("Error removing thumbnails for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+ }
}
return removed, nil
}
+
+func PurgeQuarantined(ctx context.Context, log *logrus.Entry) ([]*types.Media, error) {
+ mediaDb := storage.GetDatabase().GetMediaStore(ctx, log)
+ records, err := mediaDb.GetAllQuarantinedMedia()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, r := range records {
+ err = doPurge(r, ctx, log)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return records, nil
+}
+
+func PurgeMedia(origin string, mediaId string, ctx context.Context, log *logrus.Entry) error {
+ media, err := download_controller.FindMediaRecord(origin, mediaId, false, ctx, log)
+ if err != nil {
+ return err
+ }
+
+ return doPurge(media, ctx, log)
+}
+
+func doPurge(media *types.Media, ctx context.Context, log *logrus.Entry) error {
+ // Delete all the thumbnails first
+ thumbsDb := storage.GetDatabase().GetThumbnailStore(ctx, log)
+ thumbs, err := thumbsDb.GetAllForMedia(media.Origin, media.MediaId)
+ if err != nil {
+ return err
+ }
+ for _, thumb := range thumbs {
+ log.Info("Deleting thumbnail with hash: ", thumb.Sha256Hash)
+ ds, err := datastore.LocateDatastore(ctx, log, thumb.DatastoreId)
+ if err != nil {
+ return err
+ }
+
+ err = ds.DeleteObject(thumb.Location)
+ if err != nil {
+ return err
+ }
+ }
+ err = thumbsDb.DeleteAllForMedia(media.Origin, media.MediaId)
+ if err != nil {
+ return err
+ }
+
+ ds, err := datastore.LocateDatastore(ctx, log, media.DatastoreId)
+ if err != nil {
+ return err
+ }
+
+ err = ds.DeleteObject(media.Location)
+ if err != nil {
+ return err
+ }
+
+ mediaDb := storage.GetDatabase().GetMediaStore(ctx, log)
+ err = mediaDb.Delete(media.Origin, media.MediaId)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/docs/admin.md b/docs/admin.md
index 48fe21e60f01e4d4c87fbedf68ad4a8c71e1ea91..945e48bc60264ed2025cc4d536916d0384d0508e 100644
--- a/docs/admin.md
+++ b/docs/admin.md
@@ -2,14 +2,30 @@
All the API calls here require your user ID to be listed in the configuration as an administrator. After that, your access token for your homeserver will grant you access to these APIs. The URLs should be hit against a configured homeserver. For example, if you have `t2bot.io` configured as a homeserver, then the admin API can be used at `https://t2bot.io/_matrix/media/unstable/admin/...`.
-## Remote media purge
+## Media purge
-URL: `POST /_matrix/media/unstable/admin/purge_remote?before_ts=1234567890&access_token=your_access_token` (`before_ts` is in milliseconds)
+Sometimes you just want your disk space back - purging media is the best way to do that. **Be careful about what you're purging.** The media repo will happily purge a local media object, making it highly unlikely to ever exist in Matrix again. When the media repo deletes remote media, it is only deleting its copy of it - it cannot delete media on the remote server itself. Thumbnails will also be deleted for the media.
+
+#### Purge remote media
+
+URL: `POST /_matrix/media/unstable/admin/purge/remote?before_ts=1234567890&access_token=your_access_token` (`before_ts` is in milliseconds)
This will delete remote media from the file store that was downloaded before the timestamp specified. If the file is referenced by newer remote media or local files to any of the configured homeservers, it will not be deleted. Be aware that removing a homeserver from the config will cause it to be considered a remote server, and therefore the media may be deleted.
Any remote media that is deleted and requested by a user will be downloaded again.
+#### Purge quarantined media
+
+URL: `POST /_matrix/media/unstable/admin/purge/quarantined?access_token=your_access_token`
+
+This will delete all media that has previously been quarantined, local or remote.
+
+#### Purge individual record
+
+URL: `POST /_matrix/media/unstable/admin/purge/<server>/<media id>?access_token=your_access_token`
+
+This will delete the media record, regardless of it being local or remote.
+
## Quarantine media
URL: `POST /_matrix/media/unstable/admin/quarantine/<server>/<media id>?access_token=your_access_token`
diff --git a/storage/stores/media_store.go b/storage/stores/media_store.go
index d2e86a2fbe0c25ba839bedcb039da74c6af5e58b..e13b2a6a2c95370828ac871eec273cf0040e673c 100644
--- a/storage/stores/media_store.go
+++ b/storage/stores/media_store.go
@@ -26,6 +26,7 @@ const selectAllDatastores = "SELECT datastore_id, ds_type, uri FROM datastores;"
const selectAllMediaForServer = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1"
const selectAllMediaForServerUsers = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND user_id = ANY($2)"
const selectAllMediaForServerIds = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND media_id = ANY($2)"
+const selectQuarantinedMedia = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE quarantined = true;"
var dsCacheByPath = sync.Map{} // [string] => Datastore
var dsCacheById = sync.Map{} // [string] => Datastore
@@ -48,6 +49,7 @@ type mediaStoreStatements struct {
selectAllMediaForServer *sql.Stmt
selectAllMediaForServerUsers *sql.Stmt
selectAllMediaForServerIds *sql.Stmt
+ selectQuarantinedMedia *sql.Stmt
}
type MediaStoreFactory struct {
@@ -116,6 +118,9 @@ func InitMediaStore(sqlDb *sql.DB) (*MediaStoreFactory, error) {
if store.stmts.selectAllMediaForServerIds, err = store.sqlDb.Prepare(selectAllMediaForServerIds); err != nil {
return nil, err
}
+ if store.stmts.selectQuarantinedMedia, err = store.sqlDb.Prepare(selectQuarantinedMedia); err != nil {
+ return nil, err
+ }
return &store, nil
}
@@ -489,3 +494,34 @@ func (s *MediaStore) GetAllMediaInIds(serverName string, mediaIds []string) ([]*
return results, nil
}
+
+func (s *MediaStore) GetAllQuarantinedMedia() ([]*types.Media, error) {
+ rows, err := s.statements.selectQuarantinedMedia.QueryContext(s.ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ var results []*types.Media
+ for rows.Next() {
+ obj := &types.Media{}
+ err = rows.Scan(
+ &obj.Origin,
+ &obj.MediaId,
+ &obj.UploadName,
+ &obj.ContentType,
+ &obj.UserId,
+ &obj.Sha256Hash,
+ &obj.SizeBytes,
+ &obj.DatastoreId,
+ &obj.Location,
+ &obj.CreationTs,
+ &obj.Quarantined,
+ )
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, obj)
+ }
+
+ return results, nil
+}
diff --git a/storage/stores/thumbnail_store.go b/storage/stores/thumbnail_store.go
index 769545b70f6b6ebf6de352c00505cd53dd62665b..4502147ffcf9eb06258ab3853a6938386540f447 100644
--- a/storage/stores/thumbnail_store.go
+++ b/storage/stores/thumbnail_store.go
@@ -14,6 +14,8 @@ const updateThumbnailHash = "UPDATE thumbnails SET sha256_hash = $7 WHERE origin
const selectThumbnailsWithoutHash = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE sha256_hash IS NULL OR sha256_hash = '';"
const selectThumbnailsWithoutDatastore = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE datastore_id IS NULL OR datastore_id = '';"
const updateThumbnailDatastoreAndLocation = "UPDATE thumbnails SET location = $8, datastore_id = $7 WHERE origin = $1 and media_id = $2 and width = $3 and height = $4 and method = $5 and animated = $6;"
+const selectThumbnailsForMedia = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE origin = $1 AND media_id = $2;"
+const deleteThumbnailsForMedia = "DELETE FROM thumbnails WHERE origin = $1 AND media_id = $2;"
type thumbnailStatements struct {
selectThumbnail *sql.Stmt
@@ -22,6 +24,8 @@ type thumbnailStatements struct {
selectThumbnailsWithoutHash *sql.Stmt
selectThumbnailsWithoutDatastore *sql.Stmt
updateThumbnailDatastoreAndLocation *sql.Stmt
+ selectThumbnailsForMedia *sql.Stmt
+ deleteThumbnailsForMedia *sql.Stmt
}
type ThumbnailStoreFactory struct {
@@ -60,6 +64,12 @@ func InitThumbnailStore(sqlDb *sql.DB) (*ThumbnailStoreFactory, error) {
if store.stmts.updateThumbnailDatastoreAndLocation, err = store.sqlDb.Prepare(updateThumbnailDatastoreAndLocation); err != nil {
return nil, err
}
+ if store.stmts.selectThumbnailsForMedia, err = store.sqlDb.Prepare(selectThumbnailsForMedia); err != nil {
+ return nil, err
+ }
+ if store.stmts.deleteThumbnailsForMedia, err = store.sqlDb.Prepare(deleteThumbnailsForMedia); err != nil {
+ return nil, err
+ }
return &store, nil
}
@@ -206,3 +216,43 @@ func (s *ThumbnailStore) GetAllWithoutDatastore() ([]*types.Thumbnail, error) {
return results, nil
}
+
+func (s *ThumbnailStore) GetAllForMedia(origin string, mediaId string) ([]*types.Thumbnail, error) {
+ rows, err := s.statements.selectThumbnailsForMedia.QueryContext(s.ctx, origin, mediaId)
+ if err != nil {
+ return nil, err
+ }
+
+ var results []*types.Thumbnail
+ for rows.Next() {
+ obj := &types.Thumbnail{}
+ err = rows.Scan(
+ &obj.Origin,
+ &obj.MediaId,
+ &obj.Width,
+ &obj.Height,
+ &obj.Method,
+ &obj.Animated,
+ &obj.ContentType,
+ &obj.SizeBytes,
+ &obj.DatastoreId,
+ &obj.Location,
+ &obj.CreationTs,
+ &obj.Sha256Hash,
+ )
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, obj)
+ }
+
+ return results, nil
+}
+
+func (s *ThumbnailStore) DeleteAllForMedia(origin string, mediaId string) error {
+ _, err := s.statements.deleteThumbnailsForMedia.ExecContext(s.ctx, origin, mediaId)
+ if err != nil {
+ return err
+ }
+ return nil
+}