Add an endpoint to delete quarantined media and individual media

Fixes https://github.com/turt2live/matrix-media-repo/issues/180

api/custom/purge.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/api"
 	"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
@@ -36,3 +37,42 @@ func PurgeRemoteMedia(r *http.Request, log *logrus.Entry, user api.UserInfo) int
 
 	return &api.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}}
 }
+
+func PurgeIndividualRecord(r *http.Request, log *logrus.Entry, user api.UserInfo) interface{} {
+	// TODO: Allow non-repo-admins to delete things
+
+	params := mux.Vars(r)
+
+	server := params["server"]
+	mediaId := params["mediaId"]
+
+	log = log.WithFields(logrus.Fields{
+		"server":  server,
+		"mediaId": mediaId,
+	})
+
+	err := maintenance_controller.PurgeMedia(server, mediaId, r.Context(), log)
+	if err != nil {
+		log.Error("Error purging media: " + err.Error())
+		return api.InternalServerError("error purging media")
+	}
+
+	return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}}
+}
+
+func PurgeQurantined(r *http.Request, log *logrus.Entry, user api.UserInfo) interface{} {
+	// TODO: Allow non-repo-admins to delete things
+
+	affected, err := maintenance_controller.PurgeQuarantined(r.Context(), log)
+	if err != nil {
+		log.Error("Error purging media: " + err.Error())
+		return api.InternalServerError("error purging media")
+	}
+
+	mxcs := make([]string, 0)
+	for _, a := range affected {
+		mxcs = append(mxcs, a.MxcUri())
+	}
+
+	return &api.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}}
+}

api/webserver/webserver.go

@@ -31,7 +31,9 @@ func Init() {
 	thumbnailHandler := handler{api.AccessTokenOptionalRoute(r0.ThumbnailMedia), "thumbnail", counter, false}
 	previewUrlHandler := handler{api.AccessTokenRequiredRoute(r0.PreviewUrl), "url_preview", counter, false}
 	identiconHandler := handler{api.AccessTokenOptionalRoute(r0.Identicon), "identicon", counter, false}
-	purgeHandler := handler{api.RepoAdminRoute(custom.PurgeRemoteMedia), "purge_remote_media", counter, false}
+	purgeRemote := handler{api.RepoAdminRoute(custom.PurgeRemoteMedia), "purge_remote_media", counter, false}
+	purgeOneHandler := handler{api.RepoAdminRoute(custom.PurgeIndividualRecord), "purge_individual_media", counter, false}
+	purgeQuarantinedHandler := handler{api.RepoAdminRoute(custom.PurgeQurantined), "purge_quarantined", counter, false}
 	quarantineHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineMedia), "quarantine_media", counter, false}
 	quarantineRoomHandler := handler{api.AccessTokenRequiredRoute(custom.QuarantineRoomMedia), "quarantine_room", counter, false}
 	localCopyHandler := handler{api.AccessTokenRequiredRoute(unstable.LocalCopy), "local_copy", counter, false}
@@ -63,7 +65,10 @@ func Init() {
 		routes["/_matrix/media/"+version+"/config"] = route{"GET", configHandler}
 
 		// Routes that we define but are not part of the spec (management)
-		routes["/_matrix/media/"+version+"/admin/purge_remote"] = route{"POST", purgeHandler}
+		routes["/_matrix/media/"+version+"/admin/purge_remote"] = route{"POST", purgeRemote}
+		routes["/_matrix/media/"+version+"/admin/purge/remote"] = route{"POST", purgeRemote}
+		routes["/_matrix/media/"+version+"/admin/purge/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"POST", purgeOneHandler}
+		routes["/_matrix/media/"+version+"/admin/purge/quarantined"] = route{"POST", purgeQuarantinedHandler}
 		routes["/_matrix/media/"+version+"/admin/quarantine/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}"] = route{"POST", quarantineHandler}
 		routes["/_matrix/media/"+version+"/admin/room/{roomId:[^/]+}/quarantine"] = route{"POST", quarantineRoomHandler}
 		routes["/_matrix/media/"+version+"/admin/datastores/{datastoreId:[^/]+}/size_estimate"] = route{"GET", storageEstimateHandler}
@@ -78,7 +83,7 @@ func Init() {
 		routes["/_matrix/media/"+version+"/admin/tasks/unfinished"] = route{"GET", listUnfinishedBackgroundTasksHandler}
 
 		// Routes that we should handle but aren't in the media namespace (synapse compat)
-		routes["/_matrix/client/"+version+"/admin/purge_media_cache"] = route{"POST", purgeHandler}
+		routes["/_matrix/client/"+version+"/admin/purge_media_cache"] = route{"POST", purgeRemote}
 		routes["/_matrix/client/"+version+"/admin/quarantine_media/{roomId:[^/]+}"] = route{"POST", quarantineRoomHandler}
 
 		if version == "unstable" {

controllers/maintenance_controller/maintainance_controller.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/sirupsen/logrus"
+	"github.com/turt2live/matrix-media-repo/controllers/download_controller"
 	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/storage/datastore"
 	"github.com/turt2live/matrix-media-repo/types"
@@ -19,7 +20,7 @@ func StartStorageMigration(sourceDs *datastore.DatastoreRef, targetDs *datastore
 	task, err := db.CreateBackgroundTask("storage_migration", map[string]interface{}{
 		"source_datastore_id": sourceDs.DatastoreId,
 		"target_datastore_id": targetDs.DatastoreId,
-		"before_ts": beforeTs,
+		"before_ts":           beforeTs,
 	})
 	if err != nil {
 		return nil, err
@@ -149,6 +150,7 @@ func EstimateDatastoreSizeWithAge(beforeTs int64, datastoreId string, ctx contex
 
 func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Entry) (int, error) {
 	db := storage.GetDatabase().GetMediaStore(ctx, log)
+	thumbsDb := storage.GetDatabase().GetThumbnailStore(ctx, log)
 
 	origins, err := db.GetOrigins()
 	if err != nil {
@@ -196,7 +198,101 @@ func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Ent
 		if err != nil {
 			log.Warn("Error removing media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
 		}
+
+		// Delete the thumbnails too
+		thumbs, err := thumbsDb.GetAllForMedia(media.Origin, media.MediaId)
+		if err != nil {
+			log.Warn("Error getting thumbnails for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+			continue
+		}
+		for _, thumb := range thumbs {
+			log.Info("Deleting thumbnail with hash: ", thumb.Sha256Hash)
+			ds, err := datastore.LocateDatastore(ctx, log, thumb.DatastoreId)
+			if err != nil {
+				log.Warn("Error removing thumbnail for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+				continue
+			}
+
+			err = ds.DeleteObject(thumb.Location)
+			if err != nil {
+				log.Warn("Error removing thumbnail for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+				continue
+			}
+		}
+		err = thumbsDb.DeleteAllForMedia(media.Origin, media.MediaId)
+		if err != nil {
+			log.Warn("Error removing thumbnails for media " + media.Origin + "/" + media.MediaId + " from database: " + err.Error())
+		}
 	}
 
 	return removed, nil
 }
+
+func PurgeQuarantined(ctx context.Context, log *logrus.Entry) ([]*types.Media, error) {
+	mediaDb := storage.GetDatabase().GetMediaStore(ctx, log)
+	records, err := mediaDb.GetAllQuarantinedMedia()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, r := range records {
+		err = doPurge(r, ctx, log)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return records, nil
+}
+
+func PurgeMedia(origin string, mediaId string, ctx context.Context, log *logrus.Entry) error {
+	media, err := download_controller.FindMediaRecord(origin, mediaId, false, ctx, log)
+	if err != nil {
+		return err
+	}
+
+	return doPurge(media, ctx, log)
+}
+
+func doPurge(media *types.Media, ctx context.Context, log *logrus.Entry) error {
+	// Delete all the thumbnails first
+	thumbsDb := storage.GetDatabase().GetThumbnailStore(ctx, log)
+	thumbs, err := thumbsDb.GetAllForMedia(media.Origin, media.MediaId)
+	if err != nil {
+		return err
+	}
+	for _, thumb := range thumbs {
+		log.Info("Deleting thumbnail with hash: ", thumb.Sha256Hash)
+		ds, err := datastore.LocateDatastore(ctx, log, thumb.DatastoreId)
+		if err != nil {
+			return err
+		}
+
+		err = ds.DeleteObject(thumb.Location)
+		if err != nil {
+			return err
+		}
+	}
+	err = thumbsDb.DeleteAllForMedia(media.Origin, media.MediaId)
+	if err != nil {
+		return err
+	}
+
+	ds, err := datastore.LocateDatastore(ctx, log, media.DatastoreId)
+	if err != nil {
+		return err
+	}
+
+	err = ds.DeleteObject(media.Location)
+	if err != nil {
+		return err
+	}
+
+	mediaDb := storage.GetDatabase().GetMediaStore(ctx, log)
+	err = mediaDb.Delete(media.Origin, media.MediaId)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

docs/admin.md

@@ -2,14 +2,30 @@
 
 All the API calls here require your user ID to be listed in the configuration as an administrator. After that, your access token for your homeserver will grant you access to these APIs. The URLs should be hit against a configured homeserver. For example, if you have `t2bot.io` configured as a homeserver, then the admin API can be used at `https://t2bot.io/_matrix/media/unstable/admin/...`.
 
-## Remote media purge
+## Media purge
 
-URL: `POST /_matrix/media/unstable/admin/purge_remote?before_ts=1234567890&access_token=your_access_token` (`before_ts` is in milliseconds)
+Sometimes you just want your disk space back - purging media is the best way to do that. **Be careful about what you're purging.** The media repo will happily purge a local media object, making it highly unlikely to ever exist in Matrix again. When the media repo deletes remote media, it is only deleting its copy of it - it cannot delete media on the remote server itself. Thumbnails will also be deleted for the media.
+
+#### Purge remote media
+
+URL: `POST /_matrix/media/unstable/admin/purge/remote?before_ts=1234567890&access_token=your_access_token` (`before_ts` is in milliseconds)
 
 This will delete remote media from the file store that was downloaded before the timestamp specified. If the file is referenced by newer remote media or local files to any of the configured homeservers, it will not be deleted. Be aware that removing a homeserver from the config will cause it to be considered a remote server, and therefore the media may be deleted.
 
 Any remote media that is deleted and requested by a user will be downloaded again.
 
+#### Purge quarantined media
+
+URL: `POST /_matrix/media/unstable/admin/purge/quarantined?access_token=your_access_token`
+
+This will delete all media that has previously been quarantined, local or remote.
+
+#### Purge individual record
+
+URL: `POST /_matrix/media/unstable/admin/purge/<server>/<media id>?access_token=your_access_token`
+
+This will delete the media record, regardless of it being local or remote.
+
 ## Quarantine media
 
 URL: `POST /_matrix/media/unstable/admin/quarantine/<server>/<media id>?access_token=your_access_token`

storage/stores/media_store.go

@@ -26,6 +26,7 @@ const selectAllDatastores = "SELECT datastore_id, ds_type, uri FROM datastores;"
 const selectAllMediaForServer = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1"
 const selectAllMediaForServerUsers = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND user_id = ANY($2)"
 const selectAllMediaForServerIds = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE origin = $1 AND media_id = ANY($2)"
+const selectQuarantinedMedia = "SELECT origin, media_id, upload_name, content_type, user_id, sha256_hash, size_bytes, datastore_id, location, creation_ts, quarantined FROM media WHERE quarantined = true;"
 
 var dsCacheByPath = sync.Map{} // [string] => Datastore
 var dsCacheById = sync.Map{}   // [string] => Datastore
@@ -48,6 +49,7 @@ type mediaStoreStatements struct {
 	selectAllMediaForServer         *sql.Stmt
 	selectAllMediaForServerUsers    *sql.Stmt
 	selectAllMediaForServerIds      *sql.Stmt
+	selectQuarantinedMedia          *sql.Stmt
 }
 
 type MediaStoreFactory struct {
@@ -116,6 +118,9 @@ func InitMediaStore(sqlDb *sql.DB) (*MediaStoreFactory, error) {
 	if store.stmts.selectAllMediaForServerIds, err = store.sqlDb.Prepare(selectAllMediaForServerIds); err != nil {
 		return nil, err
 	}
+	if store.stmts.selectQuarantinedMedia, err = store.sqlDb.Prepare(selectQuarantinedMedia); err != nil {
+		return nil, err
+	}
 
 	return &store, nil
 }
@@ -489,3 +494,34 @@ func (s *MediaStore) GetAllMediaInIds(serverName string, mediaIds []string) ([]*
 
 	return results, nil
 }
+
+func (s *MediaStore) GetAllQuarantinedMedia() ([]*types.Media, error) {
+	rows, err := s.statements.selectQuarantinedMedia.QueryContext(s.ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*types.Media
+	for rows.Next() {
+		obj := &types.Media{}
+		err = rows.Scan(
+			&obj.Origin,
+			&obj.MediaId,
+			&obj.UploadName,
+			&obj.ContentType,
+			&obj.UserId,
+			&obj.Sha256Hash,
+			&obj.SizeBytes,
+			&obj.DatastoreId,
+			&obj.Location,
+			&obj.CreationTs,
+			&obj.Quarantined,
+		)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, obj)
+	}
+
+	return results, nil
+}

storage/stores/thumbnail_store.go

@@ -14,6 +14,8 @@ const updateThumbnailHash = "UPDATE thumbnails SET sha256_hash = $7 WHERE origin
 const selectThumbnailsWithoutHash = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE sha256_hash IS NULL OR sha256_hash = '';"
 const selectThumbnailsWithoutDatastore = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE datastore_id IS NULL OR datastore_id = '';"
 const updateThumbnailDatastoreAndLocation = "UPDATE thumbnails SET location = $8, datastore_id = $7 WHERE origin = $1 and media_id = $2 and width = $3 and height = $4 and method = $5 and animated = $6;"
+const selectThumbnailsForMedia = "SELECT origin, media_id, width, height, method, animated, content_type, size_bytes, datastore_id, location, creation_ts, sha256_hash FROM thumbnails WHERE origin = $1 AND media_id = $2;"
+const deleteThumbnailsForMedia = "DELETE FROM thumbnails WHERE origin = $1 AND media_id = $2;"
 
 type thumbnailStatements struct {
 	selectThumbnail                     *sql.Stmt
@@ -22,6 +24,8 @@ type thumbnailStatements struct {
 	selectThumbnailsWithoutHash         *sql.Stmt
 	selectThumbnailsWithoutDatastore    *sql.Stmt
 	updateThumbnailDatastoreAndLocation *sql.Stmt
+	selectThumbnailsForMedia            *sql.Stmt
+	deleteThumbnailsForMedia            *sql.Stmt
 }
 
 type ThumbnailStoreFactory struct {
@@ -60,6 +64,12 @@ func InitThumbnailStore(sqlDb *sql.DB) (*ThumbnailStoreFactory, error) {
 	if store.stmts.updateThumbnailDatastoreAndLocation, err = store.sqlDb.Prepare(updateThumbnailDatastoreAndLocation); err != nil {
 		return nil, err
 	}
+	if store.stmts.selectThumbnailsForMedia, err = store.sqlDb.Prepare(selectThumbnailsForMedia); err != nil {
+		return nil, err
+	}
+	if store.stmts.deleteThumbnailsForMedia, err = store.sqlDb.Prepare(deleteThumbnailsForMedia); err != nil {
+		return nil, err
+	}
 
 	return &store, nil
 }
@@ -206,3 +216,43 @@ func (s *ThumbnailStore) GetAllWithoutDatastore() ([]*types.Thumbnail, error) {
 
 	return results, nil
 }
+
+func (s *ThumbnailStore) GetAllForMedia(origin string, mediaId string) ([]*types.Thumbnail, error) {
+	rows, err := s.statements.selectThumbnailsForMedia.QueryContext(s.ctx, origin, mediaId)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*types.Thumbnail
+	for rows.Next() {
+		obj := &types.Thumbnail{}
+		err = rows.Scan(
+			&obj.Origin,
+			&obj.MediaId,
+			&obj.Width,
+			&obj.Height,
+			&obj.Method,
+			&obj.Animated,
+			&obj.ContentType,
+			&obj.SizeBytes,
+			&obj.DatastoreId,
+			&obj.Location,
+			&obj.CreationTs,
+			&obj.Sha256Hash,
+		)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, obj)
+	}
+
+	return results, nil
+}
+
+func (s *ThumbnailStore) DeleteAllForMedia(origin string, mediaId string) error {
+	_, err := s.statements.deleteThumbnailsForMedia.ExecContext(s.ctx, origin, mediaId)
+	if err != nil {
+		return err
+	}
+	return nil
+}