Expire entries from the Media ID hold table

Fixes https://github.com/turt2live/matrix-media-repo/issues/426

database/table_media_hold.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/util"
 )
 
 type DbHeldMedia struct {
@@ -19,10 +20,12 @@ const (
 	ForCreateHeldReason HeldReason = "media_create"
 )
 
-const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason) VALUES ($1, $2, $3);"
+const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason, held_ts) VALUES ($1, $2, $3, $4);"
+const deleteHeldMedia = "DELETE FROM media_id_hold WHERE reason = $1 AND held_ts <= $2;"
 
 type heldMediaTableStatements struct {
 	insertHeldMedia *sql.Stmt
+	deleteHeldMedia *sql.Stmt
 }
 
 type heldMediaTableWithContext struct {
@@ -37,6 +40,9 @@ func prepareHeldMediaTables(db *sql.DB) (*heldMediaTableStatements, error) {
 	if stmts.insertHeldMedia, err = db.Prepare(insertHeldMedia); err != nil {
 		return nil, errors.New("error preparing insertHeldMedia: " + err.Error())
 	}
+	if stmts.deleteHeldMedia, err = db.Prepare(deleteHeldMedia); err != nil {
+		return nil, errors.New("error preparing deleteHeldMedia: " + err.Error())
+	}
 
 	return stmts, nil
 }
@@ -49,6 +55,11 @@ func (s *heldMediaTableStatements) Prepare(ctx rcontext.RequestContext) *heldMed
 }
 
 func (s *heldMediaTableWithContext) TryInsert(origin string, mediaId string, reason HeldReason) error {
-	_, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason)
+	_, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, util.NowMillis())
+	return err
+}
+
+func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThanTs int64) error {
+	_, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThanTs)
 	return err
 }

migrations/24_add_timestamp_to_media_id_hold_down.sql

+ALTER TABLE media_id_hold DROP COLUMN held_ts;

migrations/24_add_timestamp_to_media_id_hold_up.sql

+ALTER TABLE media_id_hold ADD COLUMN held_ts BIGINT NOT NULL DEFAULT 0;

tasks/all.go

@@ -10,6 +10,7 @@ func StartAll() {
 	scheduleHourly(RecurringTaskPurgeRemoteMedia, task_runner.PurgeRemoteMedia)
 	scheduleHourly(RecurringTaskPurgeThumbnails, task_runner.PurgeThumbnails)
 	scheduleHourly(RecurringTaskPurgePreviews, task_runner.PurgePreviews)
+	scheduleHourly(RecurringTaskPurgeHeldMediaIds, task_runner.PurgeHeldMediaIds)
 
 	scheduleUnfinished()
 }

tasks/schedule.go

@@ -24,9 +24,10 @@ const (
 	TaskImportData       TaskName = "import_data"
 )
 const (
-	RecurringTaskPurgeThumbnails  RecurringTaskName = "recurring_purge_thumbnails"
-	RecurringTaskPurgePreviews    RecurringTaskName = "recurring_purge_previews"
-	RecurringTaskPurgeRemoteMedia RecurringTaskName = "recurring_purge_remote_media"
+	RecurringTaskPurgeThumbnails   RecurringTaskName = "recurring_purge_thumbnails"
+	RecurringTaskPurgePreviews     RecurringTaskName = "recurring_purge_previews"
+	RecurringTaskPurgeRemoteMedia  RecurringTaskName = "recurring_purge_remote_media"
+	RecurringTaskPurgeHeldMediaIds RecurringTaskName = "recurring_purge_held_media_ids"
 )
 
 const ExecutingMachineId = int64(0)

tasks/task_runner/purge_held_media_ids.go

+package task_runner
+
+import (
+	"github.com/getsentry/sentry-go"
+	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/database"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+func PurgeHeldMediaIds(ctx rcontext.RequestContext) {
+	// dev note: don't use ctx for config lookup to avoid misreading it
+
+	beforeTs := util.NowMillis() - int64(7*24*60*60*1000) // 7 days
+	db := database.GetInstance().HeldMedia.Prepare(ctx)
+
+	if err := db.DeleteOlderThan(database.ForCreateHeldReason, beforeTs); err != nil {
+		ctx.Log.Error("Error deleting held media IDs: ", err)
+		sentry.CaptureException(err)
+	}
+}