From 5309d9a8dc9fcbc56ed3e853adddb5ef6cd36380 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 15 Aug 2023 16:55:36 -0600
Subject: [PATCH] Expire entries from the Media ID hold table
Fixes https://github.com/turt2live/matrix-media-repo/issues/426
---
database/table_media_hold.go | 15 ++++++++++++--
...24_add_timestamp_to_media_id_hold_down.sql | 1 +
.../24_add_timestamp_to_media_id_hold_up.sql | 1 +
tasks/all.go | 1 +
tasks/schedule.go | 7 ++++---
tasks/task_runner/purge_held_media_ids.go | 20 +++++++++++++++++++
6 files changed, 40 insertions(+), 5 deletions(-)
create mode 100644 migrations/24_add_timestamp_to_media_id_hold_down.sql
create mode 100644 migrations/24_add_timestamp_to_media_id_hold_up.sql
create mode 100644 tasks/task_runner/purge_held_media_ids.go
diff --git a/database/table_media_hold.go b/database/table_media_hold.go
index 0911f957..da05b42e 100644
--- a/database/table_media_hold.go
+++ b/database/table_media_hold.go
@@ -5,6 +5,7 @@ import (
"errors"
"github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/util"
)
type DbHeldMedia struct {
@@ -19,10 +20,12 @@ const (
ForCreateHeldReason HeldReason = "media_create"
)
-const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason) VALUES ($1, $2, $3);"
+const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason, held_ts) VALUES ($1, $2, $3, $4);"
+const deleteHeldMedia = "DELETE FROM media_id_hold WHERE reason = $1 AND held_ts <= $2;"
type heldMediaTableStatements struct {
insertHeldMedia *sql.Stmt
+ deleteHeldMedia *sql.Stmt
}
type heldMediaTableWithContext struct {
@@ -37,6 +40,9 @@ func prepareHeldMediaTables(db *sql.DB) (*heldMediaTableStatements, error) {
if stmts.insertHeldMedia, err = db.Prepare(insertHeldMedia); err != nil {
return nil, errors.New("error preparing insertHeldMedia: " + err.Error())
}
+ if stmts.deleteHeldMedia, err = db.Prepare(deleteHeldMedia); err != nil {
+ return nil, errors.New("error preparing deleteHeldMedia: " + err.Error())
+ }
return stmts, nil
}
@@ -49,6 +55,11 @@ func (s *heldMediaTableStatements) Prepare(ctx rcontext.RequestContext) *heldMed
}
func (s *heldMediaTableWithContext) TryInsert(origin string, mediaId string, reason HeldReason) error {
- _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason)
+ _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, util.NowMillis())
+ return err
+}
+
+func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThanTs int64) error {
+ _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThanTs)
return err
}
diff --git a/migrations/24_add_timestamp_to_media_id_hold_down.sql b/migrations/24_add_timestamp_to_media_id_hold_down.sql
new file mode 100644
index 00000000..53a0631f
--- /dev/null
+++ b/migrations/24_add_timestamp_to_media_id_hold_down.sql
@@ -0,0 +1 @@
+ALTER TABLE media_id_hold DROP COLUMN held_ts;
diff --git a/migrations/24_add_timestamp_to_media_id_hold_up.sql b/migrations/24_add_timestamp_to_media_id_hold_up.sql
new file mode 100644
index 00000000..b4b3ac2f
--- /dev/null
+++ b/migrations/24_add_timestamp_to_media_id_hold_up.sql
@@ -0,0 +1 @@
+ALTER TABLE media_id_hold ADD COLUMN held_ts BIGINT NOT NULL DEFAULT 0;
diff --git a/tasks/all.go b/tasks/all.go
index a4eb7e01..edbeb6d7 100644
--- a/tasks/all.go
+++ b/tasks/all.go
@@ -10,6 +10,7 @@ func StartAll() {
scheduleHourly(RecurringTaskPurgeRemoteMedia, task_runner.PurgeRemoteMedia)
scheduleHourly(RecurringTaskPurgeThumbnails, task_runner.PurgeThumbnails)
scheduleHourly(RecurringTaskPurgePreviews, task_runner.PurgePreviews)
+ scheduleHourly(RecurringTaskPurgeHeldMediaIds, task_runner.PurgeHeldMediaIds)
scheduleUnfinished()
}
diff --git a/tasks/schedule.go b/tasks/schedule.go
index 576c33b0..aeed88b7 100644
--- a/tasks/schedule.go
+++ b/tasks/schedule.go
@@ -24,9 +24,10 @@ const (
TaskImportData TaskName = "import_data"
)
const (
- RecurringTaskPurgeThumbnails RecurringTaskName = "recurring_purge_thumbnails"
- RecurringTaskPurgePreviews RecurringTaskName = "recurring_purge_previews"
- RecurringTaskPurgeRemoteMedia RecurringTaskName = "recurring_purge_remote_media"
+ RecurringTaskPurgeThumbnails RecurringTaskName = "recurring_purge_thumbnails"
+ RecurringTaskPurgePreviews RecurringTaskName = "recurring_purge_previews"
+ RecurringTaskPurgeRemoteMedia RecurringTaskName = "recurring_purge_remote_media"
+ RecurringTaskPurgeHeldMediaIds RecurringTaskName = "recurring_purge_held_media_ids"
)
const ExecutingMachineId = int64(0)
diff --git a/tasks/task_runner/purge_held_media_ids.go b/tasks/task_runner/purge_held_media_ids.go
new file mode 100644
index 00000000..b0793571
--- /dev/null
+++ b/tasks/task_runner/purge_held_media_ids.go
@@ -0,0 +1,20 @@
+package task_runner
+
+import (
+ "github.com/getsentry/sentry-go"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/database"
+ "github.com/turt2live/matrix-media-repo/util"
+)
+
+func PurgeHeldMediaIds(ctx rcontext.RequestContext) {
+ // dev note: don't use ctx for config lookup to avoid misreading it
+
+ beforeTs := util.NowMillis() - int64(7*24*60*60*1000) // 7 days
+ db := database.GetInstance().HeldMedia.Prepare(ctx)
+
+ if err := db.DeleteOlderThan(database.ForCreateHeldReason, beforeTs); err != nil {
+ ctx.Log.Error("Error deleting held media IDs: ", err)
+ sentry.CaptureException(err)
+ }
+}
--
GitLab