diff --git a/api/custom/datastores.go b/api/custom/datastores.go
index 52425fbadb6f01a8c0411660af1f17918a57e429..19119858739aa96441c109c272f719dee01146f2 100644
--- a/api/custom/datastores.go
+++ b/api/custom/datastores.go
@@ -15,12 +15,11 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
"github.com/turt2live/matrix-media-repo/storage/datastore"
- "github.com/turt2live/matrix-media-repo/types"
"github.com/turt2live/matrix-media-repo/util"
)
type DatastoreMigration struct {
- *types.DatastoreMigrationEstimate
+ *datastores.SizeEstimate
TaskID int `json:"task_id"`
}
@@ -78,24 +77,24 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use
return _responses.BadRequest("Error getting target datastore. Does it exist?")
}
- rctx.Log.Info("User ", user.UserId, " has started a datastore media transfer")
- task, err := maintenance_controller.StartStorageMigration(sourceDatastore, targetDatastore, beforeTs, rctx)
+ estimate, err := datastores.SizeOfDsIdWithAge(rctx, sourceDsId, beforeTs)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return _responses.InternalServerError("Unexpected error starting migration")
+ return _responses.InternalServerError("Unexpected error getting storage estimate")
}
- estimate, err := maintenance_controller.EstimateDatastoreSizeWithAge(beforeTs, sourceDsId, rctx)
+ rctx.Log.Infof("User %s has started a datastore media transfer", user.UserId)
+ task, err := maintenance_controller.StartStorageMigration(sourceDatastore, targetDatastore, beforeTs, rctx)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
- return _responses.InternalServerError("Unexpected error getting storage estimate")
+ return _responses.InternalServerError("Unexpected error starting migration")
}
migration := &DatastoreMigration{
- DatastoreMigrationEstimate: estimate,
- TaskID: task.ID,
+ SizeEstimate: estimate,
+ TaskID: task.ID,
}
return &_responses.DoNotCacheResponse{Payload: migration}
@@ -119,7 +118,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext,
"datastoreId": datastoreId,
})
- result, err := maintenance_controller.EstimateDatastoreSizeWithAge(beforeTs, datastoreId, rctx)
+ result, err := datastores.SizeOfDsIdWithAge(rctx, datastoreId, beforeTs)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
diff --git a/database/virtualtable_metadata.go b/database/virtualtable_metadata.go
index 3a499da1504c28bd22cd520c76f6c68ebb5c4ca1..6ad1dcb449ea6a5c3fe9d1093b6365ad60f95c78 100644
--- a/database/virtualtable_metadata.go
+++ b/database/virtualtable_metadata.go
@@ -9,9 +9,18 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
)
+type VirtLastAccess struct {
+ *Locatable
+ SizeBytes int64
+ CreationTs int64
+ LastAccessTs int64
+}
+
const selectEstimatedDatastoreSize = "SELECT COALESCE(SUM(m2.size_bytes), 0) + COALESCE((SELECT SUM(t2.size_bytes) FROM (SELECT DISTINCT t.sha256_hash, MAX(t.size_bytes) AS size_bytes FROM thumbnails AS t WHERE t.datastore_id = $1 GROUP BY t.sha256_hash) AS t2), 0) AS size_total FROM (SELECT DISTINCT m.sha256_hash, MAX(m.size_bytes) AS size_bytes FROM media AS m WHERE m.datastore_id = $1 GROUP BY m.sha256_hash) AS m2;"
const selectUploadSizesForServer = "SELECT COALESCE((SELECT SUM(size_bytes) FROM media WHERE origin = $1), 0) AS media, COALESCE((SELECT SUM(size_bytes) FROM thumbnails WHERE origin = $1), 0) AS thumbnails;"
const selectUploadCountsForServer = "SELECT COALESCE((SELECT COUNT(origin) FROM media WHERE origin = $1), 0) AS media, COALESCE((SELECT COUNT(origin) FROM thumbnails WHERE origin = $1), 0) AS thumbnails;"
+const selectMediaForDatastoreWithLastAccess = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM media AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1 AND m.datastore_id = $2;"
+const selectThumbnailsForDatastoreWithLastAccess = "SELECT m.sha256_hash, m.size_bytes, m.datastore_id, m.location, m.creation_ts, a.last_access_ts FROM thumbnails AS m JOIN last_access AS a ON m.sha256_hash = a.sha256_hash WHERE a.last_access_ts < $1 AND m.datastore_id = $2;"
type SynStatUserOrderBy string
@@ -36,9 +45,11 @@ type DbSynUserStat struct {
type metadataVirtualTableStatements struct {
db *sql.DB
- selectEstimatedDatastoreSize *sql.Stmt
- selectUploadSizesForServer *sql.Stmt
- selectUploadCountsForServer *sql.Stmt
+ selectEstimatedDatastoreSize *sql.Stmt
+ selectUploadSizesForServer *sql.Stmt
+ selectUploadCountsForServer *sql.Stmt
+ selectMediaForDatastoreWithLastAccess *sql.Stmt
+ selectThumbnailsForDatastoreWithLastAccess *sql.Stmt
}
type metadataVirtualTableWithContext struct {
@@ -61,6 +72,12 @@ func prepareMetadataVirtualTables(db *sql.DB) (*metadataVirtualTableStatements,
if stmts.selectUploadCountsForServer, err = db.Prepare(selectUploadCountsForServer); err != nil {
return nil, errors.New("error preparing selectUploadCountsForServer: " + err.Error())
}
+ if stmts.selectMediaForDatastoreWithLastAccess, err = db.Prepare(selectMediaForDatastoreWithLastAccess); err != nil {
+ return nil, errors.New("error preparing selectMediaForDatastoreWithLastAccess: " + err.Error())
+ }
+ if stmts.selectThumbnailsForDatastoreWithLastAccess, err = db.Prepare(selectThumbnailsForDatastoreWithLastAccess); err != nil {
+ return nil, errors.New("error preparing selectThumbnailsForDatastoreWithLastAccess: " + err.Error())
+ }
return stmts, nil
}
@@ -183,3 +200,30 @@ func (s *metadataVirtualTableWithContext) UnoptimizedSynapseUserStatsPage(server
return results, total, nil
}
+
+func (s *metadataVirtualTableWithContext) scanLastAccess(rows *sql.Rows, err error) ([]*VirtLastAccess, error) {
+ results := make([]*VirtLastAccess, 0)
+ if err != nil {
+ if err == sql.ErrNoRows {
+ return results, nil
+ }
+ return nil, err
+ }
+ for rows.Next() {
+ val := &VirtLastAccess{Locatable: &Locatable{}}
+ if err = rows.Scan(&val.Sha256Hash, &val.SizeBytes, &val.DatastoreId, &val.Location, &val.CreationTs, &val.LastAccessTs); err != nil {
+ return nil, err
+ }
+ results = append(results, val)
+ }
+
+ return results, nil
+}
+
+func (s *metadataVirtualTableWithContext) GetMediaForDatastoreByLastAccess(datastoreId string, lastAccessTs int64) ([]*VirtLastAccess, error) {
+ return s.scanLastAccess(s.statements.selectMediaForDatastoreWithLastAccess.QueryContext(s.ctx, lastAccessTs, datastoreId))
+}
+
+func (s *metadataVirtualTableWithContext) GetThumbnailsForDatastoreByLastAccess(datastoreId string, lastAccessTs int64) ([]*VirtLastAccess, error) {
+ return s.scanLastAccess(s.statements.selectThumbnailsForDatastoreWithLastAccess.QueryContext(s.ctx, lastAccessTs, datastoreId))
+}
diff --git a/datastores/info.go b/datastores/info.go
index 34549cd04f61d941ecbca84b98269c0fc43a6306..bed33a1c7de1b7c8dca30fc2613209e17b74ce16 100644
--- a/datastores/info.go
+++ b/datastores/info.go
@@ -5,8 +5,23 @@ import (
"fmt"
"github.com/turt2live/matrix-media-repo/common/config"
+ "github.com/turt2live/matrix-media-repo/common/rcontext"
+ "github.com/turt2live/matrix-media-repo/database"
)
+type SizeEstimate struct {
+ ThumbnailsAffected int64 `json:"thumbnails_affected"`
+ ThumbnailHashesAffected int64 `json:"thumbnail_hashes_affected"`
+ ThumbnailBytes int64 `json:"thumbnail_bytes"`
+
+ MediaAffected int64 `json:"media_affected"`
+ MediaHashesAffected int64 `json:"media_hashes_affected"`
+ MediaBytes int64 `json:"media_bytes"`
+
+ TotalHashesAffected int64 `json:"total_hashes_affected"`
+ TotalBytes int64 `json:"total_bytes"`
+}
+
func GetUri(ds config.DatastoreConfig) (string, error) {
if ds.Type == "s3" {
s3c, err := getS3(ds)
@@ -20,3 +35,53 @@ func GetUri(ds config.DatastoreConfig) (string, error) {
return "", errors.New("unknown datastore type - contact developer")
}
}
+
+func SizeOfDsIdWithAge(ctx rcontext.RequestContext, dsId string, beforeTs int64) (*SizeEstimate, error) {
+ db := database.GetInstance().MetadataView.Prepare(ctx)
+ media, err := db.GetMediaForDatastoreByLastAccess(dsId, beforeTs)
+ if err != nil {
+ return nil, err
+ }
+ thumbs, err := db.GetThumbnailsForDatastoreByLastAccess(dsId, beforeTs)
+ if err != nil {
+ return nil, err
+ }
+
+ estimate := &SizeEstimate{}
+ seenHashes := make(map[string]bool)
+ seenMediaHashes := make(map[string]bool)
+ seenThumbnailHashes := make(map[string]bool)
+
+ for _, record := range media {
+ estimate.MediaAffected++
+
+ if _, found := seenHashes[record.Sha256Hash]; !found {
+ estimate.TotalBytes += record.SizeBytes
+ estimate.TotalHashesAffected++
+ }
+ if _, found := seenMediaHashes[record.Sha256Hash]; !found {
+ estimate.MediaBytes += record.SizeBytes
+ estimate.MediaHashesAffected++
+ }
+
+ seenHashes[record.Sha256Hash] = true
+ seenMediaHashes[record.Sha256Hash] = true
+ }
+ for _, record := range thumbs {
+ estimate.ThumbnailsAffected++
+
+ if _, found := seenHashes[record.Sha256Hash]; !found {
+ estimate.TotalBytes += record.SizeBytes
+ estimate.TotalHashesAffected++
+ }
+ if _, found := seenThumbnailHashes[record.Sha256Hash]; !found {
+ estimate.ThumbnailBytes += record.SizeBytes
+ estimate.ThumbnailHashesAffected++
+ }
+
+ seenHashes[record.Sha256Hash] = true
+ seenThumbnailHashes[record.Sha256Hash] = true
+ }
+
+ return estimate, nil
+}