Allow skipping user check in upload_controller.StoreDirect

The new parameter `filterUserDuplicates` is set to true
everywhere else except in the import_synapse call. This
ensures direct duplicates from the same user will get
stored in an import from Synapse situation when migrating
to the mediarepo.

Closes #241

cmd/import_synapse/main.go

@@ -156,7 +156,7 @@ func fetchMedia(req interface{}) interface{} {
 	}
 	defer cleanup.DumpAndCloseStream(body)
 
-	_, err = upload_controller.StoreDirect(nil, body, -1, record.ContentType, record.UploadName, record.UserId, payload.serverName, record.MediaId, common.KindLocalMedia, ctx)
+	_, err = upload_controller.StoreDirect(nil, body, -1, record.ContentType, record.UploadName, record.UserId, payload.serverName, record.MediaId, common.KindLocalMedia, ctx, false)
 	if err != nil {
 		logrus.Error(err.Error())
 		return nil

controllers/data_controller/import_controller.go

@@ -229,7 +229,7 @@ func doImport(updateChannel chan *importUpdate, taskId int, importId string, ctx
 			if found {
 				ctx.Log.Info("Using file from memory")
 				closer := util.BufferToStream(buf)
-				_, err := upload_controller.StoreDirect(nil, closer, record.SizeBytes, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx)
+				_, err := upload_controller.StoreDirect(nil, closer, record.SizeBytes, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx, true)
 				if err != nil {
 					ctx.Log.Errorf("Error importing file: %s", err.Error())
 					continue
@@ -312,7 +312,7 @@ func doImport(updateChannel chan *importUpdate, taskId int, importId string, ctx
 						continue
 					}
 
-					_, err = upload_controller.StoreDirect(nil, r.Body, r.ContentLength, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx)
+					_, err = upload_controller.StoreDirect(nil, r.Body, r.ContentLength, record.ContentType, record.FileName, userId, record.Origin, record.MediaId, kind, ctx, true)
 					if err != nil {
 						ctx.Log.Errorf("Error importing file: %s", err.Error())
 						continue

controllers/download_controller/download_resource_handler.go

@@ -140,7 +140,7 @@ func downloadResourceWorkFn(request *resource_handler.WorkRequest) interface{} {
 	persistFile := func(fileStream io.ReadCloser) *workerDownloadResponse {
 		defer cleanup.DumpAndCloseStream(fileStream)
 		userId := upload_controller.NoApplicableUploadUser
-		media, err := upload_controller.StoreDirect(nil, fileStream, downloaded.ContentLength, downloaded.ContentType, downloaded.DesiredFilename, userId, info.origin, info.mediaId, common.KindRemoteMedia, ctx)
+		media, err := upload_controller.StoreDirect(nil, fileStream, downloaded.ContentLength, downloaded.ContentType, downloaded.DesiredFilename, userId, info.origin, info.mediaId, common.KindRemoteMedia, ctx, true)
 		if err != nil {
 			ctx.Log.Error("Error persisting file: ", err)
 			return &workerDownloadResponse{err: err}

controllers/upload_controller/upload_controller.go

@@ -149,7 +149,7 @@ func UploadMedia(contents io.ReadCloser, contentLength int64, contentType string
 		mediaId = fmt.Sprintf("ipfs:%s", info.Location[len("ipfs/"):])
 	}
 
-	return StoreDirect(existingFile, data, contentLength, contentType, filename, userId, origin, mediaId, common.KindLocalMedia, ctx)
+	return StoreDirect(existingFile, data, contentLength, contentType, filename, userId, origin, mediaId, common.KindLocalMedia, ctx, true)
 }
 
 func trackUploadAsLastAccess(ctx rcontext.RequestContext, media *types.Media) {
@@ -202,7 +202,7 @@ func IsAllowed(contentType string, reportedContentType string, userId string, ct
 	return allowed
 }
 
-func StoreDirect(f *AlreadyUploadedFile, contents io.ReadCloser, expectedSize int64, contentType string, filename string, userId string, origin string, mediaId string, kind string, ctx rcontext.RequestContext) (*types.Media, error) {
+func StoreDirect(f *AlreadyUploadedFile, contents io.ReadCloser, expectedSize int64, contentType string, filename string, userId string, origin string, mediaId string, kind string, ctx rcontext.RequestContext, filterUserDuplicates bool) (*types.Media, error) {
 	var ds *datastore.DatastoreRef
 	var info *types.ObjectInfo
 	if f == nil {
@@ -255,7 +255,7 @@ func StoreDirect(f *AlreadyUploadedFile, contents io.ReadCloser, expectedSize in
 		// If the user is a real user (ie: actually uploaded media), then we'll see if there's
 		// an exact duplicate that we can return. Otherwise we'll just pick the first record and
 		// clone that.
-		if userId != NoApplicableUploadUser {
+		if filterUserDuplicates && userId != NoApplicableUploadUser {
 			for _, record := range records {
 				if record.Quarantined {
 					ctx.Log.Warn("User attempted to upload quarantined content - rejecting")