From 438ac23e2a3564fbce50e862336db56bbf0d747c Mon Sep 17 00:00:00 2001
From: Morris Jobke <hey@morrisjobke.de>
Date: Thu, 30 Jul 2020 22:09:19 +0200
Subject: [PATCH] Distribute preview folders in appdata in multibucket setup to
multiple buckets
* introduces a new IRootMountProvider to register mount points inside the root storage
* adds a AppdataPreviewObjectStoreStorage to handle the split between preview folders and bucket number
Ref #22033
Signed-off-by: Morris Jobke <hey@morrisjobke.de>
---
lib/composer/composer/autoload_classmap.php | 3 +
lib/composer/composer/autoload_static.php | 3 +
.../Files/Config/MountProviderCollection.php | 19 ++++
.../ObjectStorePreviewCacheMountProvider.php | 102 ++++++++++++++++++
.../AppdataPreviewObjectStoreStorage.php | 44 ++++++++
lib/private/Preview/Storage/Root.php | 1 -
lib/private/Server.php | 3 +
lib/private/legacy/OC_Util.php | 11 ++
.../Files/Config/IRootMountProvider.php | 41 +++++++
9 files changed, 226 insertions(+), 1 deletion(-)
create mode 100644 lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
create mode 100644 lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php
create mode 100644 lib/public/Files/Config/IRootMountProvider.php
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 254840b3542..65542e477e7 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -230,6 +230,7 @@ return array(
'OCP\\Files\\Config\\IHomeMountProvider' => $baseDir . '/lib/public/Files/Config/IHomeMountProvider.php',
'OCP\\Files\\Config\\IMountProvider' => $baseDir . '/lib/public/Files/Config/IMountProvider.php',
'OCP\\Files\\Config\\IMountProviderCollection' => $baseDir . '/lib/public/Files/Config/IMountProviderCollection.php',
+ 'OCP\\Files\\Config\\IRootMountProvider' => $baseDir . '/lib/public/Files/Config/IRootMountProvider.php',
'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php',
@@ -1029,6 +1030,7 @@ return array(
'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php',
'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php',
'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
+ 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php',
'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php',
'OC\\Files\\Node\\HookConnector' => $baseDir . '/lib/private/Files/Node/HookConnector.php',
@@ -1040,6 +1042,7 @@ return array(
'OC\\Files\\Node\\Root' => $baseDir . '/lib/private/Files/Node/Root.php',
'OC\\Files\\Notify\\Change' => $baseDir . '/lib/private/Files/Notify/Change.php',
'OC\\Files\\Notify\\RenameChange' => $baseDir . '/lib/private/Files/Notify/RenameChange.php',
+ 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 45bf93241d5..835603a3340 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -259,6 +259,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OCP\\Files\\Config\\IHomeMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IHomeMountProvider.php',
'OCP\\Files\\Config\\IMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProvider.php',
'OCP\\Files\\Config\\IMountProviderCollection' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProviderCollection.php',
+ 'OCP\\Files\\Config\\IRootMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IRootMountProvider.php',
'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php',
@@ -1058,6 +1059,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php',
'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php',
'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
+ 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php',
'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php',
'OC\\Files\\Node\\HookConnector' => __DIR__ . '/../../..' . '/lib/private/Files/Node/HookConnector.php',
@@ -1069,6 +1071,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\Node\\Root' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Root.php',
'OC\\Files\\Notify\\Change' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/Change.php',
'OC\\Files\\Notify\\RenameChange' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/RenameChange.php',
+ 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php
index 34db652290f..2b57ffe6e4c 100644
--- a/lib/private/Files/Config/MountProviderCollection.php
+++ b/lib/private/Files/Config/MountProviderCollection.php
@@ -30,6 +30,7 @@ use OC\Hooks\EmitterTrait;
use OCP\Files\Config\IHomeMountProvider;
use OCP\Files\Config\IMountProvider;
use OCP\Files\Config\IMountProviderCollection;
+use OCP\Files\Config\IRootMountProvider;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\Mount\IMountManager;
use OCP\Files\Mount\IMountPoint;
@@ -49,6 +50,9 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
*/
private $providers = [];
+ /** @var \OCP\Files\Config\IRootMountProvider[] */
+ private $rootProviders = [];
+
/**
* @var \OCP\Files\Storage\IStorageFactory
*/
@@ -198,4 +202,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
public function getMountCache() {
return $this->mountCache;
}
+
+ public function registerRootProvider(IRootMountProvider $provider) {
+ $this->rootProviders[] = $provider;
+ }
+
+ public function getRootMounts(): array {
+ $loader = $this->loader;
+ $mounts = array_map(function (IRootMountProvider $provider) use ($loader) {
+ return $provider->getRootMounts($loader);
+ }, $this->rootProviders);
+ $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) {
+ return array_merge($mounts, $providerMounts);
+ }, []);
+ return $mounts;
+ }
}
diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
new file mode 100644
index 00000000000..a4acdb6bb0f
--- /dev/null
+++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
@@ -0,0 +1,102 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Mount;
+
+use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage;
+use OCP\Files\Config\IRootMountProvider;
+use OCP\Files\Storage\IStorageFactory;
+use OCP\IConfig;
+use OCP\ILogger;
+
+/**
+ * Mount provider for object store app data folder for previews
+ */
+class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
+ /** @var ILogger */
+ private $logger;
+ /** @var IConfig */
+ private $config;
+
+ public function __construct(ILogger $logger, IConfig $config) {
+ $this->logger = $logger;
+ $this->config = $config;
+ }
+
+ public function getRootMounts(IStorageFactory $loader): array {
+ if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) {
+ return [];
+ }
+
+ $instanceId = $this->config->getSystemValueString('instanceid', '');
+ $mountPoints = [];
+ $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
+ $i = 0;
+ foreach ($directoryRange as $parent) {
+ foreach ($directoryRange as $child) {
+ $mountPoints[] = new MountPoint(
+ AppdataPreviewObjectStoreStorage::class,
+ '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child,
+ $this->getMultiBucketObjectStore($i),
+ $loader
+ );
+ $i++;
+ }
+ }
+ return $mountPoints;
+ }
+
+ /**
+ * @return array
+ */
+ protected function getMultiBucketObjectStore(int $number): array {
+ $config = $this->config->getSystemValue('objectstore_multibucket');
+
+ // sanity checks
+ if (empty($config['class'])) {
+ $this->logger->error('No class given for objectstore', ['app' => 'files']);
+ }
+ if (!isset($config['arguments'])) {
+ $config['arguments'] = [];
+ }
+
+ /*
+ * Use any provided bucket argument as prefix
+ * and add the mapping from parent/child => bucket
+ */
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+ $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64;
+ $config['arguments']['bucket'] .= (string)($number % $numBuckets);
+
+ // instantiate object store implementation
+ $config['arguments']['objectstore'] = new $config['class']($config['arguments']);
+
+ $config['arguments']['internal-id'] = $number;
+
+ return $config['arguments'];
+ }
+}
diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php
new file mode 100644
index 00000000000..ab6cb07cdc3
--- /dev/null
+++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php
@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage {
+
+ /** @var string */
+ private $internalId;
+
+ public function __construct($params) {
+ if (!isset($params['internal-id'])) {
+ throw new \Exception('missing id in parameters');
+ }
+ $this->internalId = (string)$params['internal-id'];
+ parent::__construct($params);
+ }
+
+ public function getId() {
+ return 'object::appdata::preview:' . $this->internalId;
+ }
+}
diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php
index a9a72026a51..107d87c6301 100644
--- a/lib/private/Preview/Storage/Root.php
+++ b/lib/private/Preview/Storage/Root.php
@@ -1,7 +1,6 @@
<?php
declare(strict_types=1);
-
/**
* @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
*
diff --git a/lib/private/Server.php b/lib/private/Server.php
index e054d8230d9..8d771bec3a1 100644
--- a/lib/private/Server.php
+++ b/lib/private/Server.php
@@ -88,6 +88,7 @@ use OC\Files\Config\UserMountCacheListener;
use OC\Files\Mount\CacheMountProvider;
use OC\Files\Mount\LocalHomeMountProvider;
use OC\Files\Mount\ObjectHomeMountProvider;
+use OC\Files\Mount\ObjectStorePreviewCacheMountProvider;
use OC\Files\Node\HookConnector;
use OC\Files\Node\LazyRoot;
use OC\Files\Node\Root;
@@ -904,9 +905,11 @@ class Server extends ServerContainer implements IServerContainer {
// builtin providers
$config = $c->getConfig();
+ $logger = $c->getLogger();
$manager->registerProvider(new CacheMountProvider($config));
$manager->registerHomeProvider(new LocalHomeMountProvider());
$manager->registerHomeProvider(new ObjectHomeMountProvider($config));
+ $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config));
return $manager;
});
diff --git a/lib/private/legacy/OC_Util.php b/lib/private/legacy/OC_Util.php
index ab386ab6172..fd55962447e 100644
--- a/lib/private/legacy/OC_Util.php
+++ b/lib/private/legacy/OC_Util.php
@@ -297,6 +297,17 @@ class OC_Util {
self::initLocalStorageRootFS();
}
+ /** @var \OCP\Files\Config\IMountProviderCollection $mountProviderCollection */
+ $mountProviderCollection = \OC::$server->query(\OCP\Files\Config\IMountProviderCollection::class);
+ /** @var \OCP\Files\Mount\IMountPoint[] $rootMountProviders */
+ $rootMountProviders = $mountProviderCollection->getRootMounts();
+
+ /** @var \OC\Files\Mount\Manager $mountManager */
+ $mountManager = \OC\Files\Filesystem::getMountManager();
+ foreach ($rootMountProviders as $rootMountProvider) {
+ $mountManager->addMount($rootMountProvider);
+ }
+
if ($user != '' && !\OC::$server->getUserManager()->userExists($user)) {
\OC::$server->getEventLogger()->end('setup_fs');
return false;
diff --git a/lib/public/Files/Config/IRootMountProvider.php b/lib/public/Files/Config/IRootMountProvider.php
new file mode 100644
index 00000000000..0f7b0eca3d4
--- /dev/null
+++ b/lib/public/Files/Config/IRootMountProvider.php
@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCP\Files\Config;
+
+use OCP\Files\Storage\IStorageFactory;
+
+/**
+ * @since 20.0.0
+ */
+interface IRootMountProvider {
+ /**
+ * Get all root mountpoints
+ *
+ * @return \OCP\Files\Mount\IMountPoint[]
+ * @since 20.0.0
+ */
+ public function getRootMounts(IStorageFactory $loader): array;
+}
--
GitLab