From 159f28cd521e94eff23e32b5bf3f4d2447da403f Mon Sep 17 00:00:00 2001
From: Morris Jobke <hey@morrisjobke.de>
Date: Thu, 30 Jul 2020 23:53:54 +0200
Subject: [PATCH] Mount the old previews in a separate folder for the multi
bucket setup and check in them before using the actual locations
Signed-off-by: Morris Jobke <hey@morrisjobke.de>
---
.../ObjectStorePreviewCacheMountProvider.php | 51 ++++++++-
lib/private/Preview/Storage/Root.php | 11 ++
...jectStorePreviewCacheMountProviderTest.php | 106 ++++++++++++++++++
3 files changed, 165 insertions(+), 3 deletions(-)
create mode 100644 tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php
diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
index a4acdb6bb0f..9ab0327684b 100644
--- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
+++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php
@@ -26,6 +26,8 @@ declare(strict_types=1);
namespace OC\Files\Mount;
use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage;
+use OC\Files\ObjectStore\ObjectStoreStorage;
+use OC\Files\Storage\Wrapper\Jail;
use OCP\Files\Config\IRootMountProvider;
use OCP\Files\Storage\IStorageFactory;
use OCP\IConfig;
@@ -45,6 +47,10 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
$this->config = $config;
}
+ /**
+ * @return MountPoint[]
+ * @throws \Exception
+ */
public function getRootMounts(IStorageFactory $loader): array {
if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) {
return [];
@@ -65,12 +71,25 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
$i++;
}
}
+
+ $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot();
+ $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments);
+ $fakeRootStorageJail = new Jail([
+ 'storage' => $fakeRootStorage,
+ 'root' => '/appdata_' . $instanceId . '/preview',
+ ]);
+
+ // add a fallback location to be able to fetch existing previews from the old bucket
+ $mountPoints[] = new MountPoint(
+ $fakeRootStorageJail,
+ '/appdata_' . $instanceId . '/preview/old-multibucket',
+ null,
+ $loader
+ );
+
return $mountPoints;
}
- /**
- * @return array
- */
protected function getMultiBucketObjectStore(int $number): array {
$config = $this->config->getSystemValue('objectstore_multibucket');
@@ -99,4 +118,30 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
return $config['arguments'];
}
+
+ protected function getMultiBucketObjectStoreForRoot(): array {
+ $config = $this->config->getSystemValue('objectstore_multibucket');
+
+ // sanity checks
+ if (empty($config['class'])) {
+ $this->logger->error('No class given for objectstore', ['app' => 'files']);
+ }
+ if (!isset($config['arguments'])) {
+ $config['arguments'] = [];
+ }
+
+ /*
+ * Use any provided bucket argument as prefix
+ * and add the mapping from parent/child => bucket
+ */
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+ $config['arguments']['bucket'] .= '0';
+
+ // instantiate object store implementation
+ $config['arguments']['objectstore'] = new $config['class']($config['arguments']);
+
+ return $config['arguments'];
+ }
}
diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php
index 107d87c6301..37ae1758121 100644
--- a/lib/private/Preview/Storage/Root.php
+++ b/lib/private/Preview/Storage/Root.php
@@ -40,6 +40,17 @@ class Root extends AppData {
public function getFolder(string $name): ISimpleFolder {
$internalFolder = $this->getInternalFolder($name);
+ try {
+ return parent::getFolder('old-multibucket/' . $internalFolder);
+ } catch (NotFoundException $e) {
+ // not in multibucket fallback #1
+ }
+ try {
+ return parent::getFolder('old-multibucket/' . $name);
+ } catch (NotFoundException $e) {
+ // not in multibucket fallback #2
+ }
+
try {
return parent::getFolder($internalFolder);
} catch (NotFoundException $e) {
diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php
new file mode 100644
index 00000000000..2da07393f40
--- /dev/null
+++ b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php
@@ -0,0 +1,106 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Test\Files\Mount;
+
+use OC\Files\Mount\ObjectStorePreviewCacheMountProvider;
+use OC\Files\ObjectStore\S3;
+use OC\Files\Storage\StorageFactory;
+use OCP\Files\Storage\IStorageFactory;
+use OCP\IConfig;
+use OCP\ILogger;
+use PHPUnit\Framework\MockObject\MockObject;
+
+/**
+ * @group DB
+ *
+ * The DB permission is needed for the fake root storage initialization
+ */
+class ObjectStorePreviewCacheMountProviderTest extends \Test\TestCase {
+
+ /** @var ObjectStorePreviewCacheMountProvider */
+ protected $provider;
+
+ /** @var ILogger|MockObject */
+ protected $logger;
+ /** @var IConfig|MockObject */
+ protected $config;
+ /** @var IStorageFactory|MockObject */
+ protected $loader;
+
+
+ protected function setUp(): void {
+ parent::setUp();
+
+ $this->logger = $this->createMock(ILogger::class);
+ $this->config = $this->createMock(IConfig::class);
+ $this->loader = $this->createMock(StorageFactory::class);
+
+ $this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config);
+ }
+
+ public function testNoMultibucketObjectStorage() {
+ $this->config->expects($this->once())
+ ->method('getSystemValue')
+ ->with('objectstore_multibucket')
+ ->willReturn(null);
+
+ $this->assertEquals([], $this->provider->getRootMounts($this->loader));
+ }
+
+ public function testMultibucketObjectStorage() {
+ $this->config->expects($this->any())
+ ->method('getSystemValue')
+ ->with('objectstore_multibucket')
+ ->willReturn([
+ 'class' => S3::class,
+ 'arguments' => [
+ 'bucket' => 'abc',
+ 'num_buckets' => 64,
+ 'key' => 'KEY',
+ 'secret' => 'SECRET',
+ 'hostname' => 'IP',
+ 'port' => 'PORT',
+ 'use_ssl' => false,
+ 'use_path_style' => true,
+ ],
+ ]);
+ $this->config->expects($this->once())
+ ->method('getSystemValueString')
+ ->with('instanceid')
+ ->willReturn('INSTANCEID');
+
+ $mounts = $this->provider->getRootMounts($this->loader);
+
+ // 256 mounts for the subfolders and 1 for the fake root
+ $this->assertCount(257, $mounts);
+
+ // do some sanity checks if they have correct mount point paths
+ $this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint());
+ $this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint());
+ // also test the path of the fake bucket
+ $this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint());
+ }
+}
--
GitLab