diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 01dcecb0e47818cb961b9ff2deef97323dcd15fa..53ffecddd3bcd9ec2e2f96b9eddcbe4950cf100e 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -1153,6 +1153,7 @@ return array(
'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php',
'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php',
+ 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php',
'OC\\Preview\\Watcher' => $baseDir . '/lib/private/Preview/Watcher.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 7a9bd5652d672df3626f0a40910234778ed63459..45ab02ded2b2f7bd3885c75ed55040825063b5cd 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -1182,6 +1182,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php',
'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php',
+ 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php',
'OC\\Preview\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Preview/Watcher.php',
diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php
index 15bb6bd71887349c04983b14119646a177bfee4e..e0546c4a11c7adcbf71e3787fda0c14b11b3f408 100644
--- a/lib/private/Preview/BackgroundCleanupJob.php
+++ b/lib/private/Preview/BackgroundCleanupJob.php
@@ -27,8 +27,9 @@ declare(strict_types=1);
namespace OC\Preview;
use OC\BackgroundJob\TimedJob;
-use OC\Files\AppData\Factory;
+use OC\Preview\Storage\Root;
use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\IDBConnection;
@@ -38,28 +39,47 @@ class BackgroundCleanupJob extends TimedJob {
/** @var IDBConnection */
private $connection;
- /** @var Factory */
- private $appDataFactory;
+ /** @var Root */
+ private $previewFolder;
/** @var bool */
private $isCLI;
+ /** @var IMimeTypeLoader */
+ private $mimeTypeLoader;
+
public function __construct(IDBConnection $connection,
- Factory $appDataFactory,
+ Root $previewFolder,
+ IMimeTypeLoader $mimeTypeLoader,
bool $isCLI) {
// Run at most once an hour
$this->setInterval(3600);
$this->connection = $connection;
- $this->appDataFactory = $appDataFactory;
+ $this->previewFolder = $previewFolder;
$this->isCLI = $isCLI;
+ $this->mimeTypeLoader = $mimeTypeLoader;
}
public function run($argument) {
- $previews = $this->appDataFactory->get('preview');
+ foreach ($this->getDeletedFiles() as $fileId) {
+ try {
+ $preview = $this->previewFolder->getFolder((string)$fileId);
+ $preview->delete();
+ } catch (NotFoundException $e) {
+ // continue
+ } catch (NotPermittedException $e) {
+ // continue
+ }
+ }
+ }
- $previewFodlerId = $previews->getId();
+ private function getDeletedFiles(): \Iterator {
+ yield from $this->getOldPreviewLocations();
+ yield from $this->getNewPreviewLocations();
+ }
+ private function getOldPreviewLocations(): \Iterator {
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name')
->from('filecache', 'a')
@@ -69,7 +89,9 @@ class BackgroundCleanupJob extends TimedJob {
->where(
$qb->expr()->isNull('b.fileid')
)->andWhere(
- $qb->expr()->eq('a.parent', $qb->createNamedParameter($previewFodlerId))
+ $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId()))
+ )->andWhere(
+ $qb->expr()->like('a.name', $qb->createNamedParameter('__%'))
);
if (!$this->isCLI) {
@@ -79,14 +101,54 @@ class BackgroundCleanupJob extends TimedJob {
$cursor = $qb->execute();
while ($row = $cursor->fetch()) {
- try {
- $preview = $previews->getFolder($row['name']);
- $preview->delete();
- } catch (NotFoundException $e) {
- // continue
- } catch (NotPermittedException $e) {
- // continue
- }
+ yield $row['name'];
+ }
+
+ $cursor->closeCursor();
+ }
+
+ private function getNewPreviewLocations(): \Iterator {
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('path', 'mimetype')
+ ->from('filecache')
+ ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
+ $cursor = $qb->execute();
+ $data = $cursor->fetch();
+ $cursor->closeCursor();
+
+ if ($data === null) {
+ return [];
+ }
+
+ /*
+ * This lovely like is the result of the way the new previews are stored
+ * We take the md5 of the name (fileid) and split the first 7 chars. That way
+ * there are not a gazillion files in the root of the preview appdata.
+ */
+ $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
+
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('a.name')
+ ->from('filecache', 'a')
+ ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
+ $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
+ ))
+ ->where(
+ $qb->expr()->andX(
+ $qb->expr()->isNull('b.fileid'),
+ $qb->expr()->like('a.path', $qb->createNamedParameter($like)),
+ $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory')))
+ )
+ );
+
+ if (!$this->isCLI) {
+ $qb->setMaxResults(10);
+ }
+
+ $cursor = $qb->execute();
+
+ while ($row = $cursor->fetch()) {
+ yield $row['name'];
}
$cursor->closeCursor();
diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php
new file mode 100644
index 0000000000000000000000000000000000000000..0e9d92a44c24a3f4c0600e43da943f1880416285
--- /dev/null
+++ b/lib/private/Preview/Storage/Root.php
@@ -0,0 +1,71 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Preview\Storage;
+
+use OC\Files\AppData\AppData;
+use OC\SystemConfig;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\Files\SimpleFS\ISimpleFolder;
+
+class Root extends AppData {
+ public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) {
+ parent::__construct($rootFolder, $systemConfig, 'preview');
+ }
+
+
+ public function getFolder(string $name): ISimpleFolder {
+ $internalFolder = $this->getInternalFolder($name);
+
+ try {
+ return parent::getFolder($internalFolder);
+ } catch (NotFoundException $e) {
+ /*
+ * The new folder structure is not found.
+ * Lets try the old one
+ */
+ }
+
+ return parent::getFolder($name);
+ }
+
+ public function newFolder(string $name): ISimpleFolder {
+ $internalFolder = $this->getInternalFolder($name);
+ return parent::newFolder($internalFolder);
+ }
+
+ /*
+ * Do not allow directory listing on this special root
+ * since it gets to big and time consuming
+ */
+ public function getDirectoryListing(): array {
+ return [];
+ }
+
+ private function getInternalFolder(string $name): string {
+ return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
+ }
+}
diff --git a/lib/private/Server.php b/lib/private/Server.php
index cf4f8cc097b9a7fb3e6e380a6c3a291dd61278f5..4fe89e3098e3e7805d77a2853fa0b1f680c7e30d 100644
--- a/lib/private/Server.php
+++ b/lib/private/Server.php
@@ -281,7 +281,7 @@ class Server extends ServerContainer implements IServerContainer {
return new PreviewManager(
$c->getConfig(),
$c->getRootFolder(),
- $c->getAppDataDir('preview'),
+ new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview'),
$c->getEventDispatcher(),
$c->getGeneratorHelper(),
$c->getSession()->get('user_id')
@@ -291,7 +291,7 @@ class Server extends ServerContainer implements IServerContainer {
$this->registerService(\OC\Preview\Watcher::class, function (Server $c) {
return new \OC\Preview\Watcher(
- $c->getAppDataDir('preview')
+ new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview')
);
});
diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php
index db2bf09b6e57c583e4b0a46bbbd4d5bc757515e0..cd9f6ef03991d17caf12a419ff389558d078aa1a 100644
--- a/tests/lib/Preview/BackgroundCleanupJobTest.php
+++ b/tests/lib/Preview/BackgroundCleanupJobTest.php
@@ -22,10 +22,13 @@
namespace Test\Preview;
-use OC\Files\AppData\Factory;
use OC\Preview\BackgroundCleanupJob;
+use OC\Preview\Storage\Root;
use OC\PreviewManager;
+use OCP\Files\File;
+use OCP\Files\IMimeTypeLoader;
use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
use OCP\IDBConnection;
use Test\Traits\MountProviderTrait;
use Test\Traits\UserTrait;
@@ -47,9 +50,6 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
/** @var bool */
private $trashEnabled;
- /** @var Factory */
- private $appDataFactory;
-
/** @var IDBConnection */
private $connection;
@@ -59,6 +59,9 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
/** @var IRootFolder */
private $rootFolder;
+ /** @var IMimeTypeLoader */
+ private $mimeTypeLoader;
+
protected function setUp(): void {
parent::setUp();
@@ -76,13 +79,10 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
$this->trashEnabled = $appManager->isEnabledForUser('files_trashbin', $this->userId);
$appManager->disableApp('files_trashbin');
- $this->appDataFactory = new Factory(
- \OC::$server->getRootFolder(),
- \OC::$server->getSystemConfig()
- );
$this->connection = \OC::$server->getDatabaseConnection();
$this->previewManager = \OC::$server->getPreviewManager();
$this->rootFolder = \OC::$server->getRootFolder();
+ $this->mimeTypeLoader = \OC::$server->getMimeTypeLoader();
}
protected function tearDown(): void {
@@ -96,6 +96,13 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
parent::tearDown();
}
+ private function getRoot(): Root {
+ return new Root(
+ \OC::$server->getRootFolder(),
+ \OC::$server->getSystemConfig()
+ );
+ }
+
private function setup11Previews(): array {
$userFolder = $this->rootFolder->getUserFolder($this->userId);
@@ -110,52 +117,89 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
return $files;
}
+ private function countPreviews(Root $previewRoot, array $fileIds): int {
+ $i = 0;
+
+ foreach ($fileIds as $fileId) {
+ try {
+ $previewRoot->getFolder((string)$fileId);
+ } catch (NotFoundException $e) {
+ continue;
+ }
+
+ $i++;
+ }
+
+ return $i;
+ }
+
public function testCleanupSystemCron() {
$files = $this->setup11Previews();
+ $fileIds = array_map(function (File $f) {
+ return $f->getId();
+ }, $files);
- $preview = $this->appDataFactory->get('preview');
-
- $previews = $preview->getDirectoryListing();
- $this->assertCount(11, $previews);
+ $root = $this->getRoot();
- $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, true);
+ $this->assertSame(11, $this->countPreviews($root, $fileIds));
+ $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, true);
$job->run([]);
foreach ($files as $file) {
$file->delete();
}
- $this->assertCount(11, $previews);
+ $root = $this->getRoot();
+ $this->assertSame(11, $this->countPreviews($root, $fileIds));
$job->run([]);
- $previews = $preview->getDirectoryListing();
- $this->assertCount(0, $previews);
+ $root = $this->getRoot();
+ $this->assertSame(0, $this->countPreviews($root, $fileIds));
}
public function testCleanupAjax() {
$files = $this->setup11Previews();
+ $fileIds = array_map(function (File $f) {
+ return $f->getId();
+ }, $files);
- $preview = $this->appDataFactory->get('preview');
-
- $previews = $preview->getDirectoryListing();
- $this->assertCount(11, $previews);
+ $root = $this->getRoot();
- $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, false);
+ $this->assertSame(11, $this->countPreviews($root, $fileIds));
+ $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, false);
$job->run([]);
foreach ($files as $file) {
$file->delete();
}
- $this->assertCount(11, $previews);
+ $root = $this->getRoot();
+ $this->assertSame(11, $this->countPreviews($root, $fileIds));
$job->run([]);
- $previews = $preview->getDirectoryListing();
- $this->assertCount(1, $previews);
+ $root = $this->getRoot();
+ $this->assertSame(1, $this->countPreviews($root, $fileIds));
+ $job->run([]);
+
+ $root = $this->getRoot();
+ $this->assertSame(0, $this->countPreviews($root, $fileIds));
+ }
+
+ public function testOldPreviews() {
+ $appdata = \OC::$server->getAppDataDir('preview');
+
+ $f1 = $appdata->newFolder('123456781');
+ $f1->newFile('foo.jpg', 'foo');
+ $f2 = $appdata->newFolder('123456782');
+ $f2->newFile('foo.jpg', 'foo');
+
+ $appdata = \OC::$server->getAppDataDir('preview');
+ $this->assertSame(2, count($appdata->getDirectoryListing()));
+ $job = new BackgroundCleanupJob($this->connection, $this->getRoot(), $this->mimeTypeLoader, true);
$job->run([]);
- $previews = $preview->getDirectoryListing();
- $this->assertCount(0, $previews);
+ $appdata = \OC::$server->getAppDataDir('preview');
+ $this->assertSame(0, count($appdata->getDirectoryListing()));
}
}