From d8734b3b93d2bcc77a5eba4f0244a41b8a6b9234 Mon Sep 17 00:00:00 2001
From: Morris Jobke <hey@morrisjobke.de>
Date: Thu, 6 Aug 2020 21:30:51 +0200
Subject: [PATCH] Add occ preview:migrate to migrate previews from the old flat
structure to a subfolder structure
* `php occ preview:repair` - a preview migration tool that moves existing previews into the new location introduced with #19214
* moves `appdata_INSTANCEID/previews/FILEID` to `appdata_INSTANCEID/previews/0/5/8/4/c/e/5/FILEID`
* migration tool can be stopped during migration via `CTRL+C` - it then finishes the current folder (with the previews of one file) and stops gracefully
* if a PHP memory limit is set in the `php.ini` then it will stop automatically once it has less than 25 MiB memory left (this is to avoid hard crashes in the middle of a migration)
* the tool can be used during operation - possible drawbacks:
* there is the chance of a race condition that a new preview is generated in the moment the folder is already migrated away - so the old folder with the newly cached preview is deleted and one cached preview needs to be re-generated
* there is the chance of a race condition during access of a preview while it is migrated to the other folder - then no preview can be shown and results in a 404 (as of now this is an accepted risk)
Signed-off-by: Morris Jobke <hey@morrisjobke.de>
---
core/Command/Preview/Repair.php | 280 ++++++++++++++++++++
core/register_command.php | 2 +
lib/composer/composer/autoload_classmap.php | 1 +
lib/composer/composer/autoload_static.php | 1 +
lib/private/Preview/Storage/Root.php | 6 +-
5 files changed, 287 insertions(+), 3 deletions(-)
create mode 100644 core/Command/Preview/Repair.php
diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php
new file mode 100644
index 00000000000..26760cb0e34
--- /dev/null
+++ b/core/Command/Preview/Repair.php
@@ -0,0 +1,280 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\Preview;
+
+use bantu\IniGetWrapper\IniGetWrapper;
+use OC\Preview\Storage\Root;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\IConfig;
+use OCP\ILogger;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+class Repair extends Command {
+ /** @var IConfig */
+ protected $config;
+ /** @var IRootFolder */
+ private $rootFolder;
+ /** @var ILogger */
+ private $logger;
+
+ /** @var bool */
+ private $stopSignalReceived = false;
+ /** @var int */
+ private $memoryLimit;
+ /** @var int */
+ private $memoryTreshold;
+
+ public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni) {
+ $this->config = $config;
+ $this->rootFolder = $rootFolder;
+ $this->logger = $logger;
+
+ $this->memoryLimit = $phpIni->getBytes('memory_limit');
+ $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024;
+
+ parent::__construct();
+ }
+
+ protected function configure() {
+ $this
+ ->setName('preview:repair')
+ ->setDescription('distributes the existing previews into subfolders')
+ ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.')
+ ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.');
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output): int {
+ if ($this->memoryLimit !== -1) {
+ $limitInMiB = round($this->memoryLimit / 1024 /1024, 1);
+ $thresholdInMiB = round($this->memoryTreshold / 1024 /1024, 1);
+ $output->writeln("Memory limit is $limitInMiB MiB");
+ $output->writeln("Memory threshold is $thresholdInMiB MiB");
+ $output->writeln("");
+ $memoryCheckEnabled = true;
+ } else {
+ $output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.");
+ $output->writeln("");
+ $memoryCheckEnabled = false;
+ }
+
+ $dryMode = $input->getOption('dry');
+
+ if ($dryMode) {
+ $output->writeln("INFO: The migration is run in dry mode and will not modify anything.");
+ $output->writeln("");
+ }
+
+ $verbose = $output->isVerbose();
+
+ $instanceId = $this->config->getSystemValueString('instanceid');
+
+ $output->writeln("This will migrate all previews from the old preview location to the new one.");
+ $output->writeln('');
+
+ $output->writeln('Fetching previews that need to be migrated …');
+ /** @var \OCP\Files\Folder $currentPreviewFolder */
+ $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview");
+
+ $directoryListing = $currentPreviewFolder->getDirectoryListing();
+
+ $total = count($directoryListing);
+ /**
+ * by default there could be 0-9 a-f and the old-multibucket folder which are all fine
+ */
+ if ($total < 18) {
+ foreach ($directoryListing as $index => $dir) {
+ if ($dir->getName() === 'old-multibucket') {
+ unset($directoryListing[$index]);
+ }
+ // a-f can't be a file ID -> removing from migration
+ if (preg_match('!^[a-f]$!', $dir->getName())) {
+ unset($directoryListing[$index]);
+ }
+ if (preg_match('!^[0-9]$!', $dir->getName())) {
+ // ignore folders that only has folders in them
+ if ($dir instanceof Folder) {
+ $hasFile = false;
+ foreach ($dir->getDirectoryListing() as $entry) {
+ if (!$entry instanceof Folder) {
+ $hasFile = true;
+ break;
+ }
+ }
+ if (!$hasFile) {
+ unset($directoryListing[$index]);
+ }
+ }
+ }
+ }
+ $total = count($directoryListing);
+ }
+
+ if ($total === 0) {
+ $output->writeln("All previews are already migrated.");
+ return 0;
+ }
+
+ $output->writeln("A total of $total preview files need to be migrated.");
+ $output->writeln("");
+ $output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue.");
+
+ if ($input->getOption('batch')) {
+ $output->writeln('Batch mode active: migration is started right away.');
+ } else {
+ $helper = $this->getHelper('question');
+ $question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false);
+
+ if (!$helper->ask($input, $output, $question)) {
+ return 0;
+ }
+ }
+
+ // register the SIGINT listener late in here to be able to exit in the early process of this command
+ pcntl_signal(SIGINT, [$this, 'sigIntHandler']);
+
+ $output->writeln("");
+ $output->writeln("");
+ $section1 = $output->section();
+ $section2 = $output->section();
+ $progressBar = new ProgressBar($section2, $total);
+ $progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%");
+ $time = (new \DateTime())->format('H:i:s');
+ $progressBar->setMessage("$time Starting …");
+ $progressBar->maxSecondsBetweenRedraws(0.2);
+ $progressBar->start();
+
+ foreach ($directoryListing as $oldPreviewFolder) {
+ pcntl_signal_dispatch();
+ $name = $oldPreviewFolder->getName();
+ $time = (new \DateTime())->format('H:i:s');
+ $section1->writeln("$time Migrating previews of file with fileId $name …");
+ $progressBar->display();
+
+ if ($this->stopSignalReceived) {
+ $section1->writeln("$time Stopping migration …");
+ return 0;
+ }
+ if (!$oldPreviewFolder instanceof Folder) {
+ $section1->writeln(" Skipping non-folder $name …");
+ $progressBar->advance();
+ continue;
+ }
+ if ($name === 'old-multibucket') {
+ $section1->writeln(" Skipping fallback mount point $name …");
+ $progressBar->advance();
+ continue;
+ }
+ if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
+ $section1->writeln(" Skipping hex-digit folder $name …");
+ $progressBar->advance();
+ continue;
+ }
+ if (!preg_match('!^\d+$!', $name)) {
+ $section1->writeln(" Skipping non-numeric folder $name …");
+ $progressBar->advance();
+ continue;
+ }
+
+ $newFoldername = Root::getInternalFolder($name);
+
+ $memoryUsage = memory_get_usage();
+ if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
+ $section1->writeln("");
+ $section1->writeln("");
+ $section1->writeln("");
+ $section1->writeln(" Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
+ $time = (new \DateTime())->format('H:i:s');
+ $section1->writeln("$time Reached memory limit and stopped to avoid hard crash.");
+ return 1;
+ }
+
+ $previews = $oldPreviewFolder->getDirectoryListing();
+ if ($previews !== []) {
+ try {
+ $this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
+ } catch (NotFoundException $e) {
+ if ($verbose) {
+ $section1->writeln(" Create folder preview/$newFoldername");
+ }
+ if (!$dryMode) {
+ $this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
+ }
+ }
+
+ foreach ($previews as $preview) {
+ pcntl_signal_dispatch();
+ $previewName = $preview->getName();
+
+ if ($preview instanceof Folder) {
+ $section1->writeln(" Skipping folder $name/$previewName …");
+ $progressBar->advance();
+ continue;
+ }
+ if ($verbose) {
+ $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername");
+ }
+ if (!$dryMode) {
+ try {
+ $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
+ } catch (\Exception $e) {
+ $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]);
+ }
+ }
+ }
+ }
+ if ($oldPreviewFolder->getDirectoryListing() === []) {
+ if ($verbose) {
+ $section1->writeln(" Delete empty folder preview/$name");
+ }
+ if (!$dryMode) {
+ try {
+ $oldPreviewFolder->delete();
+ } catch (\Exception $e) {
+ $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]);
+ }
+ }
+ }
+ $section1->writeln(" Finished migrating previews of file with fileId $name …");
+ $progressBar->advance();
+ }
+
+ $progressBar->finish();
+ $output->writeln("");
+ return 0;
+ }
+
+ protected function sigIntHandler() {
+ echo "\nSignal received - will finish the step and then stop the migration.\n\n\n";
+ $this->stopSignalReceived = true;
+ }
+}
diff --git a/core/register_command.php b/core/register_command.php
index d818423d1ab..03d458a9cbc 100644
--- a/core/register_command.php
+++ b/core/register_command.php
@@ -155,6 +155,8 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
\OC::$server->getAppManager()
));
+ $application->add(\OC::$server->query(\OC\Core\Command\Preview\Repair::class));
+
$application->add(new OC\Core\Command\User\Add(\OC::$server->getUserManager(), \OC::$server->getGroupManager()));
$application->add(new OC\Core\Command\User\Delete(\OC::$server->getUserManager()));
$application->add(new OC\Core\Command\User\Disable(\OC::$server->getUserManager()));
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 254840b3542..e29e8ce2e7b 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -829,6 +829,7 @@ return array(
'OC\\Core\\Command\\Maintenance\\Repair' => $baseDir . '/core/Command/Maintenance/Repair.php',
'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => $baseDir . '/core/Command/Maintenance/UpdateHtaccess.php',
'OC\\Core\\Command\\Maintenance\\UpdateTheme' => $baseDir . '/core/Command/Maintenance/UpdateTheme.php',
+ 'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php',
'OC\\Core\\Command\\Security\\ImportCertificate' => $baseDir . '/core/Command/Security/ImportCertificate.php',
'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php',
'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 45bf93241d5..8bb4f615520 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -858,6 +858,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Core\\Command\\Maintenance\\Repair' => __DIR__ . '/../../..' . '/core/Command/Maintenance/Repair.php',
'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateHtaccess.php',
'OC\\Core\\Command\\Maintenance\\UpdateTheme' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateTheme.php',
+ 'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php',
'OC\\Core\\Command\\Security\\ImportCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/ImportCertificate.php',
'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php',
'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php',
diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php
index a9a72026a51..1f332c32085 100644
--- a/lib/private/Preview/Storage/Root.php
+++ b/lib/private/Preview/Storage/Root.php
@@ -39,7 +39,7 @@ class Root extends AppData {
public function getFolder(string $name): ISimpleFolder {
- $internalFolder = $this->getInternalFolder($name);
+ $internalFolder = self::getInternalFolder($name);
try {
return parent::getFolder($internalFolder);
@@ -54,7 +54,7 @@ class Root extends AppData {
}
public function newFolder(string $name): ISimpleFolder {
- $internalFolder = $this->getInternalFolder($name);
+ $internalFolder = self::getInternalFolder($name);
return parent::newFolder($internalFolder);
}
@@ -66,7 +66,7 @@ class Root extends AppData {
return [];
}
- private function getInternalFolder(string $name): string {
+ public static function getInternalFolder(string $name): string {
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
}
}
--
GitLab