From 068f9d10f1fec99cebf2ec29de2a09b4b34362b5 Mon Sep 17 00:00:00 2001
From: Vincent Petry <pvince81@owncloud.com>
Date: Tue, 25 Mar 2014 12:52:32 +0100
Subject: [PATCH] Added repair step for legacy storages
---
lib/private/files/cache/storage.php | 25 +-
lib/private/repair.php | 3 +-
lib/private/repair/repairlegacystorages.php | 219 +++++++++++++++
lib/private/repairexception.php | 16 ++
tests/lib/repair/repairlegacystorage.php | 282 ++++++++++++++++++++
5 files changed, 535 insertions(+), 10 deletions(-)
create mode 100644 lib/private/repair/repairlegacystorages.php
create mode 100644 lib/private/repairexception.php
create mode 100644 tests/lib/repair/repairlegacystorage.php
diff --git a/lib/private/files/cache/storage.php b/lib/private/files/cache/storage.php
index 9ad31a272ea..a38656d8499 100644
--- a/lib/private/files/cache/storage.php
+++ b/lib/private/files/cache/storage.php
@@ -28,9 +28,7 @@ class Storage {
} else {
$this->storageId = $storage;
}
- if (strlen($this->storageId) > 64) {
- $this->storageId = md5($this->storageId);
- }
+ $this->storageId = self::adjustStorageId($this->storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($this->storageId));
@@ -43,6 +41,19 @@ class Storage {
}
}
+ /**
+ * Adjusts the storage id to use md5 if too long
+ * @param string $storageId storage id
+ * @return unchanged $storageId if its length is less than 64 characters,
+ * else returns the md5 of $storageId
+ */
+ public static function adjustStorageId($storageId) {
+ if (strlen($storageId) > 64) {
+ return md5($storageId);
+ }
+ return $storageId;
+ }
+
/**
* @return string
*/
@@ -68,9 +79,7 @@ class Storage {
* @return string|null
*/
public static function getNumericStorageId($storageId) {
- if (strlen($storageId) > 64) {
- $storageId = md5($storageId);
- }
+ $storageId = self::adjustStorageId($storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($storageId));
@@ -95,9 +104,7 @@ class Storage {
* @param string $storageId
*/
public static function remove($storageId) {
- if (strlen($storageId) > 64) {
- $storageId = md5($storageId);
- }
+ $storageId = self::adjustStorageId($storageId);
$sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?';
\OC_DB::executeAudited($sql, array($storageId));
diff --git a/lib/private/repair.php b/lib/private/repair.php
index e6943c5d057..46b5ae46399 100644
--- a/lib/private/repair.php
+++ b/lib/private/repair.php
@@ -69,7 +69,8 @@ class Repair extends BasicEmitter {
*/
public static function getRepairSteps() {
return array(
- new \OC\Repair\RepairMimeTypes()
+ new \OC\Repair\RepairMimeTypes(),
+ new \OC\Repair\RepairLegacyStorages(\OC::$server->getConfig(), \OC_DB::getConnection()),
);
}
diff --git a/lib/private/repair/repairlegacystorages.php b/lib/private/repair/repairlegacystorages.php
new file mode 100644
index 00000000000..9d38b256c88
--- /dev/null
+++ b/lib/private/repair/repairlegacystorages.php
@@ -0,0 +1,219 @@
+<?php
+/**
+ * Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\Repair;
+
+use OC\Hooks\BasicEmitter;
+
+class RepairLegacyStorages extends BasicEmitter {
+ /**
+ * @var \OCP\IConfig
+ */
+ protected $config;
+
+ /**
+ * @var \OC\DB\Connection
+ */
+ protected $connection;
+
+ protected $findStorageInCacheStatement;
+ protected $renameStorageStatement;
+
+ /**
+ * @param \OCP\IConfig $config
+ * @param \OC\DB\Connection $connection
+ */
+ public function __construct($config, $connection) {
+ $this->connection = $connection;
+ $this->config = $config;
+
+ $this->findStorageInCacheStatement = $this->connection->prepare(
+ 'SELECT DISTINCT `storage` FROM `*PREFIX*filecache`'
+ . ' WHERE `storage` in (?, ?)'
+ );
+ $this->renameStorageStatement = $this->connection->prepare(
+ 'UPDATE `*PREFIX*storages`'
+ . ' SET `id` = ?'
+ . ' WHERE `id` = ?'
+ );
+ }
+
+ public function getName() {
+ return 'Repair legacy storages';
+ }
+
+ /**
+ * Extracts the user id from a legacy storage id
+ *
+ * @param string $storageId legacy storage id in the
+ * format "local::/path/to/datadir/userid"
+ * @return string user id extracted from the storage id
+ */
+ private function extractUserId($storageId) {
+ $storageId = rtrim($storageId, '/');
+ $pos = strrpos($storageId, '/');
+ return substr($storageId, $pos + 1);
+ }
+
+ /**
+ * Fix the given legacy storage by renaming the old id
+ * to the new id. If the new id already exists, whichever
+ * storage that has data in the file cache will be used.
+ * If both have data, nothing will be done and false is
+ * returned.
+ *
+ * @param string $oldId old storage id
+ * @param int $oldNumericId old storage numeric id
+ *
+ * @return bool true if fixed, false otherwise
+ */
+ private function fixLegacyStorage($oldId, $oldNumericId, $userId = null) {
+ // check whether the new storage already exists
+ if (is_null($userId)) {
+ $userId = $this->extractUserId($oldId);
+ }
+ $newId = 'home::' . $userId;
+
+ // check if target id already exists
+ $newNumericId = \OC\Files\Cache\Storage::getNumericStorageId($newId);
+ if (!is_null($newNumericId)) {
+ $newNumericId = (int)$newNumericId;
+ // try and resolve the conflict
+ // check which one of "local::" or "home::" needs to be kept
+ $result = $this->findStorageInCacheStatement->execute(array($oldNumericId, $newNumericId));
+ $row1 = $this->findStorageInCacheStatement->fetch();
+ $row2 = $this->findStorageInCacheStatement->fetch();
+ if ($row2 !== false) {
+ // two results means both storages have data, not auto-fixable
+ throw new \OC\RepairException(
+ 'Could not automatically fix legacy storage '
+ . '"' . $oldId . '" => "' . $newId . '"'
+ . ' because they both have data.'
+ );
+ }
+ if ($row1 === false || (int)$row1['storage'] === $oldNumericId) {
+ // old storage has data, then delete the empty new id
+ $toDelete = $newId;
+ } else if ((int)$row1['storage'] === $newNumericId) {
+ // new storage has data, then delete the empty old id
+ $toDelete = $oldId;
+ } else {
+ // unknown case, do not continue
+ return false;
+ }
+
+ // delete storage including file cache
+ \OC\Files\Cache\Storage::remove($toDelete);
+
+ // if we deleted the old id, the new id will be used
+ // automatically
+ if ($toDelete === $oldId) {
+ // nothing more to do
+ return true;
+ }
+ }
+
+ // rename old id to new id
+ $newId = \OC\Files\Cache\Storage::adjustStorageId($newId);
+ $oldId = \OC\Files\Cache\Storage::adjustStorageId($oldId);
+ $rowCount = $this->renameStorageStatement->execute(array($newId, $oldId));
+ return ($rowCount === 1);
+ }
+
+ /**
+ * Converts legacy home storage ids in the format
+ * "local::/data/dir/path/userid/" to the new format "home::userid"
+ */
+ public function run() {
+ // only run once
+ if ($this->config->getAppValue('core', 'repairlegacystoragesdone') === 'yes') {
+ return;
+ }
+
+ $dataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/');
+ $dataDir = rtrim($dataDir, '/') . '/';
+ $dataDirId = 'local::' . $dataDir;
+
+ $count = 0;
+
+ $this->connection->beginTransaction();
+
+ try {
+ // note: not doing a direct UPDATE with the REPLACE function
+ // because regexp search/extract is needed and it is not guaranteed
+ // to work on all database types
+ $sql = 'SELECT `id`, `numeric_id` FROM `*PREFIX*storages`'
+ . ' WHERE `id` LIKE ?'
+ . ' ORDER BY `id`';
+ $result = $this->connection->executeQuery($sql, array($dataDirId . '%'));
+ while ($row = $result->fetch()) {
+ $currentId = $row['id'];
+ // one entry is the datadir itself
+ if ($currentId === $dataDirId) {
+ continue;
+ }
+
+ if ($this->fixLegacyStorage($currentId, (int)$row['numeric_id'])) {
+ $count++;
+ }
+ }
+
+ // check for md5 ids, not in the format "prefix::"
+ $sql = 'SELECT COUNT(*) AS "c" FROM `*PREFIX*storages`'
+ . ' WHERE `id` NOT LIKE \'%::%\'';
+ $result = $this->connection->executeQuery($sql);
+ $row = $result->fetch();
+ // find at least one to make sure it's worth
+ // querying the user list
+ if ((int)$row['c'] > 0) {
+ $userManager = \OC_User::getManager();
+
+ // use chunks to avoid caching too many users in memory
+ $limit = 30;
+ $offset = 0;
+
+ do {
+ // query the next page of users
+ $results = $userManager->search('', $limit, $offset);
+ $storageIds = array();
+ $userIds = array();
+ foreach ($results as $uid => $userObject) {
+ $storageId = $dataDirId . $uid . '/';
+ if (strlen($storageId) <= 64) {
+ // skip short storage ids as they were handled in the previous section
+ continue;
+ }
+ $storageIds[$uid] = $storageId;
+ }
+
+ if (count($storageIds) > 0) {
+ // update the storages of these users
+ foreach ($storageIds as $uid => $storageId) {
+ $numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId);
+ if (!is_null($numericId) && $this->fixLegacyStorage($storageId, (int)$numericId)) {
+ $count++;
+ }
+ }
+ }
+ $offset += $limit;
+ } while (count($results) >= $limit);
+ }
+
+ $this->emit('\OC\Repair', 'info', array('Updated ' . $count . ' legacy home storage ids'));
+
+ $this->connection->commit();
+ }
+ catch (\OC\RepairException $e) {
+ $this->connection->rollback();
+ throw $e;
+ }
+
+ $this->config->setAppValue('core', 'repairlegacystoragesdone', 'yes');
+ }
+}
diff --git a/lib/private/repairexception.php b/lib/private/repairexception.php
new file mode 100644
index 00000000000..642bf90ee40
--- /dev/null
+++ b/lib/private/repairexception.php
@@ -0,0 +1,16 @@
+<?php
+/**
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC;
+
+/**
+ * Exception thrown whenever a database/migration repair
+ * could not be done.
+ */
+class RepairException extends \Exception {
+}
diff --git a/tests/lib/repair/repairlegacystorage.php b/tests/lib/repair/repairlegacystorage.php
new file mode 100644
index 00000000000..4528c5288df
--- /dev/null
+++ b/tests/lib/repair/repairlegacystorage.php
@@ -0,0 +1,282 @@
+<?php
+/**
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+/**
+ * Tests for the converting of legacy storages to home storages.
+ *
+ * @see \OC\Repair\RepairLegacyStorages
+ */
+class TestRepairLegacyStorages extends PHPUnit_Framework_TestCase {
+
+ private $user;
+ private $repair;
+
+ private $dataDir;
+ private $oldDataDir;
+
+ private $legacyStorageId;
+ private $newStorageId;
+
+ public function setUp() {
+ $this->config = \OC::$server->getConfig();
+ $this->connection = \OC_DB::getConnection();
+ $this->oldDataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/');
+
+ $this->repair = new \OC\Repair\RepairLegacyStorages($this->config, $this->connection);
+ }
+
+ public function tearDown() {
+ \OC_User::deleteUser($this->user);
+
+ $sql = 'DELETE FROM `*PREFIX*storages`';
+ $this->connection->executeQuery($sql);
+ $sql = 'DELETE FROM `*PREFIX*filecache`';
+ $this->connection->executeQuery($sql);
+ \OCP\Config::setSystemValue('datadirectory', $this->oldDataDir);
+ $this->config->setAppValue('core', 'repairlegacystoragesdone', 'no');
+ }
+
+ function prepareSettings($dataDir, $userId) {
+ // hard-coded string as we want a predictable fixed length
+ // no data will be written there
+ $this->dataDir = $dataDir;
+ \OCP\Config::setSystemValue('datadirectory', $this->dataDir);
+
+ $this->user = $userId;
+ $this->legacyStorageId = 'local::' . $this->dataDir . $this->user . '/';
+ $this->newStorageId = 'home::' . $this->user;
+ \OC_User::createUser($this->user, $this->user);
+ }
+
+ /**
+ * Create a storage entry
+ *
+ * @param string $storageId
+ */
+ private function createStorage($storageId) {
+ $sql = 'INSERT INTO `*PREFIX*storages` (`id`)'
+ . ' VALUES (?)';
+
+ $storageId = \OC\Files\Cache\Storage::adjustStorageId($storageId);
+ $numRows = $this->connection->executeUpdate($sql, array($storageId));
+ $this->assertEquals(1, $numRows);
+
+ return \OC_DB::insertid('*PREFIX*storages');
+ }
+
+ /**
+ * Returns the storage id based on the numeric id
+ *
+ * @param int $numericId numeric id of the storage
+ * @return string storage id or null if not found
+ */
+ private function getStorageId($storageId) {
+ $numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId);
+ if (!is_null($numericId)) {
+ return (int)$numericId;
+ }
+ return null;
+ }
+
+ /**
+ * Create dummy data in the filecache for the given storage numeric id
+ *
+ * @param string $storageId storage id
+ */
+ private function createData($storageId) {
+ $cache = new \OC\Files\Cache\Cache($storageId);
+ $cache->put(
+ 'dummyfile.txt',
+ array('size' => 5, 'mtime' => 12, 'mimetype' => 'text/plain')
+ );
+ }
+
+ /**
+ * Test that existing home storages are left alone when valid.
+ * @dataProvider settingsProvider
+ */
+ public function testNoopWithExistingHomeStorage($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $newStorageNumId = $this->createStorage($this->newStorageId);
+
+ $this->repair->run();
+
+ $this->assertNull($this->getStorageId($this->legacyStorageId));
+ $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+ }
+
+ /**
+ * Test that legacy storages are converted to home storages when
+ * the latter does not exist.
+ * @dataProvider settingsProvider
+ */
+ public function testConvertLegacyToHomeStorage($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+
+ $this->repair->run();
+
+ $this->assertNull($this->getStorageId($this->legacyStorageId));
+ $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId));
+ }
+
+ /**
+ * Test that legacy storages are converted to home storages
+ * when home storage already exists but has no data.
+ * @dataProvider settingsProvider
+ */
+ public function testConvertLegacyToExistingEmptyHomeStorage($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+ $newStorageNumId = $this->createStorage($this->newStorageId);
+
+ $this->createData($this->legacyStorageId);
+
+ $this->repair->run();
+
+ $this->assertNull($this->getStorageId($this->legacyStorageId));
+ $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId));
+ }
+
+ /**
+ * Test that legacy storages are converted to home storages
+ * when home storage already exists and the legacy storage
+ * has no data.
+ * @dataProvider settingsProvider
+ */
+ public function testConvertEmptyLegacyToHomeStorage($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+ $newStorageNumId = $this->createStorage($this->newStorageId);
+
+ $this->createData($this->newStorageId);
+
+ $this->repair->run();
+
+ $this->assertNull($this->getStorageId($this->legacyStorageId));
+ $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+ }
+
+ /**
+ * Test that nothing is done when both conflicting legacy
+ * and home storage have data.
+ * @dataProvider settingsProvider
+ */
+ public function testConflictNoop($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+ $newStorageNumId = $this->createStorage($this->newStorageId);
+
+ $this->createData($this->legacyStorageId);
+ $this->createData($this->newStorageId);
+
+ try {
+ $thrown = false;
+ $this->repair->run();
+ }
+ catch (\OC\RepairException $e) {
+ $thrown = true;
+ }
+
+ $this->assertTrue($thrown);
+
+ // storages left alone
+ $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->legacyStorageId));
+ $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+
+ // did not set the done flag
+ $this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+ }
+
+ /**
+ * Test that the data dir local entry is left alone
+ * @dataProvider settingsProvider
+ */
+ public function testDataDirEntryNoop($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $storageId = 'local::' . $this->dataDir;
+ $numId = $this->createStorage($storageId);
+
+ $this->repair->run();
+
+ $this->assertEquals($numId, $this->getStorageId($storageId));
+ }
+
+ /**
+ * Test that external local storages are left alone
+ * @dataProvider settingsProvider
+ */
+ public function testLocalExtStorageNoop($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $storageId = 'local::/tmp/somedir/' . $this->user;
+ $numId = $this->createStorage($storageId);
+
+ $this->repair->run();
+
+ $this->assertEquals($numId, $this->getStorageId($storageId));
+ }
+
+ /**
+ * Test that other external storages are left alone
+ * @dataProvider settingsProvider
+ */
+ public function testExtStorageNoop($dataDir, $userId) {
+ $this->prepareSettings($dataDir, $userId);
+ $storageId = 'smb::user@password/tmp/somedir/' . $this->user;
+ $numId = $this->createStorage($storageId);
+
+ $this->repair->run();
+
+ $this->assertEquals($numId, $this->getStorageId($storageId));
+ }
+
+ /**
+ * Provides data dir and user name
+ */
+ function settingsProvider() {
+ return array(
+ // regular data dir
+ array(
+ '/tmp/oc-autotest/datadir/',
+ uniqid('user_'),
+ ),
+ // long datadir / short user
+ array(
+ '/tmp/oc-autotest/datadir01234567890123456789012345678901234567890123456789END/',
+ uniqid('user_'),
+ ),
+ // short datadir / long user
+ array(
+ '/tmp/oc-autotest/datadir/',
+ 'u123456789012345678901234567890123456789012345678901234567890END', // 64 chars
+ ),
+ );
+ }
+
+ /**
+ * Only run the repair once
+ */
+ public function testOnlyRunOnce() {
+ $output = array();
+ $this->repair->listen('\OC\Repair', 'info', function ($description) use (&$output) {
+ $output[] = 'info: ' . $description;
+ });
+
+ $this->prepareSettings('/tmp/oc-autotest/datadir', uniqid('user_'));
+ $this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+ $this->repair->run();
+ $this->assertEquals(1, count($output));
+ $this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+
+ $output = array();
+ $this->repair->run();
+ // no output which means it did not run
+ $this->assertEquals(0, count($output));
+ $this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+ }
+}
--
GitLab