From cc28f82b369c2e8ebf2d0b4390379b9cda4af40b Mon Sep 17 00:00:00 2001
From: Morris Jobke <hey@morrisjobke.de>
Date: Thu, 30 Jul 2015 13:57:04 +0200
Subject: [PATCH] Add config option to update charset of mysql to utf8mb4
* fully optional
* requires additional options set in the database
---
config/config.sample.php | 29 ++++++++++++++++++++++++++++
core/register_command.php | 2 +-
lib/private/DB/AdapterMySQL.php | 3 ++-
lib/private/DB/ConnectionFactory.php | 7 +++++++
lib/private/DB/MDB2SchemaReader.php | 14 +++++++++++++-
lib/private/DB/MDB2SchemaWriter.php | 6 +++++-
lib/private/Repair/Collation.php | 7 +++++--
lib/private/Server.php | 2 +-
lib/private/Setup/MySQL.php | 5 +++--
9 files changed, 66 insertions(+), 9 deletions(-)
diff --git a/config/config.sample.php b/config/config.sample.php
index 3aa0f353c59..df1e2d16fc0 100644
--- a/config/config.sample.php
+++ b/config/config.sample.php
@@ -129,6 +129,7 @@ $CONFIG = array(
*/
'dbtableprefix' => '',
+
/**
* Indicates whether the Nextcloud instance was installed successfully; ``true``
* indicates a successful installation, and ``false`` indicates an unsuccessful
@@ -1079,6 +1080,34 @@ $CONFIG = array(
*/
'sqlite.journal_mode' => 'DELETE',
+/**
+ * If this setting is set to true MySQL can handle 4 byte characters instead of
+ * 3 byte characters
+ *
+ * MySQL requires a special setup for longer indexes (> 767 bytes) which are
+ * needed:
+ *
+ * [mysqld]
+ * innodb_large_prefix=true
+ * innodb_file_format=barracuda
+ * innodb_file_per_table=true
+ *
+ * Tables will be created with
+ * * character set: utf8mb4
+ * * collation: utf8mb4_bin
+ * * row_format: compressed
+ *
+ * See:
+ * https://dev.mysql.com/doc/refman/5.7/en/charset-unicode-utf8mb4.html
+ * https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_large_prefix
+ * https://mariadb.com/kb/en/mariadb/xtradbinnodb-server-system-variables/#innodb_large_prefix
+ * http://www.tocker.ca/2013/10/31/benchmarking-innodb-page-compression-performance.html
+ * http://mechanics.flite.com/blog/2014/07/29/using-innodb-large-prefix-to-avoid-error-1071/
+ *
+ * WARNING: EXPERIMENTAL
+ */
+'mysql.utf8mb4' => false,
+
/**
* Database types that are supported for installation.
*
diff --git a/core/register_command.php b/core/register_command.php
index a6da3cbd899..fa85aea8956 100644
--- a/core/register_command.php
+++ b/core/register_command.php
@@ -83,7 +83,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
$application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig()));
$application->add(new OC\Core\Command\Db\GenerateChangeScript());
- $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory()));
+ $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig()));
$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager()));
diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php
index 3e2fceda8db..0c0c6b31021 100644
--- a/lib/private/DB/AdapterMySQL.php
+++ b/lib/private/DB/AdapterMySQL.php
@@ -39,7 +39,8 @@ class AdapterMySQL extends Adapter {
}
public function fixupStatement($statement) {
- $statement = str_replace(' ILIKE ', ' COLLATE utf8_general_ci LIKE ', $statement);
+ $characterSet = \OC::$server->getConfig()->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
+ $statement = str_replace(' ILIKE ', ' COLLATE ' . $characterSet . '_general_ci LIKE ', $statement);
return $statement;
}
}
diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php
index b2c356edef7..a7aae32f670 100644
--- a/lib/private/DB/ConnectionFactory.php
+++ b/lib/private/DB/ConnectionFactory.php
@@ -28,6 +28,7 @@ namespace OC\DB;
use Doctrine\DBAL\Event\Listeners\OracleSessionInit;
use Doctrine\DBAL\Event\Listeners\SQLSessionInit;
use Doctrine\DBAL\Event\Listeners\MysqlSessionInit;
+use OC\SystemConfig;
/**
* Takes care of creating and configuring Doctrine connections.
@@ -64,6 +65,12 @@ class ConnectionFactory {
),
);
+ public function __construct(SystemConfig $systemConfig) {
+ if($systemConfig->getValue('mysql.utf8mb4', false)) {
+ $defaultConnectionParams['mysql']['charset'] = 'utf8mb4';
+ }
+ }
+
/**
* @brief Get default connection parameters for a given DBMS.
* @param string $type DBMS type
diff --git a/lib/private/DB/MDB2SchemaReader.php b/lib/private/DB/MDB2SchemaReader.php
index 3f183c9723a..c198bb31e00 100644
--- a/lib/private/DB/MDB2SchemaReader.php
+++ b/lib/private/DB/MDB2SchemaReader.php
@@ -33,6 +33,7 @@ namespace OC\DB;
use Doctrine\DBAL\Platforms\AbstractPlatform;
use Doctrine\DBAL\Schema\SchemaConfig;
+use Doctrine\DBAL\Platforms\MySqlPlatform;
use OCP\IConfig;
class MDB2SchemaReader {
@@ -54,12 +55,16 @@ class MDB2SchemaReader {
/** @var \Doctrine\DBAL\Schema\SchemaConfig $schemaConfig */
protected $schemaConfig;
+ /** @var IConfig */
+ protected $config;
+
/**
* @param \OCP\IConfig $config
* @param \Doctrine\DBAL\Platforms\AbstractPlatform $platform
*/
public function __construct(IConfig $config, AbstractPlatform $platform) {
$this->platform = $platform;
+ $this->config = $config;
$this->DBNAME = $config->getSystemValue('dbname', 'owncloud');
$this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_');
@@ -118,8 +123,15 @@ class MDB2SchemaReader {
$name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name);
$name = $this->platform->quoteIdentifier($name);
$table = $schema->createTable($name);
- $table->addOption('collate', 'utf8_bin');
$table->setSchemaConfig($this->schemaConfig);
+
+ if($this->platform instanceof MySqlPlatform && $this->config->getSystemValue('mysql.utf8mb4', false)) {
+ $table->addOption('charset', 'utf8mb4');
+ $table->addOption('collate', 'utf8mb4_bin');
+ $table->addOption('row_format', 'compressed');
+ } else {
+ $table->addOption('collate', 'utf8_bin');
+ }
break;
case 'create':
case 'overwrite':
diff --git a/lib/private/DB/MDB2SchemaWriter.php b/lib/private/DB/MDB2SchemaWriter.php
index 26e32b036fe..7664b4359ab 100644
--- a/lib/private/DB/MDB2SchemaWriter.php
+++ b/lib/private/DB/MDB2SchemaWriter.php
@@ -43,7 +43,11 @@ class MDB2SchemaWriter {
$xml->addChild('name', $config->getSystemValue('dbname', 'owncloud'));
$xml->addChild('create', 'true');
$xml->addChild('overwrite', 'false');
- $xml->addChild('charset', 'utf8');
+ if($config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false)) {
+ $xml->addChild('charset', 'utf8mb4');
+ } else {
+ $xml->addChild('charset', 'utf8');
+ }
// FIX ME: bloody work around
if ($config->getSystemValue('dbtype', 'sqlite') === 'oci') {
diff --git a/lib/private/Repair/Collation.php b/lib/private/Repair/Collation.php
index 74c4ca2e6ac..c19b8eea5ec 100644
--- a/lib/private/Repair/Collation.php
+++ b/lib/private/Repair/Collation.php
@@ -61,10 +61,12 @@ class Collation implements IRepairStep {
return;
}
+ $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
+
$tables = $this->getAllNonUTF8BinTables($this->connection);
foreach ($tables as $table) {
$output->info("Change collation for $table ...");
- $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;');
+ $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;');
$query->execute();
}
}
@@ -75,11 +77,12 @@ class Collation implements IRepairStep {
*/
protected function getAllNonUTF8BinTables($connection) {
$dbName = $this->config->getSystemValue("dbname");
+ $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
$rows = $connection->fetchAll(
"SELECT DISTINCT(TABLE_NAME) AS `table`" .
" FROM INFORMATION_SCHEMA . COLUMNS" .
" WHERE TABLE_SCHEMA = ?" .
- " AND (COLLATION_NAME <> 'utf8_bin' OR CHARACTER_SET_NAME <> 'utf8')" .
+ " AND (COLLATION_NAME <> '" . $characterSet . "_bin' OR CHARACTER_SET_NAME <> '" . $characterSet . "')" .
" AND TABLE_NAME LIKE \"*PREFIX*%\"",
array($dbName)
);
diff --git a/lib/private/Server.php b/lib/private/Server.php
index 291714b23d1..063ff4a3d3b 100644
--- a/lib/private/Server.php
+++ b/lib/private/Server.php
@@ -407,8 +407,8 @@ class Server extends ServerContainer implements IServerContainer {
return new CredentialsManager($c->getCrypto(), $c->getDatabaseConnection());
});
$this->registerService('DatabaseConnection', function (Server $c) {
- $factory = new \OC\DB\ConnectionFactory();
$systemConfig = $c->getSystemConfig();
+ $factory = new \OC\DB\ConnectionFactory($systemConfig);
$type = $systemConfig->getValue('dbtype', 'sqlite');
if (!$factory->isValidType($type)) {
throw new \OC\DatabaseException('Invalid database type');
diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php
index 4ad6926c2d7..c022616d8b3 100644
--- a/lib/private/Setup/MySQL.php
+++ b/lib/private/Setup/MySQL.php
@@ -58,8 +58,9 @@ class MySQL extends AbstractDatabase {
try{
$name = $this->dbName;
$user = $this->dbUser;
- //we can't use OC_BD functions here because we need to connect as the administrative user.
- $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET utf8 COLLATE utf8_bin;";
+ //we can't use OC_DB functions here because we need to connect as the administrative user.
+ $characterSet = \OC::$server->getSystemConfig()->getValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
+ $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET $characterSet COLLATE ${characterSet}_bin;";
$connection->executeUpdate($query);
} catch (\Exception $ex) {
$this->logger->error('Database creation failed: {error}', [
--
GitLab