From b6e2057179a4668c7329315991deda9a6b41b792 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 25 Feb 2021 12:19:46 +0100 Subject: [PATCH 01/11] Try to migrate mysql db to utf8 --- scripts/install | 3 +++ scripts/upgrade | 1 + 2 files changed, 4 insertions(+) diff --git a/scripts/install b/scripts/install index 1e1621d..2031b37 100755 --- a/scripts/install +++ b/scripts/install @@ -66,6 +66,9 @@ db_name=$(ynh_sanitize_dbid --db_name=$app) ynh_app_setting_set --app=$app --key=db_name --value=$db_name ynh_mysql_setup_db --db_user=$db_name --db_name=$db_name +ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ + <<< "ALTER DATABASE $db_name CHARACTER SET utf8 COLLATE utf8_general_ci;" + #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 5805432..0569ce4 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -221,6 +221,7 @@ then # Print the current version number of Nextcloud exec_occ -V + exec_occ db:convert-mysql-charset # Upgrade may fail if this app is enabled # Take all apps enabled, and check if mail is one of them From a649f030d32365f6f165d39347a84e652ea7a484 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 8 Mar 2021 16:20:31 +0100 Subject: [PATCH 02/11] utf8 -> utf8mb4 https://github.com/YunoHost-Apps/nextcloud_ynh/pull/384#issuecomment-792766121 --- scripts/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index 2031b37..f010b25 100755 --- a/scripts/install +++ b/scripts/install @@ -67,7 +67,7 @@ ynh_app_setting_set --app=$app --key=db_name --value=$db_name ynh_mysql_setup_db --db_user=$db_name --db_name=$db_name ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ - <<< "ALTER DATABASE $db_name CHARACTER SET utf8 COLLATE utf8_general_ci;" + <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8_general_ci;" #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE From 4f66d838b94cb0fb52abde121547f1998b4357c5 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 11 Mar 2021 13:45:21 +0100 Subject: [PATCH 03/11] fix sql --- scripts/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index f010b25..983dd65 100755 --- a/scripts/install +++ b/scripts/install @@ -67,7 +67,7 @@ ynh_app_setting_set --app=$app --key=db_name --value=$db_name ynh_mysql_setup_db --db_user=$db_name --db_name=$db_name ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ - <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8_general_ci;" + <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE From 80bb0a1051f8f60fbd9c893d7fbcfc3f855fb024 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 18 Mar 2021 11:22:26 +0100 Subject: [PATCH 04/11] add default_character_set parameter --- scripts/_common.sh | 1 + scripts/_ynh_mysql_connect_as.sh | 34 ++++++++++++++++++++++++++++++++ scripts/_ynh_mysql_dump_db.sh | 29 +++++++++++++++++++++++++++ scripts/backup | 3 ++- scripts/install | 3 ++- scripts/restore | 6 +++++- scripts/upgrade | 4 +++- 7 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 scripts/_ynh_mysql_connect_as.sh create mode 100644 scripts/_ynh_mysql_dump_db.sh diff --git a/scripts/_common.sh b/scripts/_common.sh index ebffb0e..bd77d63 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,3 +1,4 @@ +#!/bin/bash #================================================= # COMMON VARIABLES diff --git a/scripts/_ynh_mysql_connect_as.sh b/scripts/_ynh_mysql_connect_as.sh new file mode 100644 index 0000000..0176800 --- /dev/null +++ b/scripts/_ynh_mysql_connect_as.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Open a connection as a user +# +# example: ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;" +# example: ynh_mysql_connect_as --user="user" --password="pass" --default_character_set="utf8mb4" < /path/to/file.sql +# +# usage: ynh_mysql_connect_as --user=user --password=password [--database=database] [--default_character_set=character-set] +# | arg: -u, --user= - the user name to connect as +# | arg: -p, --password= - the user password +# | arg: -d, --database= - the database to connect to +# | arg: -c, --default_character_set= - the charset to use +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=updc + local -A args_array=( [u]=user= [p]=password= [d]=database= [c]=default_character_set= ) + local user + local password + local database + local default_character_set + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + default_character_set="${default_character_set:-}" + + if [ -n "$default_character_set" ] + then + default_character_set="--default-character-set=$default_character_set" + fi + + mysql --user="$user" --password="$password" "$default_character_set" --batch "$database" +} \ No newline at end of file diff --git a/scripts/_ynh_mysql_dump_db.sh b/scripts/_ynh_mysql_dump_db.sh new file mode 100644 index 0000000..743c96a --- /dev/null +++ b/scripts/_ynh_mysql_dump_db.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Dump a database +# +# example: ynh_mysql_dump_db --database=roundcube --default_character_set="utf8mb4" > ./dump.sql +# +# usage: ynh_mysql_dump_db --database=database +# | arg: -d, --database= - the database name to dump +# | arg: -c, --default_character_set= - the charset to use +# | ret: the mysqldump output +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=dc + local -A args_array=( [d]=database= [c]=default_character_set= ) + local database + local default_character_set + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + default_character_set="${default_character_set:-}" + + if [ -n "$default_character_set" ] + then + default_character_set="--default-character-set=$default_character_set" + fi + + mysqldump --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$default_character_set" "$database" +} \ No newline at end of file diff --git a/scripts/backup b/scripts/backup index 7ed3c7f..6a115ba 100755 --- a/scripts/backup +++ b/scripts/backup @@ -8,6 +8,7 @@ source ../settings/scripts/_common.sh source /usr/share/yunohost/helpers +source ../settings/scripts/_ynh_mysql_dump_db.sh #================================================= # MANAGE SCRIPT FAILURE @@ -56,7 +57,7 @@ ynh_backup --src_path="/etc/php/$phpversion/fpm/pool.d/$app.conf" #================================================= ynh_print_info --message="Backing up the MySQL database..." -ynh_mysql_dump_db --database="$db_name" > db.sql +ynh_mysql_dump_db --database="$db_name" --default_character_set="utf8mb4" > db.sql #================================================= # SPECIFIC BACKUP diff --git a/scripts/install b/scripts/install index 983dd65..8d444fb 100755 --- a/scripts/install +++ b/scripts/install @@ -8,6 +8,7 @@ source _common.sh source /usr/share/yunohost/helpers +source _ynh_mysql_connect_as.sh #================================================= # MANAGE SCRIPT FAILURE @@ -250,7 +251,7 @@ ynh_replace_string --match_string="'overwrite.cli.url' => 'http://localhost'," - #================================================= # Set the user as admin -ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ +ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name --default_character_set="utf8mb4" \ <<< "INSERT INTO oc_group_user VALUES ('admin','$admin');" # And delete admin user exec_occ user:delete admin diff --git a/scripts/restore b/scripts/restore index 44f8384..5dea17e 100755 --- a/scripts/restore +++ b/scripts/restore @@ -8,6 +8,7 @@ source ../settings/scripts/_common.sh source /usr/share/yunohost/helpers +source ../settings/scripts/_ynh_mysql_connect_as.sh #================================================= # MANAGE SCRIPT FAILURE @@ -73,7 +74,10 @@ ynh_script_progression --message="Restoring the MySQL database..." --weight=9 db_pwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) ynh_mysql_setup_db --db_user=$db_name --db_name=$db_name --db_pwd=$db_pwd -ynh_mysql_connect_as --user=$db_name --password=$db_pwd --database=$db_name < ./db.sql +ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ + <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" + +ynh_mysql_connect_as --user=$db_name --password=$db_pwd --database=$db_name --default_character_set="utf8mb4" < ./db.sql #================================================= # RECREATE THE DEDICATED USER diff --git a/scripts/upgrade b/scripts/upgrade index 0569ce4..7ab6a0f 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -221,7 +221,9 @@ then # Print the current version number of Nextcloud exec_occ -V - exec_occ db:convert-mysql-charset + if [ "$(exec_occ config:system:get mysql.utf8mb4)" != "true" ]; then + exec_occ db:convert-mysql-charset + fi # Upgrade may fail if this app is enabled # Take all apps enabled, and check if mail is one of them From 4ba5a6516b841e706620287a248e8a03d02cdb19 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 18 Mar 2021 14:50:44 +0100 Subject: [PATCH 05/11] fix helpers --- scripts/_ynh_mysql_connect_as.sh | 2 ++ scripts/_ynh_mysql_dump_db.sh | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/_ynh_mysql_connect_as.sh b/scripts/_ynh_mysql_connect_as.sh index 0176800..a22963e 100644 --- a/scripts/_ynh_mysql_connect_as.sh +++ b/scripts/_ynh_mysql_connect_as.sh @@ -28,6 +28,8 @@ ynh_mysql_connect_as() { if [ -n "$default_character_set" ] then default_character_set="--default-character-set=$default_character_set" + else + default_character_set="--default-character-set=latin1" fi mysql --user="$user" --password="$password" "$default_character_set" --batch "$database" diff --git a/scripts/_ynh_mysql_dump_db.sh b/scripts/_ynh_mysql_dump_db.sh index 743c96a..387a7e3 100644 --- a/scripts/_ynh_mysql_dump_db.sh +++ b/scripts/_ynh_mysql_dump_db.sh @@ -5,7 +5,7 @@ # example: ynh_mysql_dump_db --database=roundcube --default_character_set="utf8mb4" > ./dump.sql # # usage: ynh_mysql_dump_db --database=database -# | arg: -d, --database= - the database name to dump +# | arg: -d, --database= - the database name to dump # | arg: -c, --default_character_set= - the charset to use # | ret: the mysqldump output # @@ -23,7 +23,14 @@ ynh_mysql_dump_db() { if [ -n "$default_character_set" ] then default_character_set="--default-character-set=$default_character_set" + else + # By default, default character set is "latin1" + default_character_set="--default-character-set=latin1" + fi + + if [ -f "$MYSQL_ROOT_PWD_FILE" ]; then + mysqldump --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$default_character_set" "$database" + else + mysqldump --single-transaction --skip-dump-date "$default_character_set" "$database" fi - - mysqldump --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$default_character_set" "$database" } \ No newline at end of file From 2765ab4ef27410bbeffc437a957a235285b62345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sat, 16 Jul 2022 15:49:34 +0200 Subject: [PATCH 06/11] Fix --- scripts/install | 3 +++ scripts/upgrade | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index 345082b..7a1d8d7 100755 --- a/scripts/install +++ b/scripts/install @@ -162,6 +162,9 @@ chown -R $app: "$final_path" "$datadir" # picked up by Yunohost's auto-redact mecanism admin_password="$(ynh_string_random --length=6)" +# Set the mysql.utf8mb4 config to true in config.php +exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" + # Install Nextcloud using a temporary admin user exec_occ maintenance:install \ --database "mysql" --database-name $db_name \ diff --git a/scripts/upgrade b/scripts/upgrade index a12f13a..26b9981 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -246,7 +246,7 @@ then exec_occ -V if [ "$(exec_occ config:system:get mysql.utf8mb4)" != "true" ]; then - exec_occ db:convert-mysql-charset + exec_occ maintenance:repair fi # Upgrade may fail if this app is enabled From 79cdc2a70a5031841af1c33b89bc7d55d860ad1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sat, 16 Jul 2022 16:02:39 +0200 Subject: [PATCH 07/11] Update install --- scripts/install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install b/scripts/install index 7a1d8d7..dd4ac31 100755 --- a/scripts/install +++ b/scripts/install @@ -162,9 +162,6 @@ chown -R $app: "$final_path" "$datadir" # picked up by Yunohost's auto-redact mecanism admin_password="$(ynh_string_random --length=6)" -# Set the mysql.utf8mb4 config to true in config.php -exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" - # Install Nextcloud using a temporary admin user exec_occ maintenance:install \ --database "mysql" --database-name $db_name \ @@ -294,6 +291,9 @@ exec_occ db:add-missing-indices exec_occ db:add-missing-columns exec_occ db:convert-filecache-bigint -n +# Set the mysql.utf8mb4 config to true in config.php +exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" + #================================================= # YUNOHOST MULTIMEDIA INTEGRATION #================================================= From bd99046423ebea379f6f4c5de1b2de7e37ab9f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sat, 16 Jul 2022 17:32:43 +0200 Subject: [PATCH 08/11] Update _ynh_mysql_dump_db.sh --- scripts/_ynh_mysql_dump_db.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/_ynh_mysql_dump_db.sh b/scripts/_ynh_mysql_dump_db.sh index 387a7e3..3b922bb 100644 --- a/scripts/_ynh_mysql_dump_db.sh +++ b/scripts/_ynh_mysql_dump_db.sh @@ -19,6 +19,7 @@ ynh_mysql_dump_db() { # Manage arguments with getopts ynh_handle_getopts_args "$@" default_character_set="${default_character_set:-}" + MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql if [ -n "$default_character_set" ] then From f233e555b514c81982665e828f17da252590e098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sun, 17 Jul 2022 09:46:59 +0200 Subject: [PATCH 09/11] Set mysql.utf8mb4 to true --- scripts/upgrade | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/upgrade b/scripts/upgrade index 26b9981..9029dfd 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -246,6 +246,8 @@ then exec_occ -V if [ "$(exec_occ config:system:get mysql.utf8mb4)" != "true" ]; then + # Set the mysql.utf8mb4 config to true in config.php + exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" exec_occ maintenance:repair fi From 208c22d13094797065525a9e6db443636109eb8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sun, 17 Jul 2022 09:54:48 +0200 Subject: [PATCH 10/11] CChange database character set and collation --- scripts/upgrade | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/upgrade b/scripts/upgrade index 9029dfd..c9e0414 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -21,6 +21,7 @@ path_url=$(ynh_app_setting_get --app=$app --key=path) admin=$(ynh_app_setting_get --app=$app --key=admin) final_path=$(ynh_app_setting_get --app=$app --key=final_path) db_name=$(ynh_app_setting_get --app=$app --key=db_name) +db_user=$db_name user_home=$(ynh_app_setting_get --app=$app --key=user_home) phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) @@ -246,6 +247,10 @@ then exec_occ -V if [ "$(exec_occ config:system:get mysql.utf8mb4)" != "true" ]; then + db_pwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) + # Change your databases character set and collation + ynh_mysql_connect_as --user=$db_user --password="$db_pwd" --database=$db_name \ + <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" # Set the mysql.utf8mb4 config to true in config.php exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" exec_occ maintenance:repair From b5c526be5c24486608d03bc11f44f142f2b82ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sun, 17 Jul 2022 09:59:21 +0200 Subject: [PATCH 11/11] Update install --- scripts/install | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/install b/scripts/install index dd4ac31..d4fe45b 100755 --- a/scripts/install +++ b/scripts/install @@ -65,10 +65,11 @@ ynh_install_app_dependencies $pkg_dependencies ynh_script_progression --message="Creating a MySQL database..." --weight=2 db_name=$(ynh_sanitize_dbid --db_name=$app) +db_user=$db_name ynh_app_setting_set --app=$app --key=db_name --value=$db_name -ynh_mysql_setup_db --db_user=$db_name --db_name=$db_name +ynh_mysql_setup_db --db_user=$db_user --db_name=$db_name -ynh_mysql_connect_as --user=$db_name --password="$db_pwd" --database=$db_name \ +ynh_mysql_connect_as --user=$db_user --password="$db_pwd" --database=$db_name \ <<< "ALTER DATABASE $db_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" #================================================= @@ -175,6 +176,9 @@ exec_occ maintenance:install \ #================================================= ynh_script_progression --message="Configuring Nextcloud..." --weight=8 +# Set the mysql.utf8mb4 config to true in config.php +exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" + # Ensure that UpdateNotification app is disabled exec_occ app:disable updatenotification @@ -291,9 +295,6 @@ exec_occ db:add-missing-indices exec_occ db:add-missing-columns exec_occ db:convert-filecache-bigint -n -# Set the mysql.utf8mb4 config to true in config.php -exec_occ config:system:set mysql.utf8mb4 --type boolean --value="true" - #================================================= # YUNOHOST MULTIMEDIA INTEGRATION #=================================================