From fb2c6fbbb0f73cb2d943ff59440da4f81573e553 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 1 Sep 2017 03:03:17 +0200 Subject: [PATCH] Fix owncloud migration + new helper ynh_handle_app_migration --- check_process | 8 +- conf/owncloud-migration.sh | 21 --- conf/owncloud_migration | 13 ++ conf/owncloud_post_migration.sh | 45 ++++++ scripts/_common.sh | 244 +++++++++++++++++++++++++++++--- scripts/upgrade | 115 +++++++-------- scripts/upgrade.d/owncloud.sh | 84 ----------- 7 files changed, 338 insertions(+), 192 deletions(-) delete mode 100644 conf/owncloud-migration.sh create mode 100644 conf/owncloud_migration create mode 100644 conf/owncloud_post_migration.sh delete mode 100755 scripts/upgrade.d/owncloud.sh diff --git a/check_process b/check_process index 83318ca..dc17706 100644 --- a/check_process +++ b/check_process @@ -21,10 +21,10 @@ Level 1=auto Level 2=auto Level 3=auto -# Level 4: - Level 4=0 -# Level 5: - Level 5=0 +# Level 4: LDAP and http auth + Level 4=1 +# Level 5: https://github.com/YunoHost-Apps/nextcloud_ynh/issues/58 + Level 5=1 Level 6=auto Level 7=auto Level 8=0 diff --git a/conf/owncloud-migration.sh b/conf/owncloud-migration.sh deleted file mode 100644 index 6829b3b..0000000 --- a/conf/owncloud-migration.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -u - -app="#APP#" - -# rename hook -hooks_dir="/etc/yunohost/hooks.d/post_user_create" -[[ -f "${hooks_dir}/50-${app}" ]] \ - && mv "${hooks_dir}/50-${app}" "${hooks_dir}/50-nextcloud" - -# move yunohost app settings -apps_dir="/etc/yunohost/apps" -if [[ -d "${apps_dir}/${app}" ]]; then - yunohost app setting "$app" id -v nextcloud - mv "${apps_dir}/${app}" "${apps_dir}/nextcloud" - yunohost app ssowatconf --quiet -fi - -# remove cron job -rm /etc/cron.d/owncloud-migration diff --git a/conf/owncloud_migration b/conf/owncloud_migration new file mode 100644 index 0000000..4c1be85 --- /dev/null +++ b/conf/owncloud_migration @@ -0,0 +1,13 @@ +# File to migrate from Owncloud + +# Final path +/var/www/$app + +# Data directory +/home/yunohost.app/$app + +# Nginx config +/etc/nginx/conf.d/$domain.d/$app.conf + +# php-fpm config +/etc/php5/fpm/pool.d/$app.conf diff --git a/conf/owncloud_post_migration.sh b/conf/owncloud_post_migration.sh new file mode 100644 index 0000000..952baac --- /dev/null +++ b/conf/owncloud_post_migration.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Ending the migration process from Owncloud to Nextcloud + +set -u + +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source /usr/share/yunohost/helpers + +#================================================= +# SET VARIABLES +#================================================= + +old_app="__OLD_APP__" +new_app="__NEW_APP__" +script_name="$0" + +#================================================= +# MOVE HOOKS +#================================================= + +hooks_dir="/etc/yunohost/hooks.d/" +mv "$hooks_dir/post_user_create/50-$old_app" "$hooks_dir/post_user_create/50-$new_app" + +#================================================= +# DELETE OLD APP'S SETTINGS +#================================================= + +ynh_secure_remove "/etc/yunohost/apps/$old_app" +yunohost app ssowatconf + +#================================================= +# REMOVE THE OLD USER +#================================================= + +ynh_system_user_delete $old_app + +#================================================= +# DELETE THIS SCRIPT +#================================================= + +echo "rm $script_name" | at now + 1 minutes diff --git a/scripts/_common.sh b/scripts/_common.sh index 585c19d..6622e25 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -9,19 +9,6 @@ dependencies="php5-gd php5-json php5-intl php5-mcrypt php5-curl php5-apcu php5-i # COMMON HELPERS #================================================= -# Execute a command as another user -# usage: exec_as USER COMMAND [ARG ...] -exec_as() { - local USER=$1 - shift 1 - - if [[ $USER = $(whoami) ]]; then - eval "$@" - else - sudo -u "$USER" "$@" - fi -} - # Execute a command with occ exec_occ() { (cd "$final_path" && exec_as "$app" \ @@ -37,15 +24,6 @@ create_home_external_storage() { || exec_occ files_external:option "$mount_id" enable_sharing true } -# Check if an URL is already handled -# usage: is_url_handled URL -is_url_handled() { - local output=($(curl -k -s -o /dev/null \ - -w 'x%{redirect_url} %{http_code}' "$1")) - # It's handled if it does not redirect to the SSO nor return 404 - [[ ! ${output[0]} =~ \/yunohost\/sso\/ && ${output[1]} != 404 ]] -} - # Rename a MySQL database and user # Usage: rename_mysql_db DBNAME DBUSER DBPASS NEW_DBNAME_AND_USER rename_mysql_db() { @@ -63,3 +41,225 @@ rename_mysql_db() { ynh_mysql_remove_db $db_name $db_name ynh_secure_remove "$sqlpath" } + +#================================================= +# COMMON HELPERS -- SHOULD BE ADDED TO YUNOHOST +#================================================= + +# Execute a command as another user +# usage: exec_as USER COMMAND [ARG ...] +exec_as() { + local USER=$1 + shift 1 + + if [[ $USER = $(whoami) ]]; then + eval "$@" + else + sudo -u "$USER" "$@" + fi +} + +# Check if an URL is already handled +# usage: is_url_handled URL +is_url_handled() { + local output=($(curl -k -s -o /dev/null \ + -w 'x%{redirect_url} %{http_code}' "$1")) + # It's handled if it does not redirect to the SSO nor return 404 + [[ ! ${output[0]} =~ \/yunohost\/sso\/ && ${output[1]} != 404 ]] +} + +# ynh_handle_app_migration "ID FROM WHICH TO MIGRATE" "MIGRATION FILE" +# +# WARNING You have to replace manually any reference to a moved file in the settings.yml +# Also for the config of the app or anything else that not handled by this helper. +# +ynh_handle_app_migration () { + #================================================= + # LOAD SETTINGS + #================================================= + + old_app=$YNH_APP_INSTANCE_NAME + local old_app_id=$YNH_APP_ID + local old_app_number=$YNH_APP_INSTANCE_NUMBER + + # Get the id from which to migrate + local migration_id="$1" + # And the file with the paths to move + local migration_list="$2" + + # Get the new app id in the manifest + local new_app_id=$(grep \"id\": ../manifest.json | cut -d\" -f4) + if [ $old_app_number -eq 1 ]; then + local new_app=$new_app_id + else + local new_app=${new_app_id}__${old_app_number} + fi + + #================================================= + # CHECK IF IT HAS TO MIGRATE + #================================================= + + migration_process=0 + + if [ "$old_app_id" == "$new_app_id" ] + then + # If the 2 id are the same + # No migration to do. + echo 0 + return 0 + else + if [ "$old_app_id" != "$migration_id" ] + then + # If the new app is not the authorized id, fail. + ynh_die "Incompatible application for migration from $old_app_id to $new_app_id" + fi + + echo "Migrate from $old_app_id to $new_app_id" >&2 + + #================================================= + # CHECK IF THE MIGRATION CAN BE DONE + #================================================= + + # TODO Handle multi instance apps... + # Check that there not already an app installed for this id. + (yunohost app list --installed -f "$new_app" | grep -q id) \ + && ynh_die "$new_app is already installed" + + #================================================= + # CHECK THE LIST OF FILES TO MOVE + #================================================= + + local temp_migration_list="$(tempfile)" + + # Build the list by removing blank lines and comment lines + sed '/^#.*\|^$/d' "../conf/$migration_list" > "$temp_migration_list" + + # Check if there no files in the destination + local file_to_move="" + while read file_to_move + do + # Replace all occurence of $app by $new_app in each file to move. + local move_to_destination="${file_to_move//\$app/$new_app}" + test -e "$move_to_destination" && ynh_die "A file named $move_to_destination already exist." + done < "$temp_migration_list" + + #================================================= + # COPY THE YUNOHOST SETTINGS FOR THIS APP + #================================================= + + local settings_dir="/etc/yunohost/apps" + cp -a "$settings_dir/$old_app" "$settings_dir/$new_app" + + # Replace the old id by the new one + ynh_replace_string "\(^id: .*\)$old_app" "\1$new_app" "$settings_dir/$new_app/settings.yml" + # INFO: There a special behavior with yunohost app setting + # if the id given in argument does not match with the id + # stored in the config file. The config file will be purged. + # That's why we use sed instead of app setting here. + # https://github.com/YunoHost/yunohost/blob/c6b5284be8da39cf2da4e1036a730eb5e0515096/src/yunohost/app.py#L1316-L1321 + + #================================================= + # MOVE FILES TO THE NEW DESTINATION + #================================================= + + while read file_to_move + do + # Replace all occurence of $app by $new_app in each file to move. + move_to_destination="$(eval echo "${file_to_move//\$app/$new_app}")" + local real_file_to_move="$(eval echo "${file_to_move//\$app/$old_app}")" + echo "Move the file $real_file_to_move to $move_to_destination" >&2 + mv "$real_file_to_move" "$move_to_destination" + done < "$temp_migration_list" + + #================================================= + # UPDATE KNOWN FILES IN THE SETTINGS + #================================================= + + # Replace nginx checksum + ynh_replace_string "\(^checksum__etc_nginx.*\)_$old_app" "\1_$new_app/" "$settings_dir/$new_app/settings.yml" + + # Replace php5-fpm checksums + ynh_replace_string "\(^checksum__etc_php5.*[-_]\)$old_app" "\1$new_app/" "$settings_dir/$new_app/settings.yml" + + # Replace final_path + ynh_replace_string "\(^final_path: .*\)$old_app" "\1$new_app" "$settings_dir/$new_app/settings.yml" + + #================================================= + # MOVE THE DATABASE + #================================================= + + db_pwd=$(ynh_app_setting_get $old_app mysqlpwd) + db_name=$(ynh_app_setting_get $old_app db_name) + + # Check if a database exist before trying to move it + local mysql_root_password=$(cat $MYSQL_ROOT_PWD_FILE) + if [ -n "$db_name" ] && mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name" + then + new_db_name=$(ynh_sanitize_dbid $new_app) + echo "Rename the database $db_name to $new_db_name" >&2 + + local sql_dump="/tmp/${db_name}-$(date '+%s').sql" + + # Dump the old database + ynh_mysql_dump_db "$db_name" > "$sql_dump" + + # Create a new database + ynh_mysql_setup_db $new_db_name $new_db_name $db_pwd + # Then restore the old one into the new one + ynh_mysql_connect_as $new_db_name $db_pwd $new_db_name < "$sql_dump" + + # Remove the old database + ynh_mysql_remove_db $db_name $db_name + # And the dump + ynh_secure_remove "$sql_dump" + + # Update the value of $db_name + db_name=$new_db_name + ynh_app_setting_set $new_app db_name $db_name + fi + + #================================================= + # CREATE A NEW USER + #================================================= + + # Check if the user exists on the system + if ynh_system_user_exists "$old_app" + then + echo "Create a new user $new_app to replace $old_app" >&2 + ynh_system_user_create $new_app + fi + + #================================================= + # CHANGE THE FAKE DEPENDENCIES PACKAGE + #================================================= + + # Check if a variable $dependencies exists + # If this variable doesn't exist, this part shall be manage in the upgrade script. + if [ -n "${dependencies:-}" ] + then + # Define the name of the package + local old_package_name="${old_app//_/-}-ynh-deps" + local new_package_name="${new_app//_/-}-ynh-deps" + + if ynh_package_is_installed "$old_package_name" + then + # Install a new fake package + app=$new_app + ynh_install_app_dependencies $dependencies + # Then remove the old one + app=$old_app + ynh_remove_app_dependencies + fi + fi + + #================================================= + # UPDATE THE ID OF THE APP + #================================================= + + app=$new_app + + + # Set migration_process to 1 to inform that an upgrade has been made + migration_process=1 + fi +} diff --git a/scripts/upgrade b/scripts/upgrade index 538dcf0..3e50167 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -14,8 +14,6 @@ source /usr/share/yunohost/helpers #================================================= app=$YNH_APP_INSTANCE_NAME -# app may contains owncloud instead of nextcloud in case of migration -# You should be really careful with this variable if you want to set multi-instance installation domain=$(ynh_app_setting_get $app domain) path_url=$(ynh_app_setting_get $app path) @@ -24,15 +22,10 @@ final_path=$(ynh_app_setting_get $app final_path) db_name=$(ynh_app_setting_get $app db_name) user_home=$(ynh_app_setting_get $app user_home) -# Define app's data directory -datadir="/home/yunohost.app/${app}/data" - #================================================= # ENSURE DOWNWARD COMPATIBILITY #================================================= -migration_name=nextcloud - # If db_name doesn't exist, create it if [ -z $db_name ]; then db_name=$(ynh_sanitize_dbid $app) @@ -45,26 +38,6 @@ if [ -z $final_path ]; then ynh_app_setting_set $app final_path $final_path fi - -# Handle old migrations from ownCloud -# Get the database name in the config file -nc_conf="$final_path/config/config.php" -curr_dbname=$(grep dbname "$nc_conf" \ - | sed "s|.*=> '\(.*\)'.*|\1|g") -# If the database hasn't the name nextcloud, rename the database -if [ "$curr_dbname" != "$db_name" ] -then - # Get the database user in the config file - curr_dbuser=$(grep dbuser "$nc_conf" \ - | sed "s|.*=> '\(.*\)'.*|\1|g") - db_pwd=$(ynh_app_setting_get "$app" mysqlpwd) - - # Rename the database - rename_mysql_db "$curr_dbname" "$curr_dbuser" "$db_pwd" "$db_name" - ynh_replace_string "^(\s*'dbname' =>).*," "\1 '${db_name}'," "$nc_conf" - ynh_replace_string "#^(\s*'dbuser' =>).*," "${db_name}'," "$nc_conf" -fi - #================================================= # BACKUP BEFORE UPGRADE THEN ACTIVE TRAP #================================================= @@ -74,8 +47,9 @@ fi # Get the current version number of nextcloud/owncloud current_version=$(grep OC_VersionString "$final_path/version.php" | cut -d\' -f2) +current_major_version=${current_version%%.*} -if [ "$current_version" \> "11.0.0" ] +if [ $current_major_version -gt 11 ] then # Inform the backup/restore process that it should not save the data directory ynh_app_setting_set $app backup_core_only 1 @@ -83,6 +57,9 @@ then # Backup the current version of the app ynh_backup_before_upgrade ynh_clean_setup () { + # Remove the post migration script before its execution ! + ynh_secure_remove "/tmp/owncloud_post_migration.sh" 2>&1 + # restore it if the upgrade fails ynh_restore_upgradebackup } @@ -92,17 +69,28 @@ fi ynh_abort_if_errors #================================================= -# MIGRATE FROM OWNCLOUD TO NEXTCLOUD +# HANDLE MIGRATION FROM OWNCLOUD #================================================= -# If the name of the app is not nextcloud ($migration_name), check if it's a migration from owncloud -if [ $YNH_APP_ID != $migration_name ] +ynh_handle_app_migration "owncloud" "owncloud_migration" +if [ $migration_process -eq 1 ] then - [ $YNH_APP_ID == owncloud ] \ - || ynh_die "Incompatible application to migrate to Nextcloud" + # If a migration has been perform + # Reload some values changed by the migration process + final_path=$(ynh_app_setting_get $app final_path) + db_name=$(ynh_app_setting_get $app db_name) - # Prepare owcloud to migrate to nextcloud - ./upgrade.d/owncloud.sh + # Remove the old fake package for owncloud. + # Its name it's not regular, so the migration process can't remove it. + ynh_package_autopurge owncloud-deps + + # Change the database access in the config. + ynh_replace_string "\('dbname' =>\).*" "\1 '$db_name'," "$final_path/config/config.php" + ynh_replace_string "\('dbuser' =>\).*" "\1 '$db_name'," "$final_path/config/config.php" + + # Change the path of the data directory + ynh_replace_string "\('dbuser' =>\).*" "\1 '$db_name'," "$final_path/config/config.php" + ynh_replace_string "\('datadirectory' =>.*\)$old_app" "\1$app" "$final_path/config/config.php" fi #================================================= @@ -120,7 +108,7 @@ path_url=$(ynh_normalize_url_path $path_url) # Delete current nginx configuration to be able to check if .well-known is already served. ynh_remove_nginx_config -ynh_app_setting_delete $app "checksum__etc_nginx_conf.d_$domain.d_$app.conf" +ynh_app_setting_delete $app "checksum__etc_nginx_conf.d_$domain.d_$app.conf" || true # Do not serve .well-known if it's already served on the domain if is_url_handled "https://${domain}/.well-known/caldav" ; then sed -ri '/^location = \/\.well\-known\/(caldav|carddav) \{/,/\}/d' \ @@ -166,6 +154,12 @@ ynh_install_app_dependencies $dependencies source upgrade.d/upgrade.last.sh last_version=$next_version +# Define app's data directory +datadir="/home/yunohost.app/${app}/data" + +# Set write access for the following commands +chown -R $app: "$final_path" "$datadir" + # Print the current version number of nextcloud exec_occ -V @@ -174,7 +168,8 @@ while [ "$last_version" != "$current_version" ] do # The major version is the first part of the version number - major_version=${next_version%%.*} +# major_version=${next_version%%.*} + major_version=${last_version%%.*} current_major_version=${current_version%%.*} # If the current version have the same major version than the next. @@ -186,7 +181,7 @@ do # Load the value for this version source upgrade.d/upgrade.$current_major_version.sh - echo -e "\nUpdate to nextcloud $next_version" >&2 + echo -e "\nUpgrade to nextcloud $next_version" >&2 # Create an app.src for this version of nextcloud cp ../conf/app.src.default ../conf/app.src @@ -212,7 +207,6 @@ do # Replace the old nextcloud by the new one ynh_secure_remove "$final_path" mv "$tmpdir" "$final_path" -ls -alh "$final_path" >&2 # Set write access for the following commands chown -R $app: "$final_path" "$datadir" @@ -327,28 +321,6 @@ for u in $(ynh_user_list); do setfacl -m g:$app:rwx "/home/$u" || true done -#================================================= -# ADD A JOB FOR FINISHING UPGRADING OWNCLOUD -#================================================= - -# Finish ownCloud migration -if [ $YNH_APP_ID == owncloud ] -then - echo "ownCloud has been successfully migrated to Nextcloud! \ -A last scheduled operation will run in a couple of minutes to finish the \ -migration in YunoHost side. Do not proceed any application operation while \ -you don't see Nextcloud as installed." >&2 - - # Install a cron job and script for final migration step - script_path="/usr/local/sbin/owncloud-migration.sh" - ynh_replace_string "#APP#" "$YNH_APP_ID" ../conf/owncloud-migration.sh - cp ../conf/owncloud-migration.sh "$script_path" - chmod 755 "$script_path" - cron_path="/etc/cron.d/owncloud-migration" - echo "*/1 * * * * root $script_path" | tee "$cron_path" >/dev/null - chmod 644 "$cron_path" -fi - #================================================= # WARNING ABOUT THIRD-PARTY APPS #================================================= @@ -379,3 +351,24 @@ ynh_app_setting_set $app skipped_regex \ #================================================= systemctl reload nginx + +#================================================= +# FINISH MIGRATION PROCESS +#================================================= + +if [ $migration_process -eq 1 ] +then + echo "ownCloud has been successfully migrated to Nextcloud! \ +A last scheduled operation will run in a couple of minutes to finish the \ +migration in YunoHost side. Do not proceed any application operation while \ +you don't see Nextcloud as installed." >&2 + + # Execute a post migration script after the end of this upgrade. + # Mainly for some cleaning + script_post_migration=owncloud_post_migration.sh + ynh_replace_string "__OLD_APP__" "$old_app" ../conf/$script_post_migration + ynh_replace_string "__NEW_APP__" "$app" ../conf/$script_post_migration + cp ../conf/$script_post_migration /tmp + chmod +x /tmp/$script_post_migration + (cd /tmp; echo "/tmp/$script_post_migration > /tmp/$script_post_migration.log 2>&1" | at now + 2 minutes) +fi diff --git a/scripts/upgrade.d/owncloud.sh b/scripts/upgrade.d/owncloud.sh deleted file mode 100755 index e31dbd2..0000000 --- a/scripts/upgrade.d/owncloud.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# Prepare the migration from owncloud to nextcloud - -#================================================= -# GENERIC START -#================================================= -# IMPORT GENERIC HELPERS -#================================================= - -source _common.sh -source /usr/share/yunohost/helpers - -#================================================= -# RETRIEVE ARGUMENTS FROM THE MANIFEST -#================================================= - -app=$YNH_APP_INSTANCE_NAME - -domain=$(ynh_app_setting_get $app domain) -oc_dbpass=$(ynh_app_setting_get $app mysqlpwd) -oc_dbname=$app -oc_dbuser=$app - -#================================================= -# CHECK IF THE MIGRATION CAN BE DONE -#================================================= - -# check that Nextcloud is not already installed -(yunohost app list --installed -f "$app" | grep -q id) \ -&& ynh_die "Nextcloud is already installed" - -echo "Migration to nextcloud." >&2 - -#================================================= -# REMOVE NGINX AND PHP-FPM CONFIG FILES -#================================================= - -ynh_remove_nginx_config -ynh_remove_fpm_config - -#================================================= -# REMOVE OLD DEPENDENCIES -#================================================= - -ynh_package_remove owncloud-deps || true - -#================================================= -# DELETE NEXTCLOUD DIRECTORIES -#================================================= - -# Clean new destination and data directories -nextcloud_path="/var/www/$migration_name" -nextcloud_data="/home/yunohost.app/$migration_name/data" -ynh_secure_remove "$nextcloud_path" -ynh_secure_remove "/home/yunohost.app/$migration_name" - -#================================================= -# RENAME OWNCLOUD DIRECTORIES -#================================================= - -mv "/var/www/$app" "$nextcloud_path" -mv "/home/yunohost.app/$app" "/home/yunohost.app/$migration_name" - -#================================================= -# CHANGE THE OWNCLOUD CONFIG -#================================================= - -oc_conf=$nextcloud_path/config/config.php -# Change the path of the data file inf the config -ynh_replace_string "^(\s*'datadirectory' =>).*," "\1 '${DATADIR}'," "$oc_conf" - -# Rename the MySQL database -db_name=$(ynh_sanitize_dbid $migration_name) -rename_mysql_db "$oc_dbname" "$oc_dbuser" "$oc_dbpass" "$db_name" "$db_name" -ynh_replace_string "^(\s*'dbname' =>).*," "\1 '${db_name}'," "$oc_conf" -ynh_replace_string "^(\s*'dbuser' =>).*," "\1 '${db_name}'," "$oc_conf" - -#================================================= -# RENAME OWNCLOUD USER -#================================================= - -groupmod -n "$migration_name" "$app" -usermod -l "$migration_name" "$app"