From c98cd8230a34ed420cff7092035032506b85cca4 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 6 Feb 2023 23:05:30 +0100 Subject: [PATCH] [fix] mariadb to postgres migration --- scripts/_common.sh | 73 +++++++++++++++++++++++++++++++++------------- scripts/upgrade | 57 ++++++++++++++++++++++++++++-------- 2 files changed, 97 insertions(+), 33 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index b7d3ca7..9955e3f 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -11,41 +11,72 @@ pkg_dependencies="postgresql postgresql-contrib pgloader" # PERSONAL HELPERS #================================================= -mysql-to-pg() { - - if mysqlshow | grep -q "^| $db_name "; then +mariadb-to-pg() { ynh_script_progression --message="Migrating to PostgreSQL database..." --weight=10 mysqlpwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) + + # In old instance db_user is `mmuser` + mysql_db_user="$db_user" + if ynh_mysql_connect_as --user="mmuser" --password="$mysqlpwd" 2> /dev/null <<< ";"; then + mysql_db_user="mmuser" + fi + # Initialize Postgresql db ynh_psql_test_if_first_run ynh_psql_setup_db --db_user=$db_user --db_name=$db_name --db_pwd=$mysqlpwd psqlpwd=$(ynh_app_setting_get --app=$app --key=psqlpwd) + # Configure the new db and run mattermost in order to create tables + ynh_write_var_in_file --file="$final_path/config/config.json" --key="DriverName" --value="postgres" --after="SqlSettings" + ynh_write_var_in_file --file="$final_path/config/config.json" --key="DataSource" --value="postgres://$db_user:$psqlpwd@localhost:5432/$db_name?sslmode=disable&connect_timeout=10" --after="SqlSettings" + cat "$final_path/config/config.json" + pushd $final_path + ynh_systemd_action --service_name="$app" --action="stop" + set +e + sudo -u mattermost timeout --preserve-status 30 "./bin/mattermost" + if [ "$?" != "0" ] && [ "$?" != "143" ] ; then + ynh_die --message="Mattermost creation failed on mattermost running" --ret_code=1 + fi + set -e + popd - # Migrating from MySQL to PostgreSQL - pgloader mysql://$db_user:$mysqlpwd@localhost:3306/$db_name postgresql://$db_user@localhost:5432/$db_name + # Some fixes to let the mariadb -> postgresql conversion working + ynh_psql_execute_as_root --sql='DROP INDEX public.idx_fileinfo_content_txt;' --database=mattermost + ynh_psql_execute_as_root --sql='DROP INDEX public.idx_posts_message_txt;' --database=mattermost + ynh_mysql_execute_as_root --sql="ALTER TABLE mattermost.Users DROP COLUMN IF EXISTS acceptedtermsofserviceid;" --database=mattermost + + # Migrating from MySQL to PostgreSQL + tmpdir="$(mktemp -d)" + + cat < $tmpdir/commands.load +LOAD DATABASE + FROM mysql://$mysql_db_user:$mysqlpwd@127.0.0.1:3306/$db_name + INTO postgresql://$db_user:$psqlpwd@127.0.0.1:5432/$db_name + +WITH include no drop, truncate, create no tables, + create no indexes, preserve index names, no foreign keys, + data only, workers = 16, concurrency = 1 + +SET MySQL PARAMETERS +net_read_timeout = '90', +net_write_timeout = '180' + +ALTER SCHEMA '$db_name' RENAME TO 'public' + +; +EOT + pgloader $tmpdir/commands.load + + # Rebuild INDEX + ynh_psql_execute_as_root --sql='CREATE INDEX idx_fileinfo_content_txt ON public.fileinfo USING gin (to_tsvector('\''english'\''::regconfig, content))' --database=mattermost + ynh_psql_execute_as_root --sql='CREATE INDEX idx_posts_message_txt ON public.posts USING gin (to_tsvector('\''english'\''::regconfig, (message)::text));' --database=mattermost - ynh_replace_string --match_string="\"DriverName\":*," --replace_string="\"DriverName\": \"postgres\"," --target_file="$final_path/config/config.json" - ynh_replace_string --match_string="\"DataSource\":*," --replace_string="\"DataSource\": \"postgres://$db_user:$psqlpwd@localhost:5432/$db_name?sslmode=disable&connect_timeout=10\"," --target_file="$final_path/config/config.json" - - #================================================= - # MODIFY A CONFIG FILE - #================================================= - #ynh_script_progression --message="Modifying a config file..." --weight=3 - - #smtp_user_pwd=$(ynh_string_random --length=24) - #url=https://$domain$path_url - - #ynh_add_config --template="../conf/config.json" --destination="$final_path/config/config.json" # Removinging MySQL database - ynh_mysql_remove_db --db_user=$db_user --db_name=$db_name + ynh_mysql_remove_db --db_user=$mysql_db_user --db_name=$db_name - else - ynh_print_info --message="No migration needed" - fi } #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 7c783d1..fea6ce5 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -102,6 +102,24 @@ if ynh_legacy_permissions_exists; then ynh_app_setting_delete --app=$app --key=is_public fi +#================================================= +# UPGRADE DEPENDENCIES +#================================================= +ynh_script_progression --message="Upgrading dependencies..." --weight=10 + +ynh_exec_warn_less ynh_install_app_dependencies $pkg_dependencies + +#================================================= +# MIGRATING DATABASE +#================================================= + +# Migrating from MySQL to PostgreSQL +# This migration should be done before the upgrade +if mysqlshow | grep -q "^| $db_name "; then + # Mattermost only support mysql and postgresql (not mariadb...) + mariadb-to-pg +fi + #================================================= # STANDARD UPGRADE STEPS #================================================= @@ -163,19 +181,7 @@ ynh_script_progression --message="Upgrading NGINX web server configuration..." - # Create a dedicated NGINX config ynh_add_nginx_config -#================================================= -# UPGRADE DEPENDENCIES -#================================================= -ynh_script_progression --message="Upgrading dependencies..." --weight=10 -ynh_exec_warn_less ynh_install_app_dependencies $pkg_dependencies - -#================================================= -# MIGRATING DATABASE -#================================================= - -# Migrating from MySQL to PostgreSQL -mysql-to-pg #================================================= # SETUP SYSTEMD @@ -185,6 +191,33 @@ ynh_script_progression --message="Upgrading systemd configuration..." --weight=2 # Create a dedicated systemd config ynh_add_systemd_config +#================================================= +# Fix old migrations +#================================================= +# Crazy fix for old unupgraded version +# IMPORTANT: THIS fix should be done after setup new sources and running mattermost +if ynh_compare_current_package_version --comparison lt --version 5.37.1~ynh1 +then + read -r -d '' fix_old_version_sql << EOM +ALTER TABLE ChannelMembers ALTER COLUMN mentioncountroot SET DEFAULT '0'::bigint; +UPDATE ChannelMembers SET mentioncountroot=0 WHERE mentioncountroot IS NULL; +ALTER TABLE ChannelMembers ALTER COLUMN msgcountroot SET DEFAULT '0'::bigint; +UPDATE ChannelMembers SET msgcountroot=0 WHERE msgcountroot IS NULL; +ALTER TABLE Channels ALTER COLUMN totalmsgcountroot SET DEFAULT '0'::bigint; +UPDATE Channels SET totalmsgcountroot=0 WHERE totalmsgcountroot IS NULL; +UPDATE SidebarCategories SET collapsed=False where collapsed IS NULL; +UPDATE SidebarCategories SET muted=False where muted IS NULL; +UPDATE SidebarCategories set sorting = 'manual' where sorting=''; +UPDATE SidebarCategories set sorting = 'manual' where sorting IS NULL; +EOM + ynh_psql_execute_as_root --sql="$fix_old_version_sql" --database=mattermost + + # Note: it's possible that some instances need other fixes + # If nothing is displayed in the sidebar it may be needed to change the Id of SidebarCategories... +fi + + + #================================================= # SETUP LOGROTATE #=================================================