From c3bb11a4c66d8dcc40a71d7a4eee4f5523bdc6c9 Mon Sep 17 00:00:00 2001 From: zamentur Date: Sat, 3 Oct 2015 15:06:36 +0200 Subject: [PATCH 1/6] [enh] Add name and description in info.json of backup --- lib/yunohost/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index 9220c338c..9e1200f98 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -192,6 +192,8 @@ def backup_create(name=None, description=None, output_directory=None, i = app_info(app_id) info['apps'][app_id] = { 'version': i['version'], + 'name': i['name'], + 'description': i['description'], } finally: filesystem.rm(tmp_script, force=True) From 98e92bdbacba7f93e6750ac44fca99b9c68ac554 Mon Sep 17 00:00:00 2001 From: zamentur Date: Sat, 3 Oct 2015 20:41:37 +0200 Subject: [PATCH 2/6] [fix] Sudo on some backup script --- data/hooks/backup/05-conf_ldap | 6 +++--- data/hooks/backup/08-conf_ssh | 8 ++++++-- data/hooks/backup/11-conf_ynh_mysql | 2 +- data/hooks/backup/14-conf_ssowat | 2 +- data/hooks/backup/17-data_home | 2 +- data/hooks/backup/20-conf_ynh_firewall | 2 +- data/hooks/backup/21-conf_ynh_certs | 2 +- data/hooks/backup/26-conf_xmpp | 2 +- data/hooks/backup/29-conf_nginx | 2 +- data/hooks/backup/32-conf_cron | 2 +- 10 files changed, 17 insertions(+), 13 deletions(-) diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index 1e405e696..a0c7b8c09 100644 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -1,5 +1,5 @@ backup_dir="$1/conf/ldap" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir # Fix for first jessie yunohost where slapd.conf is called slapd-yuno.conf # without slapcat doesn't work @@ -11,5 +11,5 @@ fi sudo cp -a /etc/ldap/slapd.conf $backup_dir/ sudo slapcat -l $backup_dir/slapcat.ldif.raw -egrep -v "^entryCSN:" < $backup_dir/slapcat.ldif.raw > $backup_dir/slapcat.ldif -rm -f $backup_dir/slapcat.ldif.raw +sudo bash -c "egrep -v '^entryCSN:' < $backup_dir/slapcat.ldif.raw > $backup_dir/slapcat.ldif" +sudo rm -f $backup_dir/slapcat.ldif.raw diff --git a/data/hooks/backup/08-conf_ssh b/data/hooks/backup/08-conf_ssh index f17ce48ee..693dd8cf6 100644 --- a/data/hooks/backup/08-conf_ssh +++ b/data/hooks/backup/08-conf_ssh @@ -1,4 +1,8 @@ backup_dir="$1/conf/ssh" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir -sudo cp -a /etc/ssh/. $backup_dir +if [ -d /etc/ssh/ ]; then + sudo cp -a /etc/ssh/. $backup_dir +else + echo "SSH is not installed" +fi diff --git a/data/hooks/backup/11-conf_ynh_mysql b/data/hooks/backup/11-conf_ynh_mysql index e0cdc59ac..5789901f1 100644 --- a/data/hooks/backup/11-conf_ynh_mysql +++ b/data/hooks/backup/11-conf_ynh_mysql @@ -1,4 +1,4 @@ backup_dir="$1/conf/ynh/mysql" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/yunohost/mysql $backup_dir/ diff --git a/data/hooks/backup/14-conf_ssowat b/data/hooks/backup/14-conf_ssowat index bae115467..3b23c2645 100644 --- a/data/hooks/backup/14-conf_ssowat +++ b/data/hooks/backup/14-conf_ssowat @@ -1,4 +1,4 @@ backup_dir="$1/conf/ssowat" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/ssowat/. $backup_dir diff --git a/data/hooks/backup/17-data_home b/data/hooks/backup/17-data_home index acc999a81..d6faa8934 100644 --- a/data/hooks/backup/17-data_home +++ b/data/hooks/backup/17-data_home @@ -1,5 +1,5 @@ backup_dir="$1/data/home" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir . /usr/share/yunohost/apps/helpers diff --git a/data/hooks/backup/20-conf_ynh_firewall b/data/hooks/backup/20-conf_ynh_firewall index cb0a8493d..f478e0fdf 100644 --- a/data/hooks/backup/20-conf_ynh_firewall +++ b/data/hooks/backup/20-conf_ynh_firewall @@ -1,4 +1,4 @@ backup_dir="$1/conf/ynh/firewall" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/yunohost/firewall* $backup_dir diff --git a/data/hooks/backup/21-conf_ynh_certs b/data/hooks/backup/21-conf_ynh_certs index 061513f99..19483ae5a 100644 --- a/data/hooks/backup/21-conf_ynh_certs +++ b/data/hooks/backup/21-conf_ynh_certs @@ -1,4 +1,4 @@ backup_dir="$1/conf/ynh/certs" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/yunohost/certs/. $backup_dir diff --git a/data/hooks/backup/26-conf_xmpp b/data/hooks/backup/26-conf_xmpp index 68461c03d..f207975e3 100644 --- a/data/hooks/backup/26-conf_xmpp +++ b/data/hooks/backup/26-conf_xmpp @@ -1,5 +1,5 @@ backup_dir="$1/conf/xmpp" -mkdir -p $backup_dir/{etc,var} +sudo mkdir -p $backup_dir/{etc,var} sudo cp -a /etc/metronome/. $backup_dir/etc sudo cp -a /var/lib/metronome/. $backup_dir/var diff --git a/data/hooks/backup/29-conf_nginx b/data/hooks/backup/29-conf_nginx index caa9c9d59..19bd1b456 100644 --- a/data/hooks/backup/29-conf_nginx +++ b/data/hooks/backup/29-conf_nginx @@ -1,4 +1,4 @@ backup_dir="$1/conf/nginx" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/nginx/conf.d/. $backup_dir diff --git a/data/hooks/backup/32-conf_cron b/data/hooks/backup/32-conf_cron index 5052d50e5..efa18f34a 100644 --- a/data/hooks/backup/32-conf_cron +++ b/data/hooks/backup/32-conf_cron @@ -1,4 +1,4 @@ backup_dir="$1/conf/cron" -mkdir -p $backup_dir +sudo mkdir -p $backup_dir sudo cp -a /etc/cron.d/yunohost* $backup_dir/ From d83f81be66f52238cfc3319a14a75215bc096eca Mon Sep 17 00:00:00 2001 From: zamentur Date: Sat, 3 Oct 2015 22:17:02 +0200 Subject: [PATCH 3/6] [fix] Restore with the app restore script from the backup --- data/actionsmap/yunohost.yml | 3 ++ data/hooks/restore/08-conf_ssh | 9 +++- data/hooks/restore/11-conf_ynh_mysql | 2 +- data/hooks/restore/21-conf_ynh_certs | 2 +- data/hooks/restore/23-data_mail | 2 +- lib/yunohost/backup.py | 80 ++++++++++++++++++++-------- locales/en.json | 5 ++ 7 files changed, 77 insertions(+), 26 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 9dfb4a499..125858d35 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -628,6 +628,9 @@ backup: --ignore-apps: help: Do not restore apps action: store_true + --ignore-hooks: + help: Do not restore hooks + action: store_true --force: help: Force restauration on an already installed system action: store_true diff --git a/data/hooks/restore/08-conf_ssh b/data/hooks/restore/08-conf_ssh index 71ac9df38..0c0f9bf9b 100644 --- a/data/hooks/restore/08-conf_ssh +++ b/data/hooks/restore/08-conf_ssh @@ -1,4 +1,9 @@ backup_dir="$1/conf/ssh" -sudo cp -a $backup_dir/. /etc/ssh -sudo service ssh restart +if [ -d /etc/ssh/ ]; then + sudo cp -a $backup_dir/. /etc/ssh + sudo service ssh restart +else + echo "SSH is not installed" +fi + diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql index bbea73fb1..e4d71b369 100644 --- a/data/hooks/restore/11-conf_ynh_mysql +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -1,6 +1,6 @@ backup_dir="$1/conf/ynh/mysql" -service mysql restart +sudo service mysql restart sudo cp -a $backup_dir/mysql /etc/yunohost/mysql mysqlpwd=$(sudo cat /etc/yunohost/mysql) sudo mysqladmin flush-privileges -p"$mysqlpwd" diff --git a/data/hooks/restore/21-conf_ynh_certs b/data/hooks/restore/21-conf_ynh_certs index 897e303ff..d1eb532ed 100644 --- a/data/hooks/restore/21-conf_ynh_certs +++ b/data/hooks/restore/21-conf_ynh_certs @@ -1,6 +1,6 @@ backup_dir="$1/conf/ynh/certs" -mkdir -p /etc/yunohost/certs/ +sudo mkdir -p /etc/yunohost/certs/ sudo cp -a $backup_dir/. /etc/yunohost/certs/ sudo yunohost app ssowatconf diff --git a/data/hooks/restore/23-data_mail b/data/hooks/restore/23-data_mail index 594c667dd..995308273 100644 --- a/data/hooks/restore/23-data_mail +++ b/data/hooks/restore/23-data_mail @@ -1,6 +1,6 @@ backup_dir="$1/data/mail" -sudo cp -a $backup_dir/. /var/mail +sudo cp -a $backup_dir/. /var/mail/ || echo 'No mail found' # Restart services to use migrated certs sudo service postfix restart diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index 9e1200f98..a5809ff72 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -162,6 +162,8 @@ def backup_create(name=None, description=None, output_directory=None, for app_id in apps_filtered: app_setting_path = '/etc/yunohost/apps/' + app_id + tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id) + # Check if the app has a backup script app_script = app_setting_path + '/scripts/backup' if not os.path.isfile(app_script): @@ -169,8 +171,22 @@ def backup_create(name=None, description=None, output_directory=None, msignals.display(m18n.n('unbackup_app', app_id), 'warning') continue + + # Copy the app restore script + app_restore_script = app_setting_path + '/scripts/restore' + if os.path.isfile(app_script): + try: + filesystem.mkdir(tmp_app_dir, 0750, True, uid='admin') + shutil.copy(app_restore_script, tmp_app_dir) + except: + logger.exception("error while copying restore script of '%s'", app_id) + msignals.display(m18n.n('restore_app_copy_failed', app=app_id), + 'warning') + else: + logger.warning("restore script '%s' not found", app_script) + msignals.display(m18n.n('unrestorable_app', app_id), + 'warning') - tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id) tmp_app_bkp_dir = tmp_app_dir + '/backup' msignals.display(m18n.n('backup_running_app_script', app_id)) try: @@ -251,7 +267,7 @@ def backup_create(name=None, description=None, output_directory=None, return { 'archive': info } -def backup_restore(name, hooks=[], apps=[], ignore_apps=False, force=False): +def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=False, force=False): """ Restore from a local backup archive @@ -265,6 +281,7 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, force=False): """ from yunohost.hook import hook_add from yunohost.hook import hook_callback + from yunohost.hook import hook_exec # Retrieve and open the archive archive_file = backup_info(name)['path'] @@ -329,32 +346,53 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, force=False): logger.info("executing the post-install...") tools_postinstall(domain, 'yunohost', True) + # Run hooks + if not ignore_hooks: + msignals.display(m18n.n('restore_running_hooks')) + hook_callback('restore', hooks, args=[tmp_dir]) + # Add apps restore hook if not ignore_apps: # Filter applications to restore apps_list = set(info['apps'].keys()) apps_filtered = set() - if apps: - for a in apps: - if a not in apps_list: - logger.warning("app '%s' not found", a) - msignals.display(m18n.n('unrestore_app', a), 'warning') - else: - apps_filtered.add(a) - else: - apps_filtered = apps_list - - for app_id in apps_filtered: - hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id - if os.path.isfile(hook): - hook_add(app_id, hook) - logger.info("app '%s' will be restored", app_id) - else: + if not apps: + apps=apps_list + + from yunohost.app import _is_installed + for app_id in apps: + if app_id not in apps_list: + logger.warning("app '%s' not found", app_id) msignals.display(m18n.n('unrestore_app', app_id), 'warning') + elif _is_installed(app_id): + logger.warning("app '%s' already installed", app_id) + msignals.display(m18n.n('restore_already_installed_app', app=app_id), 'warning') + elif not os.path.isfile('{:s}/apps/{:s}/restore'.format(tmp_dir, app_id)): + logger.warning("backup for '%s' doesn't contain a restore script", app_id) + msignals.display(m18n.n('no_restore_script', app=app_id), 'warning') + else: + apps_filtered.add(app_id) + + for app_id in apps_filtered: + app_bkp_dir='{:s}/apps/{:s}'.format(tmp_dir, app_id) + try: + # Copy app settings + app_setting_path = '/etc/yunohost/apps/' + app_id + shutil.copytree(app_bkp_dir + '/settings', app_setting_path + '/settings') + + # Execute app restore script + app_restore_script=app_bkp_dir+'/restore' + tmp_script = '/tmp/restore_%s_%s' % (name,app_id) + subprocess.call(['install', '-Dm555', app_restore_script, tmp_script]) + hook_exec(tmp_script, args=[app_bkp_dir, app_id]) + + except: + logger.exception("error while restoring backup of '%s'", app_id) + msignals.display(m18n.n('restore_app_failed', app=app_id), + 'error') + # Cleaning settings directory + shutil.rmtree(app_setting_path + '/settings', ignore_errors=True) - # Run hooks - msignals.display(m18n.n('restore_running_hooks')) - hook_callback('restore', hooks, args=[tmp_dir]) # Remove temporary directory os.system('rm -rf %s' % tmp_dir) diff --git a/locales/en.json b/locales/en.json index 6838c7ddb..286d7257b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -156,10 +156,15 @@ "backup_complete" : "Backup complete", "backup_invalid_archive" : "Invalid backup archive", "restore_confirm_yunohost_installed" : "Do you really want to restore an already installed system? [{answers:s}]", + "restore_app_failed" : "Unable to restore the app '{app:s}'", "restore_running_hooks" : "Running restoration hooks...", "restore_failed" : "Unable to restore the system", "restore_complete" : "Restore complete", + "restore_already_installed_app": "An app is already installed with the id '{app:s}'", "unbackup_app" : "App '{:s}' will not be saved", + "unrestorable_app" : "App '{:s}' will not be restored", + "restore_app_copy_failed" : "Unable to copy the restore script of app '{app:s}'", + "no_restore_script": "No restore script found for the app '{app:s}'", "unrestore_app" : "App '{:s}' will not be restored", "backup_delete_error" : "Unable to delete '{:s}'", "backup_deleted" : "Backup successfully deleted", From 66f61be90ef4a563a241122125a420bb7ebf609b Mon Sep 17 00:00:00 2001 From: zamentur Date: Sat, 3 Oct 2015 22:46:13 +0200 Subject: [PATCH 4/6] [fix] Backup dir transmitted to app restore script --- lib/yunohost/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index a5809ff72..8fee951fe 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -378,13 +378,13 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals try: # Copy app settings app_setting_path = '/etc/yunohost/apps/' + app_id - shutil.copytree(app_bkp_dir + '/settings', app_setting_path + '/settings') + shutil.copytree(app_bkp_dir + '/settings', app_setting_path ) # Execute app restore script app_restore_script=app_bkp_dir+'/restore' tmp_script = '/tmp/restore_%s_%s' % (name,app_id) subprocess.call(['install', '-Dm555', app_restore_script, tmp_script]) - hook_exec(tmp_script, args=[app_bkp_dir, app_id]) + hook_exec(tmp_script, args=[app_bkp_dir+'/backup', app_id]) except: logger.exception("error while restoring backup of '%s'", app_id) From ffc9bcaf33cfdad1b4dc91618e6207c70ab57dea Mon Sep 17 00:00:00 2001 From: zamentur Date: Sat, 3 Oct 2015 23:27:06 +0200 Subject: [PATCH 5/6] [fix] Don't run hooks not in the backup --- lib/yunohost/backup.py | 13 +++++++++++-- locales/en.json | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index 8fee951fe..0e9ab63c5 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -284,7 +284,8 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals from yunohost.hook import hook_exec # Retrieve and open the archive - archive_file = backup_info(name)['path'] + info = backup_info(name) + archive_file = info['path'] try: tar = tarfile.open(archive_file, "r:gz") except: @@ -348,8 +349,16 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals # Run hooks if not ignore_hooks: + if hooks is None or len(hooks)==0: + hooks=info['hooks'].keys() + + hooks_filtered=list(set(hooks) & set(info['hooks'].keys())) + hooks_unexecuted=set(hooks) - set(info['hooks'].keys()) + for hook in hooks_unexecuted: + logger.warning("hook '%s' not in this backup", hook) + msignals.display(m18n.n('backup_hook_unavailable', hook), 'warning') msignals.display(m18n.n('restore_running_hooks')) - hook_callback('restore', hooks, args=[tmp_dir]) + hook_callback('restore', hooks_filtered, args=[tmp_dir]) # Add apps restore hook if not ignore_apps: diff --git a/locales/en.json b/locales/en.json index 286d7257b..ba4e6b072 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,6 +155,7 @@ "backup_cleaning_failed" : "Unable to clean backup directory", "backup_complete" : "Backup complete", "backup_invalid_archive" : "Invalid backup archive", + "backup_hook_unavailable" : "The hook '{:s}' is not in this backup", "restore_confirm_yunohost_installed" : "Do you really want to restore an already installed system? [{answers:s}]", "restore_app_failed" : "Unable to restore the app '{app:s}'", "restore_running_hooks" : "Running restoration hooks...", From 141937efdb59c56fb5abb208c411bea2db4b8d23 Mon Sep 17 00:00:00 2001 From: zamentur Date: Sun, 4 Oct 2015 12:48:56 +0200 Subject: [PATCH 6/6] [enh] Don't restore apps nginx conf in conf_nginx restore script (it's done by apps) --- data/hooks/restore/29-conf_nginx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/hooks/restore/29-conf_nginx b/data/hooks/restore/29-conf_nginx index 245c1248c..0795f53df 100644 --- a/data/hooks/restore/29-conf_nginx +++ b/data/hooks/restore/29-conf_nginx @@ -1,6 +1,7 @@ backup_dir="$1/conf/nginx" -sudo cp -a $backup_dir/. /etc/nginx/conf.d +# Copy all conf except apps specific conf located in DOMAIN.d +sudo find $backup_dir/ -mindepth 1 -maxdepth 1 -name '*.d' -or -exec sudo cp -a {} /etc/nginx/conf.d/ \; # Restart to use new conf and certs sudo service nginx restart