diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index f90c3675..6c703d02 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -596,6 +596,9 @@ backup: --hooks: help: List of backup hooks names to execute nargs: "*" + --apps: + help: List of application names to backup + nargs: "*" --ignore-apps: help: Do not backup apps action: store_true @@ -603,15 +606,16 @@ backup: ### backup_restore() restore: action_help: Restore from a local backup archive - api: POST /restore - configuration: - authenticate: false + api: POST /backup/restore/ arguments: name: help: Name of the local backup archive --hooks: help: List of restauration hooks names to execute nargs: "*" + --apps: + help: List of application names to restore + nargs: "*" --ignore-apps: help: Do not restore apps action: store_true @@ -625,6 +629,15 @@ backup: api: GET /backup/archives configuration: lock: false + arguments: + -i: + full: --with-info + help: Show backup information for each archive + action: store_true + -H: + full: --human-readable + help: Print sizes in human readable format + action: store_true ### backup_info() info: @@ -635,6 +648,14 @@ backup: arguments: name: help: Name of the local backup archive + -d: + full: --with-details + help: Show additional backup information + action: store_true + -H: + full: --human-readable + help: Print sizes in human readable format + action: store_true ############################# diff --git a/data/hooks/restore/11-system_mysql b/data/hooks/restore/11-system_mysql index 23293b15..f25e4ef6 100644 --- a/data/hooks/restore/11-system_mysql +++ b/data/hooks/restore/11-system_mysql @@ -1,5 +1,5 @@ backup_dir="$1/mysql" mysqlpwd=$(sudo cat /etc/yunohost/mysql) -sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql +#sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql sudo mysqladmin flush-privileges -p"$mysqlpwd" diff --git a/debian/control b/debian/control index 561083c2..4c96298d 100644 --- a/debian/control +++ b/debian/control @@ -9,17 +9,22 @@ Homepage: https://yunohost.org/ Package: moulinette-yunohost Architecture: all Depends: moulinette (>= 2.2.1), - python-psutil, - python-requests, - glances, - python-pip, - pyminiupnpc, - dnsutils, - bind9utils, - python-apt, - ca-certificates, - python-dnspython, - netcat-openbsd, - iproute + python-psutil, + python-requests, + glances, + python-pip, + pyminiupnpc, + dnsutils, + bind9utils, + python-apt, + ca-certificates, + python-dnspython, + netcat-openbsd, + iproute, + unzip, + git-core, + curl, + mariadb-server | mysql-server, php5-mysql | php5-mysqlnd +Conflicts: iptables-persistent Description: YunoHost Python scripts Python functions to manage a YunoHost instance diff --git a/lib/yunohost/app.py b/lib/yunohost/app.py index d8b1b7b7..01c93960 100644 --- a/lib/yunohost/app.py +++ b/lib/yunohost/app.py @@ -351,7 +351,8 @@ def app_upgrade(auth, app=[], url=None, file=None): for line in lines: sources.write(re.sub(r''+ original_app_id +'', app_id, line)) - # Add hooks + # Clean hooks and add new ones + hook_remove(app_id) if 'hooks' in os.listdir(app_tmp_folder): for hook in os.listdir(app_tmp_folder +'/hooks'): hook_add(app_id, app_tmp_folder +'/hooks/'+ hook) @@ -451,7 +452,8 @@ def app_install(auth, app, label=None, args=None): os.makedirs(app_setting_path) os.system('touch %s/settings.yml' % app_setting_path) - # Add hooks + # Clean hooks and add new ones + hook_remove(app_id) if 'hooks' in os.listdir(app_tmp_folder): for file in os.listdir(app_tmp_folder +'/hooks'): hook_add(app_id, app_tmp_folder +'/hooks/'+ file) @@ -1034,7 +1036,7 @@ def _fetch_app_from_git(app): if os.system('wget "%s" -O "%s.zip" > /dev/null 2>&1' % (url, app_tmp_folder)) == 0: return _extract_app_from_file(app_tmp_folder +'.zip', remove=True) - git_result = os.system('git clone %s %s' % (app, app_tmp_folder)) + git_result = os.system('git clone --depth=1 %s %s' % (app, app_tmp_folder)) git_result_2 = 0 try: with open(app_tmp_folder + '/manifest.json') as json_manifest: diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index 7f98b118..703f4a28 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -29,8 +29,9 @@ import sys import json import errno import time -import shutil import tarfile +import subprocess +from collections import OrderedDict from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger @@ -42,7 +43,7 @@ logger = getActionLogger('yunohost.backup') def backup_create(name=None, description=None, output_directory=None, - no_compress=False, hooks=[], ignore_apps=False): + no_compress=False, hooks=[], apps=[], ignore_apps=False): """ Create a backup local archive @@ -52,12 +53,12 @@ def backup_create(name=None, description=None, output_directory=None, output_directory -- Output directory for the backup no_compress -- Do not create an archive file hooks -- List of backup hooks names to execute + apps -- List of application names to backup ignore_apps -- Do not backup apps """ # TODO: Add a 'clean' argument to clean output directory - from yunohost.hook import hook_add - from yunohost.hook import hook_callback + from yunohost.hook import hook_callback, hook_exec tmp_dir = None @@ -121,33 +122,58 @@ def backup_create(name=None, description=None, output_directory=None, 'description': description or '', 'created_at': timestamp, 'apps': {}, + 'hooks': {}, } - # Add apps backup hook + # Run system hooks + msignals.display(m18n.n('backup_running_hooks')) + hooks_ret = hook_callback('backup', hooks, args=[tmp_dir]) + + # Add hooks results to the info + info['hooks'] = hooks_ret['succeed'] + + # Backup apps if not ignore_apps: from yunohost.app import app_info - try: - for app_id in os.listdir('/etc/yunohost/apps'): - hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id - if os.path.isfile(hook): - hook_add(app_id, hook) - # Add app info - i = app_info(app_id) - info['apps'][app_id] = { - 'version': i['version'], - } + # Filter applications to backup + apps_list = set(os.listdir('/etc/yunohost/apps')) + apps_filtered = set() + if apps: + for a in apps: + if a not in apps_list: + logger.warning("app '%s' not found", a) + msignals.display(m18n.n('unbackup_app', a), 'warning') else: - logger.warning("unable to find app's backup hook '%s'", - hook) - msignals.display(m18n.n('unbackup_app', app_id), - 'warning') - except IOError as e: - logger.info("unable to add apps backup hook: %s", str(e)) + apps_filtered.add(a) + else: + apps_filtered = apps_list - # Run hooks - msignals.display(m18n.n('backup_running_hooks')) - hook_callback('backup', hooks, args=[tmp_dir]) + # Run apps backup scripts + tmp_script = '/tmp/backup_' + str(timestamp) + for app_id in apps_filtered: + script = '/etc/yunohost/apps/{:s}/scripts/backup'.format(app_id) + if not os.path.isfile(script): + logger.warning("backup script '%s' not found", script) + msignals.display(m18n.n('unbackup_app', app_id), + 'warning') + continue + + try: + msignals.display(m18n.n('backup_running_app_script', app_id)) + subprocess.call(['install', '-Dm555', script, tmp_script]) + hook_exec(tmp_script, args=[tmp_dir]) + except: + logger.exception("error while executing script '%s'", script) + msignals.display(m18n.n('unbackup_app', app_id), + 'error') + else: + # Add app info + i = app_info(app_id) + info['apps'][app_id] = { + 'version': i['version'], + } + subprocess.call(['rm', '-f', tmp_script]) # Create backup info file with open("%s/info.json" % tmp_dir, 'w') as f: @@ -191,14 +217,19 @@ def backup_create(name=None, description=None, output_directory=None, msignals.display(m18n.n('backup_complete'), 'success') + # Return backup info + info['name'] = name + return { 'archive': info } -def backup_restore(name, hooks=[], ignore_apps=False, force=False): + +def backup_restore(name, hooks=[], apps=[], ignore_apps=False, force=False): """ Restore from a local backup archive Keyword argument: name -- Name of the local backup archive hooks -- List of restoration hooks names to execute + apps -- List of application names to restore ignore_apps -- Do not restore apps force -- Force restauration on an already installed system @@ -239,15 +270,6 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False): logger.info("restoring from backup '%s' created on %s", name, time.ctime(info['created_at'])) - # Retrieve domain from the backup - try: - with open("%s/yunohost/current_host" % tmp_dir, 'r') as f: - domain = f.readline().rstrip() - except IOError: - logger.error("unable to retrieve domain from '%s/yunohost/current_host'", - tmp_dir) - raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) - # Check if YunoHost is installed if os.path.isfile('/etc/yunohost/installed'): msignals.display(m18n.n('yunohost_already_installed'), 'warning') @@ -265,12 +287,35 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False): raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed')) else: from yunohost.tools import tools_postinstall + + # Retrieve the domain from the backup + try: + with open("%s/yunohost/current_host" % tmp_dir, 'r') as f: + domain = f.readline().rstrip() + except IOError: + logger.error("unable to retrieve domain from '%s/yunohost/current_host'", + tmp_dir) + raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) + logger.info("executing the post-install...") tools_postinstall(domain, 'yunohost', True) # Add apps restore hook if not ignore_apps: - for app_id in info['apps'].keys(): + # Filter applications to restore + apps_list = set(info['apps'].keys()) + apps_filtered = set() + if apps: + for a in apps: + if a not in apps_list: + logger.warning("app '%s' not found", a) + msignals.display(m18n.n('unrestore_app', a), 'warning') + else: + apps_filtered.add(a) + else: + apps_filtered = apps_list + + for app_id in apps_filtered: hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id if os.path.isfile(hook): hook_add(app_id, hook) @@ -288,10 +333,14 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False): msignals.display(m18n.n('restore_complete'), 'success') -def backup_list(): +def backup_list(with_info=False, human_readable=False): """ List available local backup archives + Keyword arguments: + with_info -- Show backup information for each archive + human_readable -- Print sizes in human readable format + """ result = [] @@ -308,18 +357,29 @@ def backup_list(): except ValueError: continue result.append(name) + result.sort() + + if result and with_info: + d = OrderedDict() + for a in result: + d[a] = backup_info(a, human_readable=human_readable) + result = d return { 'archives': result } -def backup_info(name): +def backup_info(name, with_details=False, human_readable=False): """ Get info about a local backup archive Keyword arguments: name -- Name of the local backup archive + with_details -- Show additional backup information + human_readable -- Print sizes in human readable format """ + from yunohost.monitor import binary_to_human + archive_file = '%s/%s.tar.gz' % (archives_path, name) if not os.path.isfile(archive_file): logger.error("no local backup archive found at '%s'", archive_file) @@ -336,10 +396,19 @@ def backup_info(name): info_file) raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) - return { + size = os.path.getsize(archive_file) + if human_readable: + size = binary_to_human(size) + 'B' + + result = { 'path': archive_file, 'created_at': time.strftime(m18n.n('format_datetime_short'), time.gmtime(info['created_at'])), 'description': info['description'], - 'apps': info['apps'], + 'size': size, } + + if with_details: + for d in ['apps', 'hooks']: + result[d] = info[d] + return result diff --git a/lib/yunohost/hook.py b/lib/yunohost/hook.py index bbfefabf..8de68c05 100644 --- a/lib/yunohost/hook.py +++ b/lib/yunohost/hook.py @@ -174,7 +174,7 @@ def hook_callback(action, hooks=[], args=None): args -- Ordered list of arguments to pass to the script """ - result = { 'succeed': list(), 'failed': list() } + result = { 'succeed': {}, 'failed': {} } hooks_dict = {} # Retrieve hooks @@ -220,15 +220,18 @@ def hook_callback(action, hooks=[], args=None): # Iterate over hooks and execute them for priority in sorted(hooks_dict): for name, info in iter(hooks_dict[priority].items()): + state = 'succeed' filename = '%s-%s' % (priority, name) try: hook_exec(info['path'], args=args) except: logger.exception("error while executing hook '%s'", info['path']) - result['failed'].append(filename) - else: - result['succeed'].append(filename) + state = 'failed' + try: + result[state][name].append(info['path']) + except KeyError: + result[state][name] = [info['path']] return result diff --git a/lib/yunohost/monitor.py b/lib/yunohost/monitor.py index 541e0892..9db5e323 100644 --- a/lib/yunohost/monitor.py +++ b/lib/yunohost/monitor.py @@ -130,7 +130,7 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False): else: if human_readable: for i in ['used', 'avail', 'size']: - d[i] = _binary_to_human(d[i]) + 'B' + d[i] = binary_to_human(d[i]) + 'B' _set(dname, d) for dname in devices_names: _set(dname, 'not-available') @@ -201,7 +201,7 @@ def monitor_network(units=None, human_readable=False): if human_readable: for k in i.keys(): if k != 'time_since_update': - i[k] = _binary_to_human(i[k]) + 'B' + i[k] = binary_to_human(i[k]) + 'B' result[u][iname] = i elif u == 'infos': try: @@ -261,10 +261,10 @@ def monitor_system(units=None, human_readable=False): if human_readable: for i in ram.keys(): if i != 'percent': - ram[i] = _binary_to_human(ram[i]) + 'B' + ram[i] = binary_to_human(ram[i]) + 'B' for i in swap.keys(): if i != 'percent': - swap[i] = _binary_to_human(swap[i]) + 'B' + swap[i] = binary_to_human(swap[i]) + 'B' result[u] = { 'ram': ram, 'swap': swap @@ -510,7 +510,7 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True): return result -def _binary_to_human(n, customary=False): +def binary_to_human(n, customary=False): """ Convert bytes or bits into human readable format with binary prefix diff --git a/locales/en.json b/locales/en.json index 2d5ff8dd..4006ff6e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -143,6 +143,7 @@ "backup_output_directory_forbidden" : "Forbidden output directory", "backup_output_directory_not_empty" : "Output directory is not empty", "backup_running_hooks" : "Running backup hooks...", + "backup_running_app_script" : "Running backup script of app '{:s}'...", "backup_creating_archive" : "Creating the backup archive...", "backup_extracting_archive" : "Extracting the backup archive...", "backup_archive_open_failed" : "Unable to open the backup archive",