Merge branch 'master' of https://github.com/YunoHost/moulinette-yunohost

Conflicts:
	bin/yunohost
	bin/yunohost-api

actionsmap/yunohost.yml

@@ -254,7 +254,7 @@ domain:
                     help: Domain name to add
                     extra:
                         pattern:
-                            - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$'
+                            - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$'
                             - pattern_domain
                 -d:
                     full: --dyndns
@@ -272,7 +272,7 @@ domain:
                     help: Domain to delete
                     extra:
                         pattern:
-                            - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$'
+                            - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$'
                             - pattern_domain
 
         ### domain_info()
@@ -565,14 +565,72 @@ backup:
     category_help: Manage backups
     actions:
 
-        ### backup_init()
+        ### backup_create()
-        init:
+        create:
-            action_help: Init Tahoe-LAFS configuration
+            action_help: Create a backup local archive
-            # api: POST /backup/init
+            api: POST /backup
+            configuration:
+                lock: false
             arguments:
-                --helper:
+                -n:
-                    help: Init as a helper node rather than a "helped" one
+                    full: --name
+                    help: Name of the backup archive
+                    extra:
+                        pattern:
+                            - '^[\w\-\.]{1,30}(?<!\.)$'
+                            - pattern_backup_archive_name
+                -d:
+                    full: --description
+                    help: Short description of the backup
+                -o:
+                    full: --output-directory
+                    help: Output directory for the backup
+                -r:
+                    full: --no-compress
+                    help: Do not create an archive file
                     action: store_true
+                --hooks:
+                    help: List of backup hooks names to execute
+                    nargs: '*'
+                --ignore-apps:
+                    help: Do not backup apps
+                    action: store_true
+
+        ### backup_restore()
+        restore:
+            action_help: Restore from a local backup archive
+            api: POST /restore
+            configuration:
+                authenticate: false
+            arguments:
+                name:
+                    help: Name of the local backup archive
+                --hooks:
+                    help: List of restauration hooks names to execute
+                    nargs: '*'
+                --ignore-apps:
+                    help: Do not restore apps
+                    action: store_true
+                --force:
+                    help: Force restauration on an already installed system
+                    action: store_true
+
+        ### backup_list()
+        list:
+            action_help: List available local backup archives
+            api: GET /backup/archives
+            configuration:
+                lock: false
+
+        ### backup_info()
+        info:
+            action_help: Show info about a local backup archive
+            api: GET /backup/archives/<name>
+            configuration:
+                lock: false
+            arguments:
+                name:
+                    help: Name of the local backup archive
 
 
 #############################
@@ -972,12 +1030,9 @@ tools:
         adminpw:
             action_help: Change admin password
             api: PUT /adminpw
+            configuration:
+                authenticate: all
             arguments:
-                -o:
-                    full: --old-password
-                    extra:
-                        password: ask_current_admin_password
-                        required: True
                 -n:
                     full: --new-password
                     extra:
@@ -998,13 +1053,13 @@ tools:
                     full: --old-domain
                     extra:
                         pattern:
-                            - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$'
+                            - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$'
                             - pattern_domain
                 -n:
                     full: --new-domain
                     extra:
                         pattern:
-                            - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$'
+                            - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$'
                             - pattern_domain
 
         ### tools_postinstall()
@@ -1021,7 +1076,7 @@ tools:
                     extra:
                         ask: ask_main_domain
                         pattern:
-                            - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$'
+                            - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$'
                             - pattern_domain
                         required: True
                 -p:
@@ -1033,8 +1088,8 @@ tools:
                         pattern:
                             - '^.{3,}$'
                             - pattern_password
-                --dyndns:
+                --ignore-dyndns:
-                    help: Subscribe domain to a DynDNS service
+                    help: Do not subscribe domain to a DynDNS service
                     action: store_true
 
         ### tools_update()
@@ -1093,6 +1148,26 @@ hook:
                 app:
                     help: Scripts related to app will be removed
 
+        ### hook_list()
+        list:
+            action_help: List available hooks for an action
+            api: GET /hooks/<action>
+            arguments:
+                action:
+                    help: Action name
+                -l:
+                    full: --list-by
+                    help: Property to list hook by
+                    choices:
+                        - name
+                        - priority
+                        - folder
+                    default: name
+                -i:
+                    full: --show-info
+                    help: Show hook information
+                    action: store_true
+
         ### hook_callback()
         callback:
             action_help: Execute all scripts binded to an action
@@ -1100,6 +1175,10 @@ hook:
             arguments:
                 action:
                     help: Action name
+                -n:
+                    full: --hooks
+                    help: List of hooks names to execute
+                    nargs: '*'
                 -a:
                     full: --args
                     help: Ordered list of arguments to pass to the script

app.py

@@ -175,7 +175,8 @@ def app_list(offset=None, limit=None, filter=None, raw=False):
                         list_dict.append({
                             'id': app_id,
                             'name': app_info['manifest']['name'],
-                            'description': app_info['manifest']['description'],
+                            'description': _value_for_locale(
+                                app_info['manifest']['description']),
                             # FIXME: Temporarly allow undefined license
                             'license': app_info['manifest'].get('license',
                                 m18n.n('license_undefined')),
@@ -216,7 +217,9 @@ def app_info(app, raw=False):
                 # FIXME: Temporarly allow undefined license
                 'license': app_info['manifest'].get('license',
                     m18n.n('license_undefined')),
-                #TODO: Add more infos
+                # FIXME: Temporarly allow undefined version
+                'version' :  app_info['manifest'].get('version', '-'),
+                #TODO: Add more info
             }
 
 
@@ -262,7 +265,7 @@ def app_map(app=None, raw=False, user=None):
     return result
 
 
-def app_upgrade(auth, app, url=None, file=None):
+def app_upgrade(auth, app=[], url=None, file=None):
     """
     Upgrade app
 
@@ -283,6 +286,7 @@ def app_upgrade(auth, app, url=None, file=None):
 
     # If no app is specified, upgrade all apps
     if not app:
+        if (not url and not file):
             app = os.listdir(apps_setting_path)
     elif not isinstance(app, list):
         app = [ app ]
@@ -393,8 +397,10 @@ def app_install(auth, app, label=None, args=None):
 
     if app in app_list(raw=True) or ('@' in app) or ('http://' in app) or ('https://' in app):
         manifest = _fetch_app_from_git(app)
-    else:
+    elif os.path.exists(app):
         manifest = _extract_app_from_file(app)
+    else:
+        raise MoulinetteError(errno.EINVAL, m18n.n('app_unknown'))
 
     # Check ID
     if 'id' not in manifest or '__' in manifest['id']:
@@ -587,7 +593,7 @@ def app_addaccess(auth, apps, users):
                         new_users = new_users +','+ allowed_user
 
             app_setting(app, 'allowed_users', new_users.strip())
-            hook_callback('post_app_addaccess', [app, new_users])
+            hook_callback('post_app_addaccess', args=[app, new_users])
 
     app_ssowatconf(auth)
 
@@ -640,7 +646,7 @@ def app_removeaccess(auth, apps, users):
                         new_users=new_users+','+user['username']
 
             app_setting(app, 'allowed_users', new_users.strip())
-            hook_callback('post_app_removeaccess', [app, new_users])
+            hook_callback('post_app_removeaccess', args=[app, new_users])
 
     app_ssowatconf(auth)
 
@@ -673,7 +679,7 @@ def app_clearaccess(auth, apps):
         if 'allowed_users' in app_settings:
             app_setting(app, 'allowed_users', delete=True)
 
-        hook_callback('post_app_clearaccess', [app])
+        hook_callback('post_app_clearaccess', args=[app])
 
     app_ssowatconf(auth)
 
@@ -923,10 +929,7 @@ def app_ssowatconf(auth):
     for domain in domains:
         skipped_urls.extend(['/yunohost/admin', '/yunohost/api'])
 
-    with open('/etc/ssowat/conf.json') as f:
+    conf_dict = {
-        conf_dict = json.load(f)
-
-    conf_dict.update({
         'portal_domain': main_domain,
         'portal_path': '/yunohost/sso/',
         'additional_headers': {
@@ -944,7 +947,7 @@ def app_ssowatconf(auth):
         'protected_regex': protected_regex,
         'redirected_regex': redirected_regex,
         'users': users,
-    })
+    }
 
     with open('/etc/ssowat/conf.json', 'w+') as f:
         json.dump(conf_dict, f, sort_keys=True, indent=4)

backup.py

@@ -24,29 +24,322 @@
     Manage backups
 """
 import os
+import re
 import sys
 import json
-import yaml
+import errno
-import glob
+import time
+import shutil
+import tarfile
 
 from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
 
-def backup_init(helper=False):
+backup_path   = '/home/yunohost.backup'
+archives_path = '%s/archives' % backup_path
+
+logger = getActionLogger('yunohost.backup')
+
+
+def backup_create(name=None, description=None, output_directory=None,
+                  no_compress=False, hooks=[], ignore_apps=False):
     """
-    Init Tahoe-LAFS configuration
+    Create a backup local archive
+
+    Keyword arguments:
+        name -- Name of the backup archive
+        description -- Short description of the backup
+        output_directory -- Output directory for the backup
+        no_compress -- Do not create an archive file
+        hooks -- List of backup hooks names to execute
+        ignore_apps -- Do not backup apps
+
+    """
+    # TODO: Add a 'clean' argument to clean output directory
+    from yunohost.hook import hook_add
+    from yunohost.hook import hook_callback
+
+    tmp_dir = None
+
+    # Validate and define backup name
+    timestamp = int(time.time())
+    if not name:
+        name = str(timestamp)
+    if name in backup_list()['archives']:
+        raise MoulinetteError(errno.EINVAL,
+                              m18n.n('backup_archive_name_exists'))
+
+    # Validate additional arguments
+    if no_compress and not output_directory:
+        raise MoulinetteError(errno.EINVAL,
+                              m18n.n('backup_output_directory_required'))
+    if output_directory:
+        output_directory = os.path.abspath(output_directory)
+
+        # Check for forbidden folders
+        if output_directory.startswith(archives_path) or \
+           re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
+                    output_directory):
+            logger.error("forbidden output directory '%'", output_directory)
+            raise MoulinetteError(errno.EINVAL,
+                                  m18n.n('backup_output_directory_forbidden'))
+
+        # Create the output directory
+        if not os.path.isdir(output_directory):
+            logger.info("creating output directory '%s'", output_directory)
+            os.makedirs(output_directory, 0750)
+        # Check that output directory is empty
+        elif no_compress and os.listdir(output_directory):
+            logger.error("not empty output directory '%'", output_directory)
+            raise MoulinetteError(errno.EIO,
+                                  m18n.n('backup_output_directory_not_empty'))
+
+        # Define temporary directory
+        if no_compress:
+            tmp_dir = output_directory
+    else:
+        output_directory = archives_path
+
+    # Create temporary directory
+    if not tmp_dir:
+        tmp_dir = "%s/tmp/%s" % (backup_path, name)
+        if os.path.isdir(tmp_dir):
+            logger.warning("temporary directory for backup '%s' already exists",
+                           tmp_dir)
+            os.system('rm -rf %s' % tmp_dir)
+        try:
+            os.mkdir(tmp_dir, 0750)
+        except OSError:
+            # Create temporary directory recursively
+            os.makedirs(tmp_dir, 0750)
+            os.system('chown -hR admin: %s' % backup_path)
+        else:
+            os.system('chown -hR admin: %s' % tmp_dir)
+
+    # Initialize backup info
+    info = {
+        'description': description or '',
+        'created_at': timestamp,
+        'apps': {},
+    }
+
+    # Add apps backup hook
+    if not ignore_apps:
+        from yunohost.app import app_info
+        try:
+            for app_id in os.listdir('/etc/yunohost/apps'):
+                hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id
+                if os.path.isfile(hook):
+                    hook_add(app_id, hook)
+
+                    # Add app info
+                    i = app_info(app_id)
+                    info['apps'][app_id] = {
+                        'version': i['version'],
+                    }
+                else:
+                    logger.warning("unable to find app's backup hook '%s'",
+                                   hook)
+                    msignals.display(m18n.n('unbackup_app', app_id),
+                                     'warning')
+        except IOError as e:
+            logger.info("unable to add apps backup hook: %s", str(e))
+
+    # Run hooks
+    msignals.display(m18n.n('backup_running_hooks'))
+    hook_callback('backup', hooks, args=[tmp_dir])
+
+    # Create backup info file
+    with open("%s/info.json" % tmp_dir, 'w') as f:
+        f.write(json.dumps(info))
+
+    # Create the archive
+    if not no_compress:
+        msignals.display(m18n.n('backup_creating_archive'))
+        archive_file = "%s/%s.tar.gz" % (output_directory, name)
+        try:
+            tar = tarfile.open(archive_file, "w:gz")
+        except:
+            tar = None
+
+            # Create the archives directory and retry
+            if not os.path.isdir(archives_path):
+                os.mkdir(archives_path, 0750)
+                try:
+                    tar = tarfile.open(archive_file, "w:gz")
+                except:
+                    logger.exception("unable to open the archive '%s' for writing "
+                                     "after creating directory '%s'",
+                                     archive_file, archives_path)
+                    tar = None
+            else:
+                logger.exception("unable to open the archive '%s' for writing",
+                                 archive_file)
+            if tar is None:
+                raise MoulinetteError(errno.EIO,
+                                      m18n.n('backup_archive_open_failed'))
+        tar.add(tmp_dir, arcname='')
+        tar.close()
+
+        # Copy info file
+        os.system('mv %s/info.json %s/%s.info.json' %
+                  (tmp_dir, archives_path, name))
+
+    # Clean temporary directory
+    if tmp_dir != output_directory:
+        os.system('rm -rf %s' % tmp_dir)
+
+    msignals.display(m18n.n('backup_complete'), 'success')
+
+
+def backup_restore(name, hooks=[], ignore_apps=False, force=False):
+    """
+    Restore from a local backup archive
 
     Keyword argument:
-        helper -- Init as a helper node rather than a "helped" one
+        name -- Name of the local backup archive
+        hooks -- List of restoration hooks names to execute
+        ignore_apps -- Do not restore apps
+        force -- Force restauration on an already installed system
 
     """
-    tahoe_cfg_dir = '/usr/share/yunohost/yunohost-config/backup'
+    from yunohost.hook import hook_add
-    if helper:
+    from yunohost.hook import hook_callback
-        configure_cmd = '/configure_tahoe.sh helper'
-    else:
-        configure_cmd = '/configure_tahoe.sh'
 
-    os.system('tahoe create-client /home/yunohost.backup/tahoe')
+    # Retrieve and open the archive
-    os.system('/bin/bash %s%s' % (tahoe_cfg_dir, configure_cmd))
+    archive_file = backup_info(name)['path']
-    os.system('cp %s/tahoe.cfg /home/yunohost.backup/tahoe/' % tahoe_cfg_dir)
+    try:
-    #os.system('update-rc.d tahoe-lafs defaults')
+        tar = tarfile.open(archive_file, "r:gz")
-    #os.system('service tahoe-lafs restart')
+    except:
+        logger.exception("unable to open the archive '%s' for reading",
+                         archive_file)
+        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed'))
+
+    # Check temporary directory
+    tmp_dir = "%s/tmp/%s" % (backup_path, name)
+    if os.path.isdir(tmp_dir):
+        logger.warning("temporary directory for restoration '%s' already exists",
+                       tmp_dir)
+        os.system('rm -rf %s' % tmp_dir)
+
+    # Extract the tarball
+    msignals.display(m18n.n('backup_extracting_archive'))
+    tar.extractall(tmp_dir)
+    tar.close()
+
+    # Retrieve backup info
+    try:
+        with open("%s/info.json" % tmp_dir, 'r') as f:
+            info = json.load(f)
+    except IOError:
+        logger.error("unable to retrieve backup info from '%s/info.json'",
+                     tmp_dir)
+        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
+    else:
+        logger.info("restoring from backup '%s' created on %s", name,
+                    time.ctime(info['created_at']))
+
+    # Retrieve domain from the backup
+    try:
+        with open("%s/yunohost/current_host" % tmp_dir, 'r') as f:
+            domain = f.readline().rstrip()
+    except IOError:
+        logger.error("unable to retrieve domain from '%s/yunohost/current_host'",
+                     tmp_dir)
+        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
+
+    # Check if YunoHost is installed
+    if os.path.isfile('/etc/yunohost/installed'):
+        msignals.display(m18n.n('yunohost_already_installed'), 'warning')
+        if not force:
+            try:
+                # Ask confirmation for restoring
+                i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed',
+                                           answers='y/N'))
+            except NotImplemented:
+                pass
+            else:
+                if i == 'y' or i == 'Y':
+                    force = True
+            if not force:
+                raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
+    else:
+        from yunohost.tools import tools_postinstall
+        logger.info("executing the post-install...")
+        tools_postinstall(domain, 'yunohost', True)
+
+    # Add apps restore hook
+    if not ignore_apps:
+        for app_id in info['apps'].keys():
+            hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id
+            if os.path.isfile(hook):
+                hook_add(app_id, hook)
+                logger.info("app '%s' will be restored", app_id)
+            else:
+                msignals.display(m18n.n('unrestore_app', app_id), 'warning')
+
+    # Run hooks
+    msignals.display(m18n.n('restore_running_hooks'))
+    hook_callback('restore', hooks, args=[tmp_dir])
+
+    # Remove temporary directory
+    os.system('rm -rf %s' % tmp_dir)
+
+    msignals.display(m18n.n('restore_complete'), 'success')
+
+
+def backup_list():
+    """
+    List available local backup archives
+
+    """
+    result = []
+
+    try:
+        # Retrieve local archives
+        archives = os.listdir(archives_path)
+    except IOError as e:
+        logging.info("unable to iterate over local archives: %s", str(e))
+    else:
+        # Iterate over local archives
+        for f in archives:
+            try:
+                name = f[:f.rindex('.tar.gz')]
+            except ValueError:
+                continue
+            result.append(name)
+
+    return { 'archives': result }
+
+
+def backup_info(name):
+    """
+    Get info about a local backup archive
+
+    Keyword arguments:
+        name -- Name of the local backup archive
+
+    """
+    archive_file = '%s/%s.tar.gz' % (archives_path, name)
+    if not os.path.isfile(archive_file):
+        logger.error("no local backup archive found at '%s'", archive_file)
+        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown'))
+
+    info_file = "%s/%s.info.json" % (archives_path, name)
+    try:
+        with open(info_file) as f:
+            # Retrieve backup info
+            info = json.load(f)
+    except:
+        # TODO: Attempt to extract backup info file from tarball
+        logger.exception("unable to retrive backup info file '%s'",
+                         info_file)
+        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
+
+    return {
+        'path': archive_file,
+        'created_at': time.strftime(m18n.n('format_datetime_short'),
+                                    time.gmtime(info['created_at'])),
+        'description': info['description'],
+        'apps': info['apps'],
+    }

bin/yunohost

@@ -4,56 +4,158 @@
 import sys
 import os
 
-from_source = False
+# Either we are in a development environment or not
+IN_DEVEL = False
 
-# Run from source
+# Either cache has to be used inside the moulinette or not
-basedir = os.path.abspath('%s/../' % os.path.dirname(__file__))
+USE_CACHE = True
-if os.path.isdir('%s/moulinette' % basedir):
-    sys.path.insert(0, basedir)
-    from_source = True
 
-from moulinette import init, cli
+# Either the output has to be encoded as a JSON encoded string or not
+PRINT_JSON = False
+
+# Level for which loggers will log
+LOGGERS_LEVEL = 'INFO'
+
+# Handlers that will be used by loggers
+#  - file: log to the file LOG_DIR/LOG_FILE
+#  - console: log to stderr
+LOGGERS_HANDLERS = ['file']
+
+# Directory and file to be used by logging
+LOG_DIR = '/var/log/yunohost'
+LOG_FILE = 'yunohost-cli.log'
+
+
+# Initialization & helpers functions -----------------------------------
+
+def _die(message, title='Error:'):
+    """Print error message and exit"""
+    try:
         from moulinette.interfaces.cli import colorize
+    except ImportError:
+        colorize = lambda msg, c: msg
+    print('%s %s' % (colorize(title, 'red'), message))
+    sys.exit(1)
+
+def _check_in_devel():
+    """Check and load if needed development environment"""
+    global IN_DEVEL, LOG_DIR
+    basedir = os.path.abspath('%s/../' % os.path.dirname(__file__))
+    if os.path.isdir('%s/moulinette' % basedir) and not IN_DEVEL:
+        # Add base directory to python path
+        sys.path.insert(0, basedir)
+
+        # Update global variables
+        IN_DEVEL = True
+        LOG_DIR = '%s/log' % basedir
+
+def _parse_argv():
+    """Parse additional arguments and return remaining ones"""
+    argv = list(sys.argv)
+    argv.pop(0)
+
+    if '--no-cache' in argv:
+        global USE_CACHE
+        USE_CACHE = False
+        argv.remove('--no-cache')
+    if '--json' in argv:
+        global PRINT_JSON
+        PRINT_JSON = True
+        argv.remove('--json')
+    if '--debug' in argv:
+        global LOGGERS_LEVEL
+        LOGGERS_LEVEL = 'DEBUG'
+        argv.remove('--debug')
+    if '--verbose' in argv:
+        global LOGGERS_HANDLERS
+        if 'console' not in LOGGERS_HANDLERS:
+            LOGGERS_HANDLERS.append('console')
+        argv.remove('--verbose')
+    return argv
+
+def _init_moulinette():
+    """Configure logging and initialize the moulinette"""
+    from moulinette import init
+
+    # Custom logging configuration
+    logging = {
+        'version': 1,
+        'disable_existing_loggers': True,
+        'formatters': {
+            'simple': {
+                'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s - %(message)s'
+            },
+            'precise': {
+                'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(message)s'
+            },
+        },
+        'handlers': {
+            'console': {
+                'class': 'logging.StreamHandler',
+                'formatter': 'simple',
+                'stream': 'ext://sys.stderr',
+            },
+            'file': {
+                'class': 'logging.FileHandler',
+                'formatter': 'precise',
+                'filename': '%s/%s' % (LOG_DIR, LOG_FILE),
+            },
+        },
+        'loggers': {
+            'moulinette': {
+                'level': LOGGERS_LEVEL,
+                'handlers': LOGGERS_HANDLERS,
+            },
+            'yunohost': {
+                'level': LOGGERS_LEVEL,
+                'handlers': LOGGERS_HANDLERS,
+            },
+        },
+    }
+
+    # Create log directory
+    if not os.path.isdir(LOG_DIR):
+        try:
+            os.makedirs(LOG_DIR, 0750)
+        except os.error as e:
+            _die(str(e))
+
+    # Initialize moulinette
+    init(logging_config=logging, _from_source=IN_DEVEL)
+
+def _retrieve_namespaces():
+    """Return the list of namespaces to load"""
+    from moulinette.actionsmap import ActionsMap
+    ret = ['yunohost']
+    for n in ActionsMap.get_namespaces():
+        # Append YunoHost modules
+        if n.startswith('ynh_'):
+            ret.append(n)
+    return ret
 
 
-## Main action
+# Main action ----------------------------------------------------------
 
 if __name__ == '__main__':
-    # Run from source
+    _check_in_devel()
-    init(_from_source=from_source)
+    args = _parse_argv()
-
+    _init_moulinette()
-    # Additional arguments
-    cache = True
-    json = False
-    if '--no-cache' in sys.argv:
-        cache = False
-        sys.argv.remove('--no-cache')
-    if '--json' in sys.argv:
-        json = True
-        sys.argv.remove('--json')
-
-    # Retrieve remaining arguments
-    args = list(sys.argv)
-    args.pop(0)
 
     # Check that YunoHost is installed
     if not os.path.isfile('/etc/yunohost/installed') and \
-       (len(args) < 2 or args[0] != 'tools' or args[1] != 'postinstall'):
+       (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \
-        from moulinette.interfaces.cli import colorize, get_locale
+                          args[0] +' '+ args[1] != 'backup restore')):
+        from moulinette.interfaces.cli import get_locale
 
         # Init i18n
         m18n.load_namespace('yunohost')
         m18n.set_locale(get_locale())
 
         # Print error and exit
-        print('%s %s' % (colorize(m18n.g('error'), 'red'),
+        _die(m18n.n('yunohost_not_installed'), m18n.g('error'))
-                         m18n.n('yunohost_not_installed')))
-        sys.exit(1)
 
     # Execute the action
-    from os import listdir
+    from moulinette import cli
-    from os.path import isfile, join
+    ret = cli(_retrieve_namespaces(), args,
-    path='/usr/share/moulinette/actionsmap/'
+              print_json=PRINT_JSON, use_cache=USE_CACHE)
-    modules = [ f[:-4] for f in listdir(path) if isfile(join(path,f))]
-    ret = cli(modules, args, print_json=json, use_cache=cache)
     sys.exit(ret)

bin/yunohost-api

@@ -7,18 +7,137 @@ import yaml
 from os import listdir
 from os.path import isfile, join
 
-from_source = False
+# Either we are in a development environment or not
+IN_DEVEL = False
 
-# Run from source
+# Either cache has to be used inside the moulinette or not
+USE_CACHE = True
+
+# Either WebSocket has to be installed by the moulinette or not
+USE_WEBSOCKET = True
+
+# Level for which loggers will log
+LOGGERS_LEVEL = 'INFO'
+
+# Handlers that will be used by loggers
+#  - file: log to the file LOG_DIR/LOG_FILE
+#  - console: log to stderr
+LOGGERS_HANDLERS = ['file']
+
+# Directory and file to be used by logging
+LOG_DIR = '/var/log/yunohost'
+LOG_FILE = 'yunohost-api.log'
+
+
+# Initialization & helpers functions -----------------------------------
+
+def _die(message, title='Error:'):
+    """Print error message and exit"""
+    try:
+        from moulinette.interfaces.cli import colorize
+    except ImportError:
+        colorize = lambda msg, c: msg
+    print('%s %s' % (colorize(title, 'red'), message))
+    sys.exit(1)
+
+def _check_in_devel():
+    """Check and load if needed development environment"""
+    global IN_DEVEL, LOG_DIR
     basedir = os.path.abspath('%s/../' % os.path.dirname(__file__))
-if os.path.isdir('%s/moulinette' % basedir):
+    if os.path.isdir('%s/moulinette' % basedir) and not IN_DEVEL:
+        # Add base directory to python path
         sys.path.insert(0, basedir)
-    from_source = True
 
-from moulinette import init, api, MoulinetteError
+        # Update global variables
+        IN_DEVEL = True
+        LOG_DIR = '%s/log' % basedir
+
+def _parse_argv():
+    """Parse additional arguments and return remaining ones"""
+    argv = list(sys.argv)
+    argv.pop(0)
+
+    if '--no-cache' in argv:
+        global USE_CACHE
+        USE_CACHE = False
+        argv.remove('--no-cache')
+    if '--no-websocket' in argv:
+        global USE_WEBSOCKET
+        USE_WEBSOCKET = False
+        argv.remove('--no-websocket')
+    if '--debug' in argv:
+        global LOGGERS_LEVEL
+        LOGGERS_LEVEL = 'DEBUG'
+        argv.remove('--debug')
+    if '--verbose' in argv:
+        global LOGGERS_HANDLERS
+        if 'console' not in LOGGERS_HANDLERS:
+            LOGGERS_HANDLERS.append('console')
+        argv.remove('--verbose')
+    return argv
+
+def _init_moulinette():
+    """Configure logging and initialize the moulinette"""
+    from moulinette import init
+
+    # Custom logging configuration
+    logging = {
+        'version': 1,
+        'disable_existing_loggers': True,
+        'formatters': {
+            'simple': {
+                'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s - %(message)s'
+            },
+            'precise': {
+                'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(message)s'
+            },
+        },
+        'handlers': {
+            'console': {
+                'class': 'logging.StreamHandler',
+                'formatter': 'simple',
+                'stream': 'ext://sys.stderr',
+            },
+            'file': {
+                'class': 'logging.handlers.WatchedFileHandler',
+                'formatter': 'precise',
+                'filename': '%s/%s' % (LOG_DIR, LOG_FILE),
+            },
+        },
+        'loggers': {
+            'moulinette': {
+                'level': LOGGERS_LEVEL,
+                'handlers': LOGGERS_HANDLERS,
+            },
+            'yunohost': {
+                'level': LOGGERS_LEVEL,
+                'handlers': LOGGERS_HANDLERS,
+            },
+        },
+    }
+
+    # Create log directory
+    if not os.path.isdir(LOG_DIR):
+        try:
+            os.makedirs(LOG_DIR, 0750)
+        except os.error as e:
+            _die(str(e))
+
+    # Initialize moulinette
+    init(logging_config=logging, _from_source=IN_DEVEL)
+
+def _retrieve_namespaces():
+    """Return the list of namespaces to load"""
+    from moulinette.actionsmap import ActionsMap
+    ret = ['yunohost']
+    for n in ActionsMap.get_namespaces():
+        # Append YunoHost modules
+        if n.startswith('ynh_'):
+            ret.append(n)
+    return ret
 
 
-## Callbacks for additional routes
+# Callbacks for additional routes --------------------------------------
 
 def is_installed():
     """
@@ -31,45 +150,20 @@ def is_installed():
     return { 'installed': installed }
 
 
-def _get_modules():
+# Main action ----------------------------------------------------------
-    """
-    Get a dict of managed services with their parameters
 
-    """
-    from os import listdir
-    from os.path import isfile, join
-    path='/usr/share/moulinette/actionsmap/'
-    onlyfiles = [ f[:-4] for f in listdir(path) if isfile(join(path,f))]
-    return onlyfiles
-
-## Main action
 
 if __name__ == '__main__':
-    # Run from source
+    _check_in_devel()
-    init(_from_source=from_source)
+    _parse_argv()
-
+    _init_moulinette()
-    # Additional arguments
-    cache = True
-    websocket = True
-    if '--no-cache' in sys.argv:
-        cache = False
-        sys.argv.remove('--no-cache')
-    if '--no-websocket' in sys.argv:
-        websocket = False
-        sys.argv.remove('--no-websocket')
-    # TODO: Add log argument
 
+    from moulinette import (api, MoulinetteError)
     try:
         # Run the server
-        from os import listdir
+        api(_retrieve_namespaces(), port=6787,
-        from os.path import isfile, join
-        path='/usr/share/moulinette/actionsmap/'
-        modules = [ f[:-4] for f in listdir(path) if isfile(join(path,f))]
-        api(modules, port=6787,
             routes={('GET', '/installed'): is_installed},
-            use_cache=False, use_websocket=websocket)
+            use_cache=USE_CACHE, use_websocket=USE_WEBSOCKET)
     except MoulinetteError as e:
-        from moulinette.interfaces.cli import colorize
+        _die(e.strerror, m18n.g('error'))
-        print('%s %s' % (colorize(m18n.g('error'), 'red'), e.strerror))
-        sys.exit(e.errno)
     sys.exit(0)

data/services.yml

@@ -39,3 +39,6 @@ yunohost-api:
 postgrey:
    status: service
    log: /var/log/mail.log
+amavis:
+   status: service
+   log: /var/log/mail.log

domain.py

@@ -92,7 +92,7 @@ def domain_add(auth, domain, dyndns=False):
         from yunohost.dyndns import dyndns_subscribe
 
         try:
-            r = requests.get('http://dyndns.yunohost.org/domains')
+            r = requests.get('https://dyndns.yunohost.org/domains')
         except ConnectionError:
             pass
         else:
@@ -147,46 +147,33 @@ def domain_add(auth, domain, dyndns=False):
 
         attr_dict['virtualdomain'] = domain
 
+        dnsmasq_config_path='/etc/dnsmasq.d'
         try:
-            with open('/var/lib/bind/%s.zone' % domain) as f: pass
+            os.listdir(dnsmasq_config_path)
+        except OSError:
+            msignals.display(m18n.n('dnsmasq_isnt_installed'),
+                                  'warning')
+            os.makedirs(dnsmasq_config_path)
+
+        try:
+            with open('%s/%s' % (dnsmasq_config_path, domain)) as f: pass
         except IOError as e:
             zone_lines = [
-             '$TTL    38400',
+             'address=/%s/%s' % (domain, ip),
-             '%s.      IN   SOA   ns.%s. root.%s. %s 10800 3600 604800 38400' % (domain, domain, domain, timestamp),
+             'txt-record=%s,"v=spf1 mx a -all"' % domain,
-             '%s.      IN   NS    ns.%s.'                         % (domain, domain),
+             'mx-host=%s,%s,5' % (domain, domain),
-             '%s.      IN   A     %s'                             % (domain, ip),
+             'srv-host=_xmpp-client._tcp.%s,%s,5222,0,5' % (domain, domain),
-             '%s.      IN   MX    5 %s.'                          % (domain, domain),
+             'srv-host=_xmpp-server._tcp.%s,%s,5269,0,5' % (domain, domain),
-             '%s.      IN   TXT   "v=spf1 mx a -all"'             % domain,
+             'srv-host=_jabber._tcp.%s,%s,5269,0,5' % (domain, domain),
-             'ns.%s.   IN   A     %s'                             % (domain, ip),
-             '_xmpp-client._tcp.%s.  IN   SRV   0  5   5222  %s.' % (domain, domain),
-             '_xmpp-server._tcp.%s.  IN   SRV   0  5   5269  %s.' % (domain, domain),
-             '_jabber._tcp.%s.       IN   SRV   0  5   5269  %s.' % (domain, domain),
             ]
-            with open('/var/lib/bind/%s.zone' % domain, 'w') as zone:
+            with open('%s/%s' % (dnsmasq_config_path, domain), 'w') as zone:
                 for line in zone_lines:
                     zone.write(line + '\n')
-
+            os.system('service dnsmasq restart')
-            os.system('chown bind /var/lib/bind/%s.zone' % domain)
 
         else:
-            raise MoulinetteError(errno.EEXIST,
+            msignals.display(m18n.n('domain_zone_exists'),
-                                  m18n.n('domain_zone_exists'))
+                                 'warning')
-
-        conf_lines = [
-            'zone "%s" {' % domain,
-            '    type master;',
-            '    file "/var/lib/bind/%s.zone";' % domain,
-            '    allow-transfer {',
-            '        127.0.0.1;',
-            '        localnets;',
-            '    };',
-            '};'
-        ]
-        with open('/etc/bind/named.conf.local', 'a') as conf:
-            for line in conf_lines:
-               conf.write(line + '\n')
-
-        os.system('service bind9 reload')
 
         # XMPP
         try:
@@ -265,7 +252,7 @@ def domain_remove(auth, domain, force=False):
     if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
         command_list = [
             'rm -rf /etc/yunohost/certs/%s' % domain,
-            'rm -f  /var/lib/bind/%s.zone' % domain,
+            'rm -f  /etc/dnsmasq.d/%s' % domain,
             'rm -rf /var/lib/metronome/%s' % domain.replace('.', '%2e'),
             'rm -f  /etc/metronome/conf.d/%s.cfg.lua' % domain,
             'rm -rf /etc/nginx/conf.d/%s.d' % domain,
@@ -275,24 +262,12 @@ def domain_remove(auth, domain, force=False):
             if os.system(command) != 0:
                 msignals.display(m18n.n('path_removal_failed', command[7:]),
                                  'warning')
-        with open('/etc/bind/named.conf.local', 'r') as conf:
-            conf_lines = conf.readlines()
-        with open('/etc/bind/named.conf.local', 'w') as conf:
-            in_block = False
-            for line in conf_lines:
-                if re.search(r'^zone "%s' % domain, line):
-                    in_block = True
-                if in_block:
-                    if re.search(r'^};$', line):
-                        in_block = False
-                else:
-                    conf.write(line)
     else:
         raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))
 
     os.system('yunohost app ssowatconf > /dev/null 2>&1')
     os.system('service nginx reload')
-    os.system('service bind9 reload')
+    os.system('service dnsmasq restart')
     os.system('service metronome restart')
 
     msignals.display(m18n.n('domain_deleted'), 'success')

dyndns.py

@@ -50,9 +50,9 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
 
     # Verify if domain is available
     try:
-        if requests.get('http://%s/test/%s' % (subscribe_host, domain)).status_code != 200:
+        if requests.get('https://%s/test/%s' % (subscribe_host, domain)).status_code != 200:
             raise MoulinetteError(errno.EEXIST, m18n.n('dyndns_unavailable'))
-    except ConnectionError:
+    except requests.ConnectionError:
         raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection'))
 
     if key is None:
@@ -71,7 +71,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
 
     # Send subscription
     try:
-        r = requests.post('http://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={ 'subdomain': domain })
+        r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={ 'subdomain': domain })
     except ConnectionError:
         raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection'))
     if r.status_code != 201:

firewall.py

@@ -186,7 +186,8 @@ def firewall_reload():
         for port in firewall['ipv4'][protocol]:
             os.system("iptables -A INPUT -p %s --dport %d -j ACCEPT" % (protocol, port))
 
-    hook_callback('post_iptable_rules', [upnp, os.path.exists("/proc/net/if_inet6")])
+    hook_callback('post_iptable_rules',
+                  args=[upnp, os.path.exists("/proc/net/if_inet6")])
 
     os.system("iptables -A INPUT -i lo -j ACCEPT")
     os.system("iptables -A INPUT -p icmp -j ACCEPT")
@@ -235,8 +236,7 @@ def firewall_upnp(action=None):
 
         with open('/etc/cron.d/yunohost-firewall', 'w+') as f:
             f.write('PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-            \n*/50 * * * * root yunohost firewall upnp reload >>/dev/null \
+            \n*/50 * * * * root yunohost firewall upnp reload >>/dev/null')
-            \n*/50 * * * * root iptables -L | grep ^fail2ban-dovecot > /dev/null 2>&1; if [ $? != 0 ]; then yunohost firewall reload; fi >>/dev/null')
 
         msignals.display(m18n.n('upnp_enabled'), 'success')
 

hook.py

@@ -32,8 +32,13 @@ import subprocess
 from shlex import split as arg_split
 
 from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
 
 hook_folder = '/usr/share/yunohost/hooks/'
+custom_hook_folder = '/etc/yunohost/hooks.d/'
+
+logger = getActionLogger('yunohost.hook')
+
 
 def hook_add(app, file):
     """
@@ -45,16 +50,12 @@ def hook_add(app, file):
 
     """
     path, filename = os.path.split(file)
-    if '-' in filename:
+    priority, action = _extract_filename_parts(filename)
-        priority, action = filename.split('-')
-    else:
-        priority = '50'
-        action = filename
 
-    try: os.listdir(hook_folder + action)
+    try: os.listdir(custom_hook_folder + action)
-    except OSError: os.makedirs(hook_folder + action)
+    except OSError: os.makedirs(custom_hook_folder + action)
 
-    finalpath = hook_folder + action +'/'+ priority +'-'+ app
+    finalpath = custom_hook_folder + action +'/'+ priority +'-'+ app
     os.system('cp %s %s' % (file, finalpath))
     os.system('chown -hR admin: %s' % hook_folder)
 
@@ -70,34 +71,155 @@ def hook_remove(app):
 
     """
     try:
-        for action in os.listdir(hook_folder):
+        for action in os.listdir(custom_hook_folder):
-            for script in os.listdir(hook_folder + action):
+            for script in os.listdir(custom_hook_folder + action):
                 if script.endswith(app):
-                    os.remove(hook_folder + action +'/'+ script)
+                    os.remove(custom_hook_folder + action +'/'+ script)
     except OSError: pass
 
 
-def hook_callback(action, args=None):
+def hook_list(action, list_by='name', show_info=False):
+    """
+    List available hooks for an action
+
+    Keyword argument:
+        action -- Action name
+        list_by -- Property to list hook by
+        show_info -- Show hook information
+
+    """
+    result = {}
+
+    # Process the property to list hook by
+    if list_by == 'priority':
+        if show_info:
+            def _append_hook(d, priority, name, path):
+                # Use the priority as key and a dict of hooks names
+                # with their info as value
+                value = { 'path': path }
+                try:
+                    d[priority][name] = value
+                except KeyError:
+                    d[priority] = { name: value }
+        else:
+            def _append_hook(d, priority, name, path):
+                # Use the priority as key and the name as value
+                try:
+                    d[priority].add(name)
+                except KeyError:
+                    d[priority] = set([name])
+    elif list_by == 'name' or list_by == 'folder':
+        if show_info:
+            def _append_hook(d, priority, name, path):
+                # Use the name as key and a list of hooks info - the
+                # executed ones with this name - as value
+                l = d.get(name, list())
+                for h in l:
+                    # Only one priority for the hook is accepted
+                    if h['priority'] == priority:
+                        # Custom hooks overwrite system ones and they
+                        # are appended at the end - so overwite it
+                        if h['path'] != path:
+                            h['path'] = path
+                        return
+                l.append({ 'priority': priority, 'path': path })
+                d[name] = l
+        else:
+            if list_by == 'name':
+                result = set()
+            def _append_hook(d, priority, name, path):
+                # Add only the name
+                d.add(name)
+    else:
+        raise MoulinetteError(errno.EINVAL, m18n.n('hook_list_by_invalid'))
+
+    def _append_folder(d, folder):
+        # Iterate over and add hook from a folder
+        for f in os.listdir(folder + action):
+            path = '%s%s/%s' % (folder, action, f)
+            priority, name = _extract_filename_parts(f)
+            _append_hook(d, priority, name, path)
+
+    try:
+        # Append system hooks first
+        if list_by == 'folder':
+            result['system'] = dict() if show_info else set()
+            _append_folder(result['system'], hook_folder)
+        else:
+            _append_folder(result, hook_folder)
+    except OSError:
+        logger.debug("system hook folder not found for action '%s' in %s",
+                     action, hook_folder)
+
+    try:
+        # Append custom hooks
+        if list_by == 'folder':
+            result['custom'] = dict() if show_info else set()
+            _append_folder(result['custom'], custom_hook_folder)
+        else:
+            _append_folder(result, custom_hook_folder)
+    except OSError:
+        logger.debug("custom hook folder not found for action '%s' in %s",
+                     action, custom_hook_folder)
+
+    return { 'hooks': result }
+
+
+def hook_callback(action, hooks=[], args=None):
     """
     Execute all scripts binded to an action
 
     Keyword argument:
         action -- Action name
+        hooks -- List of hooks names to execute
         args -- Ordered list of arguments to pass to the script
 
     """
-    try: os.listdir(hook_folder + action)
+    result = { 'succeed': list(), 'failed': list() }
-    except OSError: pass
+    hooks_dict = {}
+
+    # Retrieve hooks
+    if not hooks:
+        hooks_dict = hook_list(action, list_by='priority',
+                               show_info=True)['hooks']
     else:
+        hooks_names = hook_list(action, list_by='name',
+                                show_info=True)['hooks']
+        # Iterate over given hooks names list
+        for n in hooks:
+            try:
+                hl = hooks_names[n]
+            except KeyError:
+                raise MoulinetteError(errno.EINVAL,
+                                      m18n.n('hook_name_unknown', n))
+            # Iterate over hooks with this name
+            for h in hl:
+                # Update hooks dict
+                d = hooks_dict.get(h['priority'], dict())
+                d.update({ n: { 'path': h['path'] }})
+                hooks_dict[h['priority']] = d
+    if not hooks_dict:
+        return result
+
+    # Format arguments
     if args is None:
         args = []
     elif not isinstance(args, list):
         args = [args]
 
-        for hook in os.listdir(hook_folder + action):
+    # Iterate over hooks and execute them
+    for priority in sorted(hooks_dict):
+        for name, info in iter(hooks_dict[priority].items()):
+            filename = '%s-%s' % (priority, name)
             try:
-                hook_exec(file=hook_folder + action +'/'+ hook, args=args)
+                hook_exec(info['path'], args=args)
-            except: pass
+            except:
+                logger.exception("error while executing hook '%s'",
+                                 info['path'])
+                result['failed'].append(filename)
+            else:
+                result['succeed'].append(filename)
+    return result
 
 
 def hook_check(file):
@@ -206,3 +328,13 @@ def hook_exec(file, args=None):
     stream.close()
 
     return returncode
+
+
+def _extract_filename_parts(filename):
+    """Extract hook parts from filename"""
+    if '-' in filename:
+        priority, action = filename.split('-', 1)
+    else:
+        priority = '50'
+        action = filename
+    return priority, action

hooks/backup/05-system_ldap

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+backup_dir="$1/ldap"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/ldap/slapd.conf $backup_dir/
+sudo slapcat -l $backup_dir/slapcat.ldif.raw
+egrep -v "^entryCSN:" < $backup_dir/slapcat.ldif.raw > $backup_dir/slapcat.ldif
+rm -f $backup_dir/slapcat.ldif.raw

hooks/backup/08-system_ssh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/ssh"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/ssh/. $backup_dir

hooks/backup/11-system_mysql

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+backup_dir="$1/mysql"
+mkdir -p $backup_dir
+
+mysqlpwd=$(sudo cat /etc/yunohost/mysql)
+sudo mysqldump -uroot -p"$mysqlpwd" mysql > $backup_dir/mysql.sql

hooks/backup/14-system_ssowat

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/ssowat"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/ssowat/. $backup_dir

hooks/backup/17-system_home

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/home"
+mkdir -p $backup_dir
+
+sudo rsync -a --exclude='/yunohost*' /home/ $backup_dir/

hooks/backup/20-system_yunohost

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/yunohost"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/yunohost/. $backup_dir

hooks/backup/23-system_mail

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+backup_dir="$1/mail"
+
+sudo cp -a /var/mail/. $backup_dir

hooks/backup/26-system_xmpp

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+backup_dir="$1/xmpp"
+mkdir -p $backup_dir/{etc,var}
+
+sudo cp -a /etc/metronome/. $backup_dir/etc
+sudo cp -a /var/lib/metronome/. $backup_dir/var

hooks/backup/29-system_nginx

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/nginx"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/nginx/conf.d/. $backup_dir

hooks/backup/32-system_cron

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/cron"
+mkdir -p $backup_dir
+
+sudo cp -a /etc/cron.d/yunohost* $backup_dir/

hooks/restore/05-system_ldap

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+backup_dir="$1/ldap"
+
+if [ -z "$2" ]; then
+
+    # We need to execute this script as root, since the ldap
+    # service will be shut down during the operation (and sudo
+    # won't be available)
+    sudo bash $(pwd)/$0 $1 sudoed
+
+else
+    service slapd stop
+
+    # Backup old configuration
+    mv /var/lib/ldap /var/lib/ldap.old
+
+    # Recreate new DB folder
+    mkdir /var/lib/ldap
+    chown openldap: /var/lib/ldap
+    chmod go-rwx  /var/lib/ldap
+
+    # Restore LDAP configuration (just to be sure)
+    cp -a $backup_dir/slapd.conf /etc/ldap/slapd.conf
+
+    # Regenerate the configuration
+    rm -rf /etc/ldap/slapd.d/*
+    slaptest -u -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d
+    cp -rfp /var/lib/ldap.old/DB_CONFIG /var/lib/ldap
+
+    # Import the database
+    slapadd -l $backup_dir/slapcat.ldif
+
+    # Change permissions and restart slapd
+    chown openldap: /var/lib/ldap/*
+    service slapd start
+    rm -rf /var/lib/ldap.old
+fi

hooks/restore/08-system_ssh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+backup_dir="$1/ssh"
+
+sudo cp -a $backup_dir/. /etc/ssh
+sudo service ssh restart

hooks/restore/11-system_mysql

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+backup_dir="$1/mysql"
+
+mysqlpwd=$(sudo cat /etc/yunohost/mysql)
+sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql
+sudo mysqladmin flush-privileges -p"$mysqlpwd"

hooks/restore/14-system_ssowat

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+backup_dir="$1/ssowat"
+
+sudo cp -a $backup_dir/. /etc/ssowat

hooks/restore/17-system_home

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+backup_dir="$1/home"
+
+sudo cp -a $backup_dir/. /home

hooks/restore/20-system_yunohost

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+backup_dir="$1/yunohost"
+
+sudo cp -a $backup_dir/. /etc/yunohost
+sudo yunohost app ssowatconf
+sudo yunohost firewall reload
+
+# Reload interface name
+sudo rm /etc/yunohost/interface
+sudo apt-get install --reinstall -y yunohost-config-others

hooks/restore/23-system_mail

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+backup_dir="$1/mail"
+
+sudo cp -a $backup_dir/. /var/mail
+
+# Restart services to use migrated certs
+sudo service postfix restart
+sudo service dovecot restart

hooks/restore/26-system_xmpp

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+backup_dir="$1/xmpp"
+
+sudo cp -a $backup_dir/etc/. /etc/metronome
+sudo cp -a $backup_dir/var/. /var/lib/metronome
+
+# Restart to apply new conf and certs
+sudo service metronome restart

hooks/restore/29-system_nginx

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+backup_dir="$1/nginx"
+
+sudo cp -a $backup_dir/. /etc/nginx/conf.d
+
+# Restart to use new conf and certs
+sudo service nginx restart

hooks/restore/32-system_cron

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+backup_dir="$1/cron"
+
+sudo cp -a $backup_dir/. /etc/cron.d
+
+# Restart just in case
+sudo service cron restart

locales/en.json

@@ -20,6 +20,7 @@
     "custom_app_url_required" : "You must provide an URL to upgrade your custom app {:s}",
     "app_recent_version_required" : "{:s} requires a more recent version of the moulinette",
     "app_upgraded" : "{:s} successfully upgraded",
+    "app_upgrade_failed" : "Unable to upgrade all apps",
     "app_id_invalid" : "Invalid app id",
     "app_already_installed" : "{:s} is already installed",
     "app_removed" : "{:s} successfully removed",
@@ -46,6 +47,7 @@
     "domain_dyndns_root_unknown" : "Unknown DynDNS root domain",
     "domain_cert_gen_failed" : "Unable to generate certificate",
     "domain_exists" : "Domain already exists",
+    "dnsmasq_isnt_installed" : "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'",
     "domain_zone_exists" : "DNS zone file already exists",
     "domain_zone_not_found" : "DNS zone file not found for domain {:s}",
     "domain_creation_failed" : "Unable to create domain",
@@ -76,6 +78,8 @@
     "upnp_disabled" : "uPnP successfully disabled",
     "firewall_reloaded" : "Firewall successfully reloaded",
 
+    "hook_list_by_invalid" : "Invalid property to list hook by",
+    "hook_name_unknown" : "Unknown hook name '{:s}'",
     "hook_choice_invalid" : "Invalid choice '{:s}'",
     "hook_argument_missing" : "Missing argument '{:s}'",
 
@@ -127,6 +131,24 @@
     "packages_upgrade_failed" : "Unable to upgrade all packages",
     "system_upgraded" : "System successfully upgraded",
 
+    "backup_output_directory_required" : "You must provide an output directory for the backup",
+    "backup_output_directory_forbidden" : "Forbidden output directory",
+    "backup_output_directory_not_empty" : "Output directory is not empty",
+    "backup_running_hooks" : "Running backup hooks...",
+    "backup_creating_archive" : "Creating the backup archive...",
+    "backup_extracting_archive" : "Extracting the backup archive...",
+    "backup_archive_open_failed" : "Unable to open the backup archive",
+    "backup_archive_name_unknown" : "Unknown local backup archive name",
+    "backup_archive_name_exists" : "Backup archive name already exists",
+    "backup_complete" : "Backup complete",
+    "backup_invalid_archive" : "Invalid backup archive",
+    "restore_confirm_yunohost_installed" : "Do you really want to restore an already installed system? [{answers:s}]",
+    "restore_running_hooks" : "Running restoration hooks...",
+    "restore_failed" : "Unable to restore the system",
+    "restore_complete" : "Restore complete",
+    "unbackup_app" : "App '{:s}' will not be saved",
+    "unrestore_app" : "App '{:s}' will not be restored",
+
     "field_invalid" : "Invalid field '{:s}'",
     "mail_domain_unknown" : "Unknown mail address domain '{:s}'",
     "mail_alias_remove_failed" : "Unable to remove mail alias '{:s}'",
@@ -150,13 +172,16 @@
     "ask_new_admin_password" : "New administration password",
     "ask_main_domain" : "Main domain",
     "ask_list_to_remove" : "List to remove",
-    "pattern_username" : "Must be alphanumeric and underscore characters only",
+    "pattern_username" : "Must be lower-case alphanumeric and underscore characters only",
     "pattern_firstname" : "Must be a valid first name",
     "pattern_lastname" : "Must be a valid last name",
     "pattern_email" : "Must be a valid email address (e.g. someone@domain.org)",
     "pattern_password" : "Must be at least 3 characters long",
     "pattern_domain" : "Must be a valid domain name (e.g. my-domain.org)",
     "pattern_listname" : "Must be alphanumeric and underscore characters only",
-    "pattern_port" : "Must be a valid port number (i.e. 0-65535)"
+    "pattern_port" : "Must be a valid port number (i.e. 0-65535)",
+    "pattern_backup_archive_name" : "Must be a valid filename with alphanumeric and -_. characters only",
+
+    "format_datetime_short" : "%m/%d/%Y %I:%M %p"
 }
 

locales/fr.json

@@ -20,6 +20,7 @@
     "custom_app_url_required" : "Vous devez spécifier une URL pour mettre à jour votre application locale {:s}",
     "app_recent_version_required" : "{:s} nécessite une version plus récente de la moulinette",
     "app_upgraded" : "{:s} mis à jour avec succès",
+    "app_upgrade_failed" : "Impossible de mettre à jour toutes les applications",
     "app_id_invalid" : "Id d'application incorrect",
     "app_already_installed" : "{:s} est déjà installé",
     "app_removed" : "{:s} supprimé avec succès",
@@ -46,6 +47,7 @@
     "domain_dyndns_root_unknown" : "Domaine DynDNS principal inconnu",
     "domain_cert_gen_failed" : "Impossible de générer le certificat",
     "domain_exists" : "Le domaine existe déjà",
+    "dnsmasq_isnt_installed" : "dnsmasq ne semble pas être installé, veuillez lancer 'apt-get remove bind9 && apt-get install dnsmasq'",
     "domain_zone_exists" : "Le fichier de zone DNS existe déjà",
     "domain_zone_not_found" : "Fichier de zone DNS introuvable pour le domaine {:s}",
     "domain_creation_failed" : "Impossible de créer le domaine",
@@ -76,6 +78,8 @@
     "upnp_disabled" : "uPnP désactivé avec succès",
     "firewall_reloaded" : "Pare-feu rechargé avec succès",
 
+    "hook_list_by_invalid" : "Propriété pour lister les scripts incorrecte",
+    "hook_name_unknown" : "Nom de script '{:s}' inconnu",
     "hook_choice_invalid" : "Choix incorrect : '{:s}'",
     "hook_argument_missing" : "Argument manquant : '{:s}'",
 
@@ -127,6 +131,24 @@
     "packages_upgrade_failed" : "Impossible de mettre à jour tous les paquets",
     "system_upgraded" : "Système mis à jour avec succès",
 
+    "backup_output_directory_required" : "Vous devez spécifier un dossier de sortie pour la sauvegarde",
+    "backup_output_directory_forbidden" : "Dossier de sortie interdit",
+    "backup_output_directory_not_empty" : "Le dossier de sortie n'est pas vide",
+    "backup_running_hooks" : "Exécution des scripts de sauvegarde...",
+    "backup_creating_archive" : "Création de l'archive de sauvegarde...",
+    "backup_extracting_archive" : "Extraction de l'archive de sauvegarde...",
+    "backup_archive_open_failed" : "Impossible d'ouvrir l'archive de sauvegarde",
+    "backup_archive_name_unknown" : "Nom d'archive de sauvegarde locale inconnu",
+    "backup_archive_name_exists" : "Une archive de sauvegarde avec ce nom existe déjà",
+    "backup_complete" : "Sauvegarde terminée",
+    "backup_invalid_archive" : "Archive de sauvegarde incorrecte",
+    "restore_confirm_yunohost_installed" : "Voulez-vous vraiment restaurer un système déjà installé ? [{answers:s}]",
+    "restore_running_hooks" : "Exécution des scripts de restauration...",
+    "restore_failed" : "Impossible de restaurer le système",
+    "restore_complete" : "Restauration terminée",
+    "unbackup_app" : "L'application '{:s}' ne sera pas sauvegardée",
+    "unrestore_app" : "L'application '{:s}' ne sera pas restaurée",
+
     "field_invalid" : "Champ incorrect : {:s}",
     "mail_domain_unknown" : "Domaine '{:s}' de l'adresse mail inconnu",
     "mail_alias_remove_failed" : "Impossible de supprimer l'adresse mail supplémentaire '{:s}'",
@@ -150,13 +172,16 @@
     "ask_new_admin_password" : "Nouveau mot de passe d'administration",
     "ask_main_domain" : "Domaine principal",
     "ask_list_to_remove" : "Liste à supprimer",
-    "pattern_username" : "Doit être composé uniquement de caractères alphanumérique et de tiret bas",
+    "pattern_username" : "Doit être composé uniquement de caractères alphanumérique minuscule et de tiret bas",
     "pattern_firstname" : "Doit être un prénom valide",
     "pattern_lastname" : "Doit être un nom valide",
     "pattern_email" : "Doit être une adresse mail valide (ex. : someone@domain.org)",
     "pattern_password" : "Doit être composé d'au moins 3 caractères",
     "pattern_domain" : "Doit être un nom de domaine valide (ex : mon-domaine.org)",
     "pattern_listname" : "Doit être composé uniquement de caractères alphanumérique et de tiret bas",
-    "pattern_port" : "Doit être un numéro de port valide (0-65535)"
+    "pattern_port" : "Doit être un numéro de port valide (0-65535)",
+    "pattern_backup_archive_name" : "Doit être un nom de fichier valide composé de caractères alphanumérique et -_. uniquement",
+
+    "format_datetime_short" : "%d/%m/%Y %H:%M"
 }
 

tools.py

@@ -31,13 +31,18 @@ import getpass
 import requests
 import json
 import errno
+import logging
 import apt
 import apt.progress
 
 from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
 
 apps_setting_path= '/etc/yunohost/apps/'
 
+logger = getActionLogger('yunohost.tools')
+
+
 def tools_ldapinit(auth):
     """
     YunoHost LDAP initialization
@@ -72,26 +77,22 @@ def tools_ldapinit(auth):
     msignals.display(m18n.n('ldap_initialized'), 'success')
 
 
-def tools_adminpw(old_password, new_password):
+def tools_adminpw(auth, new_password):
     """
     Change admin password
 
     Keyword argument:
         new_password
-        old_password
 
     """
-    old_password.replace('"', '\\"')
+    try:
-    old_password.replace('&', '\\&')
+        auth.con.passwd_s('cn=admin,dc=yunohost,dc=org', None, new_password)
-    new_password.replace('"', '\\"')
+    except:
-    new_password.replace('&', '\\&')
+        logger.exception('unable to change admin password')
-    result = os.system('ldappasswd -h localhost -D cn=admin,dc=yunohost,dc=org -w "%s" -a "%s" -s "%s"' % (old_password, old_password, new_password))
-
-    if result == 0:
-        msignals.display(m18n.n('admin_password_changed'), 'success')
-    else:
         raise MoulinetteError(errno.EPERM,
                               m18n.n('admin_password_change_failed'))
+    else:
+        msignals.display(m18n.n('admin_password_changed'), 'success')
 
 
 def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
@@ -123,7 +124,6 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
         '/etc/metronome/metronome.cfg.lua',
         '/etc/dovecot/dovecot.conf',
         '/usr/share/yunohost/yunohost-config/others/startup',
-        '/home/yunohost.backup/tahoe/tahoe.cfg',
         '/etc/amavis/conf.d/05-node_id',
         '/etc/amavis/conf.d/50-user'
     ]
@@ -174,6 +174,8 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
     os.system('rm /etc/ssl/certs/yunohost_crt.pem')
 
     command_list = [
+        'rm -f /etc/nginx/conf.d/%s.d/yunohost_local.conf' % old_domain,
+        'cp /usr/share/yunohost/yunohost-config/nginx/yunohost_local.conf /etc/nginx/conf.d/%s.d/' % new_domain,
         'ln -s /etc/yunohost/certs/%s/key.pem /etc/ssl/private/yunohost_key.pem' % new_domain,
         'ln -s /etc/yunohost/certs/%s/crt.pem /etc/ssl/certs/yunohost_crt.pem'   % new_domain,
         'echo %s > /etc/yunohost/current_host' % new_domain,
@@ -189,10 +191,9 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
             raise MoulinetteError(errno.EPERM,
                                   m18n.n('maindomain_change_failed'))
 
-    if dyndns: dyndns_subscribe(domain=new_domain)
+    if dyndns and len(new_domain.split('.')) >= 3:
-    elif len(new_domain.split('.')) >= 3:
         try:
-            r = requests.get('http://dyndns.yunohost.org/domains')
+            r = requests.get('https://dyndns.yunohost.org/domains')
         except ConnectionError:
             pass
         else:
@@ -204,22 +205,23 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
     msignals.display(m18n.n('maindomain_changed'), 'success')
 
 
-def tools_postinstall(domain, password, dyndns=False):
+def tools_postinstall(domain, password, ignore_dyndns=False):
     """
     YunoHost post-install
 
     Keyword argument:
         domain -- YunoHost main domain
-        dyndns -- Subscribe domain to a DynDNS service
+        ignore_dyndns -- Do not subscribe domain to a DynDNS service
         password -- YunoHost admin password
 
     """
     from moulinette.core import init_authenticator
 
-    from yunohost.backup import backup_init
     from yunohost.app import app_ssowatconf
     from yunohost.firewall import firewall_upnp, firewall_reload
 
+    dyndns = not ignore_dyndns
+
     try:
         with open('/etc/yunohost/installed') as f: pass
     except IOError:
@@ -227,16 +229,16 @@ def tools_postinstall(domain, password, dyndns=False):
     else:
         raise MoulinetteError(errno.EPERM, m18n.n('yunohost_already_installed'))
 
-    if len(domain.split('.')) >= 3:
+    if len(domain.split('.')) >= 3 and not ignore_dyndns:
         try:
-            r = requests.get('http://dyndns.yunohost.org/domains')
+            r = requests.get('https://dyndns.yunohost.org/domains')
         except ConnectionError:
             pass
         else:
             dyndomains = json.loads(r.text)
             dyndomain  = '.'.join(domain.split('.')[1:])
             if dyndomain in dyndomains:
-                if requests.get('http://dyndns.yunohost.org/test/%s' % domain).status_code == 200:
+                if requests.get('https://dyndns.yunohost.org/test/%s' % domain).status_code == 200:
                     dyndns=True
                 else:
                     raise MoulinetteError(errno.EEXIST,
@@ -255,6 +257,9 @@ def tools_postinstall(domain, password, dyndns=False):
         try: os.listdir(folder)
         except OSError: os.makedirs(folder)
 
+    # Change folders permissions
+    os.system('chmod 755 /home/yunohost.app')
+
     # Set hostname to avoid amavis bug
     if os.system('hostname -d') != 0:
         os.system('hostname yunohost.yunohost.org')
@@ -304,9 +309,6 @@ def tools_postinstall(domain, password, dyndns=False):
     # Initialize YunoHost LDAP base
     tools_ldapinit(auth)
 
-    # Initialize backup system
-    backup_init()
-
     # New domain config
     tools_maindomain(auth, old_domain='yunohost.org', new_domain=domain, dyndns=dyndns)
 
@@ -323,6 +325,9 @@ def tools_postinstall(domain, password, dyndns=False):
     except MoulinetteError:
         firewall_upnp(action=['disable'])
 
+    # Enable iptables at boot time
+    os.system('update-rc.d yunohost-firewall defaults')
+
     os.system('touch /etc/yunohost/installed')
 
     msignals.display(m18n.n('yunohost_configured'), 'success')
@@ -408,6 +413,9 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
     """
     from yunohost.app import app_upgrade
 
+    failure = False
+
+    # Retrieve interface
     is_api = True if msettings.get('interface') == 'api' else False
 
     if not ignore_packages:
@@ -441,6 +449,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
                 cache.commit(apt.progress.text.AcquireProgress(),
                              apt.progress.base.InstallProgress())
             except Exception as e:
+                failure = True
                 logging.warning('unable to upgrade packages: %s' % str(e))
                 msignals.display(m18n.n('packages_upgrade_failed'), 'error')
             else:
@@ -451,11 +460,15 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
     if not ignore_apps:
         try:
             app_upgrade(auth)
-        except: pass
+        except Exception as e:
+            failure = True
+            logging.warning('unable to upgrade apps: %s' % str(e))
+            msignals.display(m18n.n('app_upgrade_failed'), 'error')
 
+    if not failure:
         msignals.display(m18n.n('system_upgraded'), 'success')
 
     # Return API logs if it is an API call
-    if msettings.get('interface') == 'api':
+    if is_api:
         from yunohost.service import service_log
         return { "log": service_log('yunohost-api', number="100").values()[0] }

user.py

@@ -186,7 +186,8 @@ def user_create(auth, username, firstname, lastname, mail, password):
             app_ssowatconf(auth)
             #TODO: Send a welcome mail to user
             msignals.display(m18n.n('user_created'), 'success')
-            hook_callback('post_user_create', [username, mail, password, firstname, lastname])
+            hook_callback('post_user_create',
+                          args=[username, mail, password, firstname, lastname])
 
             return { 'fullname' : fullname, 'username' : username, 'mail' : mail }