Merge branch 'stretch-unstable' into app-stdinfo

data/actionsmap/yunohost.yml

@@ -1710,3 +1710,39 @@ hook:
                 -d:
                     full: --chdir
                     help: The directory from where the script will be executed
+
+#############################
+#            Log            #
+#############################
+log:
+    category_help: Manage debug logs
+    actions:
+
+        ### log_list()
+        list:
+            action_help: List logs
+            api: GET /logs
+            arguments:
+                category:
+                    help: Log category to display (default operations), could be operation, history, package, system, access, service or app 
+                    nargs: "*"
+                -l:
+                    full: --limit
+                    help: Maximum number of logs
+                    type: int
+
+        ### log_display()
+        display:
+            action_help: Display a log content
+            api: GET /logs/display
+            arguments:
+                path:
+                    help: Log file which to display the content
+                -n:
+                    full: --number
+                    help: Number of lines to display
+                    default: 50
+                    type: int
+                --share:
+                    help: Share the full log using yunopaste
+                    action: store_true

data/helpers.d/utils

@@ -257,3 +257,20 @@ ynh_local_curl () {
 	# Curl the URL
 	curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url"
 }
+
+# Render templates with Jinja2
+# 
+# Attention : Variables should be exported before calling this helper to be
+# accessible inside templates.
+#
+# usage: ynh_render_template some_template output_path
+# | arg: some_template - Template file to be rendered
+# | arg: output_path   - The path where the output will be redirected to
+ynh_render_template() {
+   local template_path=$1
+   local output_path=$2
+   # Taken from https://stackoverflow.com/a/35009576
+   python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
+                    jinja2.Template(sys.stdin.read()
+                 ).render(os.environ));' < $template_path > $output_path
+}

data/hooks/conf_regen/19-postfix

@@ -10,15 +10,25 @@ do_pre_regen() {
   postfix_dir="${pending_dir}/etc/postfix"
   mkdir -p "$postfix_dir"
 
+  default_dir="${pending_dir}/etc/default/"
+  mkdir -p "$default_dir"
+
   # install plain conf files
   cp plain/* "$postfix_dir"
 
   # prepare main.cf conf file
   main_domain=$(cat /etc/yunohost/current_host)
+  domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ')
+
   cat main.cf \
     | sed "s/{{ main_domain }}/${main_domain}/g" \
     > "${postfix_dir}/main.cf"
 
+  cat postsrsd \
+    | sed "s/{{ main_domain }}/${main_domain}/g" \
+    | sed "s/{{ domain_list }}/${domain_list}/g" \
+    > "${default_dir}/postsrsd"
+
   # adapt it for IPv4-only hosts
   if [ ! -f /proc/net/if_inet6 ]; then
     sed -i \
@@ -34,7 +44,8 @@ do_post_regen() {
   regen_conf_files=$1
 
   [[ -z "$regen_conf_files" ]] \
-    || sudo service postfix restart
+    || { sudo service postfix restart && sudo service postsrsd restart; }
+
 }
 
 FORCE=${2:-0}

data/templates/postfix/main.cf

@@ -137,8 +137,10 @@ smtpd_recipient_restrictions =
     permit
 
 # SRS
-sender_canonical_maps = regexp:/etc/postfix/sender_canonical
+sender_canonical_maps = tcp:localhost:10001
 sender_canonical_classes = envelope_sender
+recipient_canonical_maps = tcp:localhost:10002
+recipient_canonical_classes= envelope_recipient,header_recipient
 
 # Ignore some headers
 smtp_header_checks = regexp:/etc/postfix/header_checks

data/templates/postfix/postsrsd

@@ -0,0 +1,43 @@
+# Default settings for postsrsd
+
+# Local domain name.
+# Addresses are rewritten to originate from this domain. The default value
+# is taken from `postconf -h mydomain` and probably okay.
+#
+SRS_DOMAIN={{ main_domain }}
+
+# Exclude additional domains.
+# You may list domains which shall not be subjected to address rewriting.
+# If a domain name starts with a dot, it matches all subdomains, but not
+# the domain itself. Separate multiple domains by space or comma.
+# We have to put some "dummy" stuff at start and end... see this comment :
+# https://github.com/roehling/postsrsd/issues/64#issuecomment-284003762
+SRS_EXCLUDE_DOMAINS=dummy {{ domain_list }} dummy
+
+# First separator character after SRS0 or SRS1.
+# Can be one of: -+=
+SRS_SEPARATOR==
+
+# Secret key to sign rewritten addresses.
+# When postsrsd is installed for the first time, a random secret is generated
+# and stored in /etc/postsrsd.secret. For most installations, that's just fine.
+#
+SRS_SECRET=/etc/postsrsd.secret
+
+# Local ports for TCP list.
+# These ports are used to bind the TCP list for postfix. If you change
+# these, you have to modify the postfix settings accordingly. The ports
+# are bound to the loopback interface, and should never be exposed on
+# the internet.
+#
+SRS_FORWARD_PORT=10001
+SRS_REVERSE_PORT=10002
+
+# Drop root privileges and run as another user after initialization.
+# This is highly recommended as postsrsd handles untrusted input.
+#
+RUN_AS=postsrsd
+
+# Jail daemon in chroot environment
+CHROOT=/var/lib/postsrsd
+

debian/control

@@ -12,13 +12,13 @@ Architecture: all
 Depends: ${python:Depends}, ${misc:Depends}
  , moulinette (>= 2.7.1), ssowat (>= 2.7.1)
  , python-psutil, python-requests, python-dnspython, python-openssl
- , python-apt, python-miniupnpc, python-dbus
+ , python-apt, python-miniupnpc, python-dbus, python-jinja2
  , glances
  , dnsutils, bind9utils, unzip, git, curl, cron, wget
  , ca-certificates, netcat-openbsd, iproute
  , mariadb-server, php-mysql | php-mysqlnd
  , slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd
- , postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils
+ , postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils, postsrsd
  , dovecot-ldap, dovecot-lmtpd, dovecot-managesieved
  , dovecot-antispam, fail2ban
  , nginx-extras (>=1.6.2), php-fpm, php-ldap, php-intl

locales/en.json

@@ -206,6 +206,49 @@
     "invalid_url_format": "Invalid URL format",
     "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it",
     "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it",
+    "log_corrupted_md_file": "The yaml metadata file associated with logs is corrupted : '{md_file}'",
+    "log_category_404": "The log category '{category}' does not exist",
+    "log_link_to_log": "Full log of this operation: '<a href=\"#/tools/logs/{name}\" style=\"text-decoration:underline\">{desc}</a>'",
+    "log_help_to_get_log": "To view the log of the operation '{desc}', use the command 'yunohost log display {name}'",
+    "log_link_to_failed_log": "The operation '{desc}' has failed ! To get help, please <a href=\"#/tools/logs/{name}\">provide the full log of this operation</a>",
+    "log_help_to_get_failed_log": "The operation '{desc}' has failed ! To get help, please share the full log of this operation using the command 'yunohost log display {name} --share'",
+    "log_category_404": "The log category '{category}' does not exist",
+    "log_does_exists": "There is not operation log with the name '{log}', use 'yunohost log list to see all available operation logs'",
+    "log_operation_unit_unclosed_properly": "Operation unit has not been closed properly",
+    "log_app_addaccess": "Add access to '{}'",
+    "log_app_removeaccess": "Remove access to '{}'",
+    "log_app_clearaccess": "Remove all access to '{}'",
+    "log_app_fetchlist": "Add an application list",
+    "log_app_removelist": "Remove an application list",
+    "log_app_change_url": "Change the url of '{}' application",
+    "log_app_install": "Install '{}' application",
+    "log_app_remove": "Remove '{}' application",
+    "log_app_upgrade": "Upgrade '{}' application",
+    "log_app_makedefault": "Make '{}' as default application",
+    "log_available_on_yunopaste": "This log is now available via {url}",
+    "log_backup_restore_system": "Restore system from a backup archive",
+    "log_backup_restore_app": "Restore '{}' from a backup archive",
+    "log_remove_on_failed_restore": "Remove '{}' after a failed restore from a backup archive",
+    "log_remove_on_failed_install": "Remove '{}' after a failed installation",
+    "log_domain_add": "Add '{}' domain into system configuration",
+    "log_domain_remove": "Remove '{}' domain from system configuration",
+    "log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'",
+    "log_dyndns_update": "Update the ip associated with your YunoHost subdomain '{}'",
+    "log_letsencrypt_cert_install": "Install Let's encrypt certificate on '{}' domain",
+    "log_selfsigned_cert_install": "Install self signed certificate on '{}' domain",
+    "log_letsencrypt_cert_renew": "Renew '{}' Let's encrypt certificate",
+    "log_service_enable": "Enable '{}' service",
+    "log_service_regen_conf": "Regenerate system configurations '{}'",
+    "log_user_create": "Add '{}' user",
+    "log_user_delete": "Delete '{}' user",
+    "log_user_update": "Update information of '{}' user",
+    "log_tools_maindomain": "Make '{}' as main domain",
+    "log_tools_migrations_migrate_forward": "Migrate forward",
+    "log_tools_migrations_migrate_backward": "Migrate backward",
+    "log_tools_postinstall": "Postinstall your YunoHost server",
+    "log_tools_upgrade": "Upgrade debian packages",
+    "log_tools_shutdown": "Shutdown your server",
+    "log_tools_reboot": "Reboot your server",
     "ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user",
     "ldap_initialized": "LDAP has been initialized",
     "license_undefined": "undefined",

src/yunohost/app.py

@@ -44,6 +44,7 @@ from moulinette.utils.filesystem import read_json
 
 from yunohost.service import service_log, _run_service_command
 from yunohost.utils import packages
+from yunohost.log import is_unit_operation, OperationLogger
 
 logger = getActionLogger('yunohost.app')
 
@@ -109,10 +110,13 @@ def app_fetchlist(url=None, name=None):
     # the fetch only this list
     if url is not None:
         if name:
+            operation_logger = OperationLogger('app_fetchlist')
+            operation_logger.start()
             _register_new_appslist(url, name)
             # Refresh the appslists dict
             appslists = _read_appslist_list()
             appslists_to_be_fetched = [name]
+            operation_logger.success()
         else:
             raise MoulinetteError(errno.EINVAL,
                                   m18n.n('custom_appslist_name_required'))
@@ -188,7 +192,8 @@ def app_fetchlist(url=None, name=None):
     _write_appslist_list(appslists)
 
 
-def app_removelist(name):
+@is_unit_operation()
+def app_removelist(operation_logger, name):
     """
     Remove list from the repositories
 
@@ -202,6 +207,8 @@ def app_removelist(name):
     if name not in appslists.keys():
         raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name))
 
+    operation_logger.start()
+
     # Remove json
     json_path = '%s/%s.json' % (REPO_PATH, name)
     if os.path.exists(json_path):
@@ -425,7 +432,8 @@ def app_map(app=None, raw=False, user=None):
     return result
 
 
-def app_change_url(auth, app, domain, path):
+@is_unit_operation()
+def app_change_url(operation_logger, auth, app, domain, path):
     """
     Modify the URL at which an application is installed.
 
@@ -482,6 +490,11 @@ def app_change_url(auth, app, domain, path):
     env_dict["YNH_APP_NEW_DOMAIN"] = domain
     env_dict["YNH_APP_NEW_PATH"] = path.rstrip("/")
 
+    if domain != old_domain:
+        operation_logger.related_to.append(('domain', old_domain))
+    operation_logger.extra.update({'env': env_dict})
+    operation_logger.start()
+
     if os.path.exists(os.path.join(APP_TMP_FOLDER, "scripts")):
         shutil.rmtree(os.path.join(APP_TMP_FOLDER, "scripts"))
 
@@ -499,16 +512,16 @@ def app_change_url(auth, app, domain, path):
     os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts")))
     os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url")))
 
-    # XXX journal
     if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'),
                  args=args_list, env=env_dict, user="root") != 0:
-        logger.error("Failed to change '%s' url." % app)
+        msg = "Failed to change '%s' url." % app
+        logger.error(msg)
+        operation_logger.error(msg)
 
         # restore values modified by app_checkurl
         # see begining of the function
         app_setting(app, "domain", value=old_domain)
         app_setting(app, "path", value=old_path)
-
         return
 
     # this should idealy be done in the change_url script but let's avoid common mistakes
@@ -546,7 +559,6 @@ def app_upgrade(auth, app=[], url=None, file=None):
     """
     from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
 
-
     # Retrieve interface
     is_api = msettings.get('interface') == 'api'
 
@@ -617,6 +629,11 @@ def app_upgrade(auth, app=[], url=None, file=None):
         env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
         env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
 
+        # Start register change on system
+        related_to = [('app', app_instance_name)]
+        operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict)
+        operation_logger.start()
+
         # Apply dirty patch to make php5 apps compatible with php7
         _patch_php5(extracted_app_folder)
 
@@ -624,7 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None):
         os.system('chown -hR admin: %s' % INSTALL_TMP)
         if hook_exec(extracted_app_folder + '/scripts/upgrade',
                      args=args_list, env=env_dict, user="root") != 0:
-            logger.error(m18n.n('app_upgrade_failed', app=app_instance_name))
+            msg = m18n.n('app_upgrade_failed', app=app_instance_name)
+            logger.error(msg)
+            operation_logger.error(msg)
         else:
             now = int(time.time())
             # TODO: Move install_time away from app_setting
@@ -654,7 +673,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
             logger.success(m18n.n('app_upgraded', app=app_instance_name))
 
             hook_callback('post_app_upgrade', args=args_list, env=env_dict)
-
+            operation_logger.success()
 
     if not upgraded_apps:
         raise MoulinetteError(errno.ENODATA, m18n.n('app_no_upgrade'))
@@ -668,7 +687,8 @@ def app_upgrade(auth, app=[], url=None, file=None):
         return {"log": service_log('yunohost-api', number="100").values()[0]}
 
 
-def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
+@is_unit_operation()
+def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on_failure=False):
     """
     Install apps
 
@@ -680,6 +700,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
 
     """
     from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
+    from yunohost.log import OperationLogger
+
 
     # Fetch or extract sources
     try:
@@ -737,6 +759,12 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
     env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
     env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
 
+    # Start register change on system
+    operation_logger.extra.update({'env':env_dict})
+    operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
+    operation_logger.related_to.append(("app", app_id))
+    operation_logger.start()
+
     # Create app directory
     app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
     if os.path.exists(app_setting_path):
@@ -772,13 +800,15 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
     try:
         install_retcode = hook_exec(
             os.path.join(extracted_app_folder, 'scripts/install'),
-            args=args_list, env=env_dict, user="root")
+            args=args_list, env=env_dict, user="root"
+        )
     except (KeyboardInterrupt, EOFError):
         install_retcode = -1
     except:
         logger.exception(m18n.n('unexpected_error'))
     finally:
         if install_retcode != 0:
+            error_msg = operation_logger.error(m18n.n('unexpected_error'))
             if not no_remove_on_failure:
                 # Setup environment for remove script
                 env_dict_remove = {}
@@ -787,12 +817,22 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
                 env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
 
                 # Execute remove script
+                operation_logger_remove = OperationLogger('remove_on_failed_install',
+                                                 [('app', app_instance_name)],
+                                                 env=env_dict_remove)
+                operation_logger_remove.start()
+
                 remove_retcode = hook_exec(
                     os.path.join(extracted_app_folder, 'scripts/remove'),
-                    args=[app_instance_name], env=env_dict_remove, user="root")
+                    args=[app_instance_name], env=env_dict_remove, user="root"
+                )
                 if remove_retcode != 0:
-                    logger.warning(m18n.n('app_not_properly_removed',
+                    msg = m18n.n('app_not_properly_removed',
-                                          app=app_instance_name))
+                                 app=app_instance_name)
+                    logger.warning(msg)
+                    operation_logger_remove.error(msg)
+                else:
+                    operation_logger_remove.success()
 
             # Clean tmp folders
             shutil.rmtree(app_setting_path)
@@ -801,9 +841,10 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
             app_ssowatconf(auth)
 
             if install_retcode == -1:
-                raise MoulinetteError(errno.EINTR,
+                msg = m18n.n('operation_interrupted') + " " + error_msg
-                                      m18n.g('operation_interrupted'))
+                raise MoulinetteError(errno.EINTR, msg)
-            raise MoulinetteError(errno.EIO, m18n.n('installation_failed'))
+            msg = error_msg
+            raise MoulinetteError(errno.EIO, msg)
 
     # Clean hooks and add new ones
     hook_remove(app_instance_name)
@@ -828,7 +869,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
     hook_callback('post_app_install', args=args_list, env=env_dict)
 
 
-def app_remove(auth, app):
+@is_unit_operation()
+def app_remove(operation_logger, auth, app):
     """
     Remove app
 
@@ -837,11 +879,12 @@ def app_remove(auth, app):
 
     """
     from yunohost.hook import hook_exec, hook_remove, hook_callback
-
     if not _is_installed(app):
         raise MoulinetteError(errno.EINVAL,
                               m18n.n('app_not_installed', app=app))
 
+    operation_logger.start()
+
     app_setting_path = APPS_SETTING_PATH + app
 
     # TODO: display fail messages from script
@@ -865,6 +908,8 @@ def app_remove(auth, app):
     env_dict["YNH_APP_ID"] = app_id
     env_dict["YNH_APP_INSTANCE_NAME"] = app
     env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
+    operation_logger.extra.update({'env': env_dict})
+    operation_logger.flush()
 
     if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list,
                  env=env_dict, user="root") == 0:
@@ -901,6 +946,8 @@ def app_addaccess(auth, apps, users=[]):
         apps = [apps, ]
 
     for app in apps:
+
+
         app_settings = _get_app_settings(app)
         if not app_settings:
             continue
@@ -910,6 +957,12 @@ def app_addaccess(auth, apps, users=[]):
             app_settings['mode'] = 'private'
 
         if app_settings['mode'] == 'private':
+
+            # Start register change on system
+            related_to = [('app', app)]
+            operation_logger= OperationLogger('app_addaccess', related_to)
+            operation_logger.start()
+
             allowed_users = set()
             if 'allowed_users' in app_settings:
                 allowed_users = set(app_settings['allowed_users'].split(','))
@@ -922,11 +975,15 @@ def app_addaccess(auth, apps, users=[]):
                         logger.warning(m18n.n('user_unknown', user=allowed_user))
                         continue
                     allowed_users.add(allowed_user)
+                    operation_logger.related_to.append(('user', allowed_user))
 
+            operation_logger.flush()
             new_users = ','.join(allowed_users)
             app_setting(app, 'allowed_users', new_users)
             hook_callback('post_app_addaccess', args=[app, new_users])
 
+            operation_logger.success()
+
             result[app] = allowed_users
 
     app_ssowatconf(auth)
@@ -963,6 +1020,12 @@ def app_removeaccess(auth, apps, users=[]):
         allowed_users = set()
 
         if app_settings.get('skipped_uris', '') != '/':
+
+            # Start register change on system
+            related_to = [('app', app)]
+            operation_logger= OperationLogger('app_removeaccess', related_to)
+            operation_logger.start()
+
             if remove_all:
                 pass
             elif 'allowed_users' in app_settings:
@@ -972,14 +1035,18 @@ def app_removeaccess(auth, apps, users=[]):
             else:
                 for allowed_user in user_list(auth)['users'].keys():
                     if allowed_user not in users:
-                        allowed_users.add(allowed_user)
+                        allowed_users.append(allowed_user)
 
+            operation_logger.related_to += [ ('user', x) for x in allowed_users ]
+            operation_logger.flush()
             new_users = ','.join(allowed_users)
             app_setting(app, 'allowed_users', new_users)
             hook_callback('post_app_removeaccess', args=[app, new_users])
 
             result[app] = allowed_users
 
+            operation_logger.success()
+
     app_ssowatconf(auth)
 
     return {'allowed_users': result}
@@ -1003,6 +1070,11 @@ def app_clearaccess(auth, apps):
         if not app_settings:
             continue
 
+        # Start register change on system
+        related_to = [('app', app)]
+        operation_logger= OperationLogger('app_clearaccess', related_to)
+        operation_logger.start()
+
         if 'mode' in app_settings:
             app_setting(app, 'mode', delete=True)
 
@@ -1011,6 +1083,8 @@ def app_clearaccess(auth, apps):
 
         hook_callback('post_app_clearaccess', args=[app])
 
+        operation_logger.success()
+
     app_ssowatconf(auth)
 
 
@@ -1037,7 +1111,8 @@ def app_debug(app):
     }
 
 
-def app_makedefault(auth, app, domain=None):
+@is_unit_operation()
+def app_makedefault(operation_logger, auth, app, domain=None):
     """
     Redirect domain root to an app
 
@@ -1054,9 +1129,11 @@ def app_makedefault(auth, app, domain=None):
 
     if domain is None:
         domain = app_domain
+        operation_logger.related_to.append(('domain',domain))
     elif domain not in domain_list(auth)['domains']:
         raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
 
+    operation_logger.start()
     if '/' in app_map(raw=True)[domain]:
         raise MoulinetteError(errno.EEXIST,
                               m18n.n('app_make_default_location_already_used',
@@ -2204,7 +2281,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
                     app_label=app_label,
                 ))
 
-            raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', "\n".join(apps=apps)))
+            raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', apps="\n".join(apps)))
 
         # (We save this normalized path so that the install script have a
         # standard path format to deal with no matter what the user inputted)

src/yunohost/backup.py

@@ -51,6 +51,7 @@ from yunohost.hook import (
 from yunohost.monitor import binary_to_human
 from yunohost.tools import tools_postinstall
 from yunohost.service import service_regen_conf
+from yunohost.log import OperationLogger
 
 BACKUP_PATH = '/home/yunohost.backup'
 ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
@@ -1172,9 +1173,15 @@ class RestoreManager():
         if system_targets == []:
             return
 
+        # Start register change on system
+        operation_logger = OperationLogger('backup_restore_system')
+        operation_logger.start()
+
         logger.debug(m18n.n('restore_running_hooks'))
 
         env_dict = self._get_env_var()
+        operation_logger.extra['env'] = env_dict
+        operation_logger.flush()
         ret = hook_callback('restore',
                             system_targets,
                             args=[self.work_dir],
@@ -1184,9 +1191,16 @@ class RestoreManager():
         for part in ret['succeed'].keys():
             self.targets.set_result("system", part, "Success")
 
+        error_part = []
         for part in ret['failed'].keys():
             logger.error(m18n.n('restore_system_part_failed', part=part))
             self.targets.set_result("system", part, "Error")
+            error_part.append(part)
+
+        if ret['failed']:
+            operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part)))
+        else:
+            operation_logger.success()
 
         service_regen_conf()
 
@@ -1234,6 +1248,11 @@ class RestoreManager():
                 else:
                     shutil.copy2(s, d)
 
+        # Start register change on system
+        related_to = [('app', app_instance_name)]
+        operation_logger = OperationLogger('backup_restore_app', related_to)
+        operation_logger.start()
+
         # Check if the app is not already installed
         if _is_installed(app_instance_name):
             logger.error(m18n.n('restore_already_installed_app',
@@ -1283,6 +1302,9 @@ class RestoreManager():
             # Prepare env. var. to pass to script
             env_dict = self._get_env_var(app_instance_name)
 
+            operation_logger.extra['env'] = env_dict
+            operation_logger.flush()
+
             # Execute app restore script
             hook_exec(restore_script,
                       args=[app_backup_in_archive, app_instance_name],
@@ -1291,8 +1313,10 @@ class RestoreManager():
                       env=env_dict,
                       user="root")
         except:
-            logger.exception(m18n.n('restore_app_failed',
+            msg = m18n.n('restore_app_failed',app=app_instance_name)
-                                    app=app_instance_name))
+            logger.exception(msg)
+            operation_logger.error(msg)
+
             self.targets.set_result("apps", app_instance_name, "Error")
 
             remove_script = os.path.join(app_scripts_in_archive, 'remove')
@@ -1304,12 +1328,20 @@ class RestoreManager():
             env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
             env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
 
+            operation_logger = OperationLogger('remove_on_failed_restore',
+                               [('app', app_instance_name)],
+                               env=env_dict_remove)
+            operation_logger.start()
+
             # Execute remove script
             # TODO: call app_remove instead
             if hook_exec(remove_script, args=[app_instance_name],
                          env=env_dict_remove, user="root") != 0:
-                logger.warning(m18n.n('app_not_properly_removed',
+                msg = m18n.n('app_not_properly_removed', app=app_instance_name)
-                                      app=app_instance_name))
+                logger.warning(msg)
+                operation_logger.error(msg)
+            else:
+                operation_logger.success()
 
             # Cleaning app directory
             shutil.rmtree(app_settings_new_path, ignore_errors=True)
@@ -1317,6 +1349,7 @@ class RestoreManager():
             # TODO Cleaning app hooks
         else:
             self.targets.set_result("apps", app_instance_name, "Success")
+            operation_logger.success()
         finally:
             # Cleaning temporary scripts directory
             shutil.rmtree(tmp_folder_for_app_restore, ignore_errors=True)

src/yunohost/certificate.py

@@ -45,7 +45,7 @@ from yunohost.utils.network import get_public_ip
 from moulinette import m18n
 from yunohost.app import app_ssowatconf
 from yunohost.service import _run_service_command, service_regen_conf
-
+from yunohost.log import OperationLogger
 
 logger = getActionLogger('yunohost.certmanager')
 
@@ -160,6 +160,9 @@ def _certificate_install_selfsigned(domain_list, force=False):
 
     for domain in domain_list:
 
+        operation_logger = OperationLogger('selfsigned_cert_install', [('domain', domain)],
+                           args={'force': force})
+
         # Paths of files and folder we'll need
         date_tag = datetime.now().strftime("%Y%m%d.%H%M%S")
         new_cert_folder = "%s/%s-history/%s-selfsigned" % (
@@ -182,6 +185,8 @@ def _certificate_install_selfsigned(domain_list, force=False):
                 raise MoulinetteError(errno.EINVAL, m18n.n(
                     'certmanager_attempt_to_replace_valid_cert', domain=domain))
 
+        operation_logger.start()
+
         # Create output folder for new certificate stuff
         os.makedirs(new_cert_folder)
 
@@ -238,9 +243,11 @@ def _certificate_install_selfsigned(domain_list, force=False):
         if status and status["CA_type"]["code"] == "self-signed" and status["validity"] > 3648:
             logger.success(
                 m18n.n("certmanager_cert_install_success_selfsigned", domain=domain))
+            operation_logger.success()
         else:
-            logger.error(
+            msg = "Installation of self-signed certificate installation for %s failed !" % (domain)
-                "Installation of self-signed certificate installation for %s failed !", domain)
+            logger.error(msg)
+            operation_logger.error(msg)
 
 
 def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=False, staging=False):
@@ -281,6 +288,9 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
     # Actual install steps
     for domain in domain_list:
 
+        operation_logger = OperationLogger('letsencrypt_cert_install', [('domain', domain)],
+                           args={'force': force, 'no_checks': no_checks,
+                                 'staging': staging})
         logger.info(
             "Now attempting install of certificate for domain %s!", domain)
 
@@ -288,6 +298,8 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
             if not no_checks:
                 _check_domain_is_ready_for_ACME(domain)
 
+            operation_logger.start()
+
             _configure_for_acme_challenge(auth, domain)
             _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
             _install_cron()
@@ -295,10 +307,12 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
             logger.success(
                 m18n.n("certmanager_cert_install_success", domain=domain))
 
+            operation_logger.success()
         except Exception as e:
             _display_debug_information(domain)
-            logger.error("Certificate installation for %s failed !\nException: %s", domain, e)
+            msg = "Certificate installation for %s failed !\nException: %s" % (domain, e)
-
+            logger.error(msg)
+            operation_logger.error(msg)
 
 def certificate_renew(auth, domain_list, force=False, no_checks=False, email=False, staging=False):
     """
@@ -376,6 +390,11 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
 
     # Actual renew steps
     for domain in domain_list:
+
+        operation_logger = OperationLogger('letsencrypt_cert_renew', [('domain', domain)],
+                           args={'force': force, 'no_checks': no_checks,
+                                 'staging': staging, 'email': email})
+
         logger.info(
             "Now attempting renewing of certificate for domain %s !", domain)
 
@@ -383,17 +402,23 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
             if not no_checks:
                 _check_domain_is_ready_for_ACME(domain)
 
+            operation_logger.start()
+
             _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
 
             logger.success(
                 m18n.n("certmanager_cert_renew_success", domain=domain))
 
+            operation_logger.success()
+
         except Exception as e:
             import traceback
             from StringIO import StringIO
             stack = StringIO()
             traceback.print_exc(file=stack)
-            logger.error("Certificate renewing for %s failed !", domain)
+            msg = "Certificate renewing for %s failed !" % (domain)
+            logger.error(msg)
+            operation_logger.error(msg)
             logger.error(stack.getvalue())
             logger.error(str(e))
 
@@ -401,7 +426,6 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
                 logger.error("Sending email with details to root ...")
                 _email_renewing_failed(domain, e, stack.getvalue())
 
-
 ###############################################################################
 #   Back-end stuff                                                            #
 ###############################################################################

src/yunohost/domain.py

@@ -37,6 +37,7 @@ import yunohost.certificate
 
 from yunohost.service import service_regen_conf
 from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation
 
 logger = getActionLogger('yunohost.domain')
 
@@ -61,7 +62,8 @@ def domain_list(auth):
     return {'domains': result_list}
 
 
-def domain_add(auth, domain, dyndns=False):
+@is_unit_operation()
+def domain_add(operation_logger, auth, domain, dyndns=False):
     """
     Create a custom domain
 
@@ -78,6 +80,8 @@ def domain_add(auth, domain, dyndns=False):
     except MoulinetteError:
         raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))
 
+    operation_logger.start()
+
     # DynDNS domain
     if dyndns:
 
@@ -110,23 +114,27 @@ def domain_add(auth, domain, dyndns=False):
 
         # Don't regen these conf if we're still in postinstall
         if os.path.exists('/etc/yunohost/installed'):
-            service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
+            service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
             app_ssowatconf(auth)
 
-    except:
+    except Exception, e:
+        from sys import exc_info;
+        t, v, tb = exc_info()
+
         # Force domain removal silently
         try:
             domain_remove(auth, domain, True)
         except:
             pass
-        raise
+        raise t, v, tb
 
     hook_callback('post_domain_add', args=[domain])
 
     logger.success(m18n.n('domain_created'))
 
 
-def domain_remove(auth, domain, force=False):
+@is_unit_operation()
+def domain_remove(operation_logger, auth, domain, force=False):
     """
     Delete domains
 
@@ -157,12 +165,13 @@ def domain_remove(auth, domain, force=False):
                     raise MoulinetteError(errno.EPERM,
                                           m18n.n('domain_uninstall_app_first'))
 
+    operation_logger.start()
     if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
         os.system('rm -rf /etc/yunohost/certs/%s' % domain)
     else:
         raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))
 
-    service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
+    service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
     app_ssowatconf(auth)
 
     hook_callback('post_domain_remove', args=[domain])

src/yunohost/dyndns.py

@@ -40,6 +40,7 @@ from moulinette.utils.network import download_json
 
 from yunohost.domain import _get_maindomain, _build_dns_conf
 from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation
 
 logger = getActionLogger('yunohost.dyndns')
 
@@ -112,7 +113,8 @@ def _dyndns_available(provider, domain):
     return r == u"Domain %s is available" % domain
 
 
-def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None):
+@is_unit_operation()
+def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", domain=None, key=None):
     """
     Subscribe to a DynDNS service
 
@@ -124,6 +126,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
     """
     if domain is None:
         domain = _get_maindomain()
+        operation_logger.related_to.append(('domain', domain))
 
     # Verify if domain is provided by subscribe_host
     if not _dyndns_provides(subscribe_host, domain):
@@ -136,6 +139,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
         raise MoulinetteError(errno.ENOENT,
                               m18n.n('dyndns_unavailable', domain=domain))
 
+    operation_logger.start()
+
     if key is None:
         if len(glob.glob('/etc/yunohost/dyndns/*.key')) == 0:
             if not os.path.exists('/etc/yunohost/dyndns'):
@@ -170,7 +175,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
     dyndns_installcron()
 
 
-def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
+@is_unit_operation()
+def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, key=None,
                   ipv4=None, ipv6=None):
     """
     Update IP on DynDNS platform
@@ -217,13 +223,17 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
     if domain is None:
         (domain, key) = _guess_current_dyndns_domain(dyn_host)
     # If key is not given, pick the first file we find with the domain given
-    elif key is None:
+    else:
-        keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
+        if key is None:
+            keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
 
-        if not keys:
+            if not keys:
-            raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
+                raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
 
-        key = keys[0]
+            key = keys[0]
+
+    operation_logger.related_to.append(('domain', domain))
+    operation_logger.start()
 
     # This mean that hmac-md5 is used
     # (Re?)Trigger the migration to sha256 and return immediately.

src/yunohost/log.py

@@ -0,0 +1,462 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YunoHost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+""" yunohost_log.py
+
+    Manage debug logs
+"""
+
+import os
+import yaml
+import errno
+import collections
+
+from datetime import datetime
+from logging import FileHandler, getLogger, Formatter
+from sys import exc_info
+
+from moulinette import m18n, msettings
+from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file
+
+CATEGORIES_PATH = '/var/log/yunohost/categories/'
+OPERATIONS_PATH = '/var/log/yunohost/categories/operation/'
+CATEGORIES = ['operation', 'history', 'package', 'system', 'access', 'service',
+              'app']
+METADATA_FILE_EXT = '.yml'
+LOG_FILE_EXT = '.log'
+RELATED_CATEGORIES = ['app', 'domain', 'service', 'user']
+
+logger = getActionLogger('yunohost.log')
+
+
+def log_list(category=[], limit=None):
+    """
+    List available logs
+
+    Keyword argument:
+        limit -- Maximum number of logs
+    """
+
+    categories = category
+    is_api = msettings.get('interface') == 'api'
+
+    # In cli we just display `operation` logs by default
+    if not categories:
+        categories = ["operation"] if not is_api else CATEGORIES
+
+    result = collections.OrderedDict()
+    for category in categories:
+        result[category] = []
+
+        category_path = os.path.join(CATEGORIES_PATH, category)
+        if not os.path.exists(category_path):
+            logger.debug(m18n.n('log_category_404', category=category))
+
+            continue
+
+        logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
+                      os.listdir(category_path))
+        logs = reversed(sorted(logs))
+
+        if limit is not None:
+            logs = logs[:limit]
+
+        for log in logs:
+
+            base_filename = log[:-len(METADATA_FILE_EXT)]
+            md_filename = log
+            md_path = os.path.join(category_path, md_filename)
+
+            log = base_filename.split("-")
+
+            entry = {
+                "name": base_filename,
+                "path": md_path,
+            }
+            entry["description"] = _get_description_from_name(base_filename)
+            try:
+                log_datetime = datetime.strptime(" ".join(log[:2]),
+                                                 "%Y%m%d %H%M%S")
+            except ValueError:
+                pass
+            else:
+                entry["started_at"] = log_datetime
+
+            result[category].append(entry)
+
+    # Reverse the order of log when in cli, more comfortable to read (avoid
+    # unecessary scrolling)
+    if not is_api:
+        for category in result:
+            result[category] = list(reversed(result[category]))
+
+    return result
+
+
+def log_display(path, number=50, share=False):
+    """
+    Display a log file enriched with metadata if any.
+
+    If the file_name is not an absolute path, it will try to search the file in
+    the unit operations log path (see OPERATIONS_PATH).
+
+    Argument:
+        file_name
+        number
+        share
+    """
+
+    # Normalize log/metadata paths and filenames
+    abs_path = path
+    log_path = None
+    if not path.startswith('/'):
+        for category in CATEGORIES:
+            abs_path = os.path.join(CATEGORIES_PATH, category, path)
+            if os.path.exists(abs_path) or os.path.exists(abs_path + METADATA_FILE_EXT):
+                break
+
+    if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT):
+        log_path = abs_path
+
+    if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT):
+        base_path = ''.join(os.path.splitext(abs_path)[:-1])
+    else:
+        base_path = abs_path
+    base_filename = os.path.basename(base_path)
+    md_path = base_path + METADATA_FILE_EXT
+    if log_path is None:
+        log_path = base_path + LOG_FILE_EXT
+
+    if not os.path.exists(md_path) and not os.path.exists(log_path):
+        raise MoulinetteError(errno.EINVAL,
+                              m18n.n('log_does_exists', log=path))
+
+    infos = {}
+
+    # If it's a unit operation, display the name and the description
+    if base_path.startswith(CATEGORIES_PATH):
+        infos["description"] = _get_description_from_name(base_filename)
+        infos['name'] = base_filename
+
+    if share:
+        from yunohost.utils.yunopaste import yunopaste
+        content = ""
+        if os.path.exists(md_path):
+            content += read_file(md_path)
+            content += "\n============\n\n"
+        if os.path.exists(log_path):
+            content += read_file(log_path)
+
+        url = yunopaste(content)
+
+        logger.info(m18n.n("log_available_on_yunopaste", url=url))
+        if msettings.get('interface') == 'api':
+            return {"url": url}
+        else:
+            return
+
+    # Display metadata if exist
+    if os.path.exists(md_path):
+        with open(md_path, "r") as md_file:
+            try:
+                metadata = yaml.safe_load(md_file)
+                infos['metadata_path'] = md_path
+                infos['metadata'] = metadata
+                if 'log_path' in metadata:
+                    log_path = metadata['log_path']
+            except yaml.YAMLError:
+                error = m18n.n('log_corrupted_md_file', file=md_path)
+                if os.path.exists(log_path):
+                    logger.warning(error)
+                else:
+                    raise MoulinetteError(errno.EINVAL, error)
+
+    # Display logs if exist
+    if os.path.exists(log_path):
+        from yunohost.service import _tail
+        logs = _tail(log_path, int(number))
+        infos['log_path'] = log_path
+        infos['logs'] = logs
+
+    return infos
+
+
+def is_unit_operation(entities=['app', 'domain', 'service', 'user'],
+                      exclude=['auth', 'password'], operation_key=None):
+    """
+    Configure quickly a unit operation
+
+    This decorator help you to configure the record of a unit operations.
+
+    Argument:
+    entities   A list of entity types related to the unit operation. The entity
+    type is searched inside argument's names of the decorated function. If
+    something match, the argument value is added as related entity. If the
+    argument name is different you can specify it with a tuple
+    (argname, entity_type) instead of just put the entity type.
+
+    exclude    Remove some arguments from the context. By default, arguments
+    called 'password' and 'auth' are removed. If an argument is an object, you
+    need to exclude it or create manually the unit operation without this
+    decorator.
+
+    operation_key   A key to describe the unit operation log used to create the
+    filename and search a translation. Please ensure that this key prefixed by
+    'log_' is present in locales/en.json otherwise it won't be translatable.
+
+    """
+    def decorate(func):
+        def func_wrapper(*args, **kwargs):
+            op_key = operation_key
+            if op_key is None:
+                op_key = func.__name__
+
+            # If the function is called directly from an other part of the code
+            # and not by the moulinette framework, we need to complete kwargs
+            # dictionnary with the args list.
+            # Indeed, we use convention naming in this decorator and we need to
+            # know name of each args (so we need to use kwargs instead of args)
+            if len(args) > 0:
+                from inspect import getargspec
+                keys = getargspec(func).args
+                if 'operation_logger' in keys:
+                    keys.remove('operation_logger')
+                for k, arg in enumerate(args):
+                    kwargs[keys[k]] = arg
+                args = ()
+
+            # Search related entity in arguments of the decorated function
+            related_to = []
+            for entity in entities:
+                if isinstance(entity, tuple):
+                    entity_type = entity[1]
+                    entity = entity[0]
+                else:
+                    entity_type = entity
+
+                if entity in kwargs and kwargs[entity] is not None:
+                    if isinstance(kwargs[entity], basestring):
+                        related_to.append((entity_type, kwargs[entity]))
+                    else:
+                        for x in kwargs[entity]:
+                            related_to.append((entity_type, x))
+
+            context = kwargs.copy()
+
+            # Exclude unappropriate data from the context
+            for field in exclude:
+                if field in context:
+                    context.pop(field, None)
+            operation_logger = OperationLogger(op_key, related_to, args=context)
+
+            try:
+                # Start the actual function, and give the unit operation
+                # in argument to let the developper start the record itself
+                args = (operation_logger,) + args
+                result = func(*args, **kwargs)
+            except Exception as e:
+                operation_logger.error(e)
+                raise
+            else:
+                operation_logger.success()
+            return result
+        return func_wrapper
+    return decorate
+
+
+class OperationLogger(object):
+    """
+    Instances of this class represents unit operation done on the ynh instance.
+
+    Each time an action of the yunohost cli/api change the system, one or
+    several unit operations should be registered.
+
+    This class record logs and metadata like context or start time/end time.
+    """
+
+    def __init__(self, operation, related_to=None, **kwargs):
+        # TODO add a way to not save password on app installation
+        self.operation = operation
+        self.related_to = related_to
+        self.extra = kwargs
+        self.started_at = None
+        self.ended_at = None
+        self.logger = None
+        self._name = None
+
+        self.path = OPERATIONS_PATH
+
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
+
+    def start(self):
+        """
+        Start to record logs that change the system
+        Until this start method is run, no unit operation will be registered.
+        """
+
+        if self.started_at is None:
+            self.started_at = datetime.now()
+            self.flush()
+            self._register_log()
+
+    def _register_log(self):
+        """
+        Register log with a handler connected on log system
+        """
+
+        # TODO add a way to not save password on app installation
+        filename = os.path.join(self.path, self.name + LOG_FILE_EXT)
+        self.file_handler = FileHandler(filename)
+        self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
+
+        # Listen to the root logger
+        self.logger = getLogger('yunohost')
+        self.logger.addHandler(self.file_handler)
+
+    def flush(self):
+        """
+        Write or rewrite the metadata file with all metadata known
+        """
+
+        filename = os.path.join(self.path, self.name + METADATA_FILE_EXT)
+        with open(filename, 'w') as outfile:
+            yaml.safe_dump(self.metadata, outfile, default_flow_style=False)
+
+    @property
+    def name(self):
+        """
+        Name of the operation
+        This name is used as filename, so don't use space
+        """
+        if self._name is not None:
+            return self._name
+
+        name = [self.started_at.strftime("%Y%m%d-%H%M%S")]
+        name += [self.operation]
+
+        if hasattr(self, "name_parameter_override"):
+            # This is for special cases where the operation is not really
+            # unitary. For instance, the regen conf cannot be logged "per
+            # service" because of the way it's built
+            name.append(self.name_parameter_override)
+        elif self.related_to:
+            # We use the name of the first related thing
+            name.append(self.related_to[0][1])
+
+        self._name = '-'.join(name)
+        return self._name
+
+    @property
+    def metadata(self):
+        """
+        Dictionnary of all metadata collected
+        """
+
+        data = {
+            'started_at': self.started_at,
+            'operation': self.operation,
+        }
+        if self.related_to is not None:
+            data['related_to'] = self.related_to
+        if self.ended_at is not None:
+            data['ended_at'] = self.ended_at
+            data['success'] = self._success
+            if self.error is not None:
+                data['error'] = self._error
+        # TODO: detect if 'extra' erase some key of 'data'
+        data.update(self.extra)
+        return data
+
+    def success(self):
+        """
+        Declare the success end of the unit operation
+        """
+        self.close()
+
+    def error(self, error):
+        """
+        Declare the failure of the unit operation
+        """
+        return self.close(error)
+
+    def close(self, error=None):
+        """
+        Close properly the unit operation
+        """
+        if self.ended_at is not None or self.started_at is None:
+            return
+        if error is not None and not isinstance(error, basestring):
+            error = str(error)
+        self.ended_at = datetime.now()
+        self._error = error
+        self._success = error is None
+        if self.logger is not None:
+            self.logger.removeHandler(self.file_handler)
+
+        is_api = msettings.get('interface') == 'api'
+        desc = _get_description_from_name(self.name)
+        if error is None:
+            if is_api:
+                msg = m18n.n('log_link_to_log', name=self.name, desc=desc)
+            else:
+                msg = m18n.n('log_help_to_get_log', name=self.name, desc=desc)
+            logger.debug(msg)
+        else:
+            if is_api:
+                msg = "<strong>" + m18n.n('log_link_to_failed_log',
+                                    name=self.name, desc=desc) + "</strong>"
+            else:
+                msg = m18n.n('log_help_to_get_failed_log', name=self.name,
+                             desc=desc)
+            logger.info(msg)
+        self.flush()
+        return msg
+
+    def __del__(self):
+        """
+        Try to close the unit operation, if it's missing.
+        The missing of the message below could help to see an electrical
+        shortage.
+        """
+        self.error(m18n.n('log_operation_unit_unclosed_properly'))
+
+
+def _get_description_from_name(name):
+    """
+    Return the translated description from the filename
+    """
+
+    parts = name.split("-", 3)
+    try:
+        try:
+            datetime.strptime(" ".join(parts[:2]), "%Y%m%d %H%M%S")
+        except ValueError:
+            key = "log_" + parts[0]
+            args = parts[1:]
+        else:
+            key = "log_" + parts[2]
+            args = parts[3:]
+        return m18n.n(key, *args)
+    except IndexError:
+        return name

src/yunohost/service.py

@@ -40,6 +40,7 @@ from moulinette.core import MoulinetteError
 from moulinette.utils import log, filesystem
 
 from yunohost.hook import hook_callback
+from yunohost.log import is_unit_operation
 
 BASE_CONF_PATH = '/home/yunohost.conf'
 BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
@@ -150,8 +151,8 @@ def service_stop(names):
                                              logs=_get_journalctl_logs(name)))
             logger.debug(m18n.n('service_already_stopped', service=name))
 
-
+@is_unit_operation()
-def service_enable(names):
+def service_enable(operation_logger, names):
     """
     Enable one or more services
 
@@ -159,6 +160,7 @@ def service_enable(names):
         names -- Services name to enable
 
     """
+    operation_logger.start()
     if isinstance(names, str):
         names = [names]
     for name in names:
@@ -343,7 +345,8 @@ def service_log(name, number=50):
     return result
 
 
-def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
+@is_unit_operation([('names', 'service')])
+def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
                        list_pending=False):
     """
     Regenerate the configuration file(s) for a service
@@ -376,6 +379,14 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
 
         return pending_conf
 
+    if not dry_run:
+        operation_logger.related_to = [('service', x) for x in names]
+        if not names:
+            operation_logger.name_parameter_override = 'all'
+        elif len(names) != 1:
+            operation_logger.name_parameter_override = str(len(operation_logger.related_to))+'_services'
+        operation_logger.start()
+
     # Clean pending conf directory
     if os.path.isdir(PENDING_CONF_DIR):
         if not names:
@@ -414,8 +425,13 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
     # Set the processing method
     _regen = _process_regen_conf if not dry_run else lambda *a, **k: True
 
+    operation_logger.related_to = []
+
     # Iterate over services and process pending conf
     for service, conf_files in _get_pending_conf(names).items():
+        if not dry_run:
+            operation_logger.related_to.append(('service', service))
+
         logger.debug(m18n.n(
             'service_regenconf_pending_applying' if not dry_run else
             'service_regenconf_dry_pending_applying',
@@ -564,6 +580,8 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
 
     hook_callback('conf_regen', names, pre_callback=_pre_call)
 
+    operation_logger.success()
+
     return result
 
 
@@ -691,13 +709,21 @@ def _tail(file, n):
     value is a tuple in the form ``(lines, has_more)`` where `has_more` is
     an indicator that is `True` if there are more lines in the file.
 
+    This function works even with splitted logs (gz compression, log rotate...)
     """
     avg_line_length = 74
     to_read = n
 
     try:
-        with open(file, 'r') as f:
+        if file.endswith(".gz"):
-            while 1:
+            import gzip
+            f = gzip.open(file)
+            lines = f.read().splitlines()
+        else:
+            f = open(file)
+            pos = 1
+            lines = []
+            while len(lines) < to_read and pos > 0:
                 try:
                     f.seek(-(avg_line_length * to_read), 2)
                 except IOError:
@@ -708,15 +734,48 @@ def _tail(file, n):
                 pos = f.tell()
                 lines = f.read().splitlines()
 
-                if len(lines) >= to_read or pos == 0:
+                if len(lines) >= to_read:
                     return lines[-to_read:]
 
                 avg_line_length *= 1.3
+        f.close()
 
     except IOError as e:
         logger.warning("Error while tailing file '%s': %s", file, e, exc_info=1)
         return []
 
+    if len(lines) < to_read:
+        previous_log_file = _find_previous_log_file(file)
+        if previous_log_file is not None:
+            lines = _tail(previous_log_file, to_read - len(lines)) + lines
+
+    return lines
+
+
+def _find_previous_log_file(file):
+    """
+    Find the previous log file
+    """
+    import re
+
+    splitext = os.path.splitext(file)
+    if splitext[1] == '.gz':
+        file = splitext[0]
+    splitext = os.path.splitext(file)
+    ext = splitext[1]
+    i = re.findall(r'\.(\d+)', ext)
+    i = int(i[0]) + 1 if len(i) > 0 else 1
+
+    previous_file = file if i == 1 else splitext[0]
+    previous_file = previous_file + '.%d' % (i)
+    if os.path.exists(previous_file):
+        return previous_file
+
+    previous_file = previous_file + ".gz"
+    if os.path.exists(previous_file):
+        return previous_file
+
+    return None
 
 def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
     """Compare two files and return the differences

src/yunohost/tools.py

@@ -52,6 +52,7 @@ from yunohost.service import service_status, service_regen_conf, service_log, se
 from yunohost.monitor import monitor_disk, monitor_system
 from yunohost.utils.packages import ynh_packages_version
 from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation, OperationLogger
 
 # FIXME this is a duplicate from apps.py
 APPS_SETTING_PATH = '/etc/yunohost/apps/'
@@ -138,7 +139,8 @@ def tools_adminpw(auth, new_password):
         logger.success(m18n.n('admin_password_changed'))
 
 
-def tools_maindomain(auth, new_domain=None):
+@is_unit_operation()
+def tools_maindomain(operation_logger, auth, new_domain=None):
     """
     Check the current main domain, or change it
 
@@ -155,6 +157,9 @@ def tools_maindomain(auth, new_domain=None):
     if new_domain not in domain_list(auth)['domains']:
         raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
 
+    operation_logger.related_to.append(('domain', new_domain))
+    operation_logger.start()
+
     # Apply changes to ssl certs
     ssl_key = "/etc/ssl/private/yunohost_key.pem"
     ssl_crt = "/etc/ssl/private/yunohost_crt.pem"
@@ -244,7 +249,8 @@ def _is_inside_container():
     return out.split()[0] in container
 
 
-def tools_postinstall(domain, password, ignore_dyndns=False):
+@is_unit_operation()
+def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False):
     """
     YunoHost post-install
 
@@ -293,6 +299,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
     else:
         dyndns = False
 
+    operation_logger.start()
     logger.info(m18n.n('yunohost_installing'))
 
     service_regen_conf(['nslcd', 'nsswitch'], force=True)
@@ -468,7 +475,8 @@ def tools_update(ignore_apps=False, ignore_packages=False):
     return {'packages': packages, 'apps': apps}
 
 
-def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
+@is_unit_operation()
+def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False):
     """
     Update apps & package cache, then display changelog
 
@@ -509,6 +517,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
         if cache.get_changes():
             logger.info(m18n.n('upgrading_packages'))
 
+            operation_logger.start()
             try:
                 # Apply APT changes
                 # TODO: Logs output for the API
@@ -518,11 +527,14 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
                 failure = True
                 logger.warning('unable to upgrade packages: %s' % str(e))
                 logger.error(m18n.n('packages_upgrade_failed'))
+                operation_logger.error(m18n.n('packages_upgrade_failed'))
             else:
                 logger.info(m18n.n('done'))
+                operation_logger.success()
         else:
             logger.info(m18n.n('packages_no_upgrade'))
 
+
     if not ignore_apps:
         try:
             app_upgrade(auth)
@@ -703,7 +715,8 @@ def tools_port_available(port):
         return False
 
 
-def tools_shutdown(force=False):
+@is_unit_operation()
+def tools_shutdown(operation_logger, force=False):
     shutdown = force
     if not shutdown:
         try:
@@ -716,11 +729,13 @@ def tools_shutdown(force=False):
                 shutdown = True
 
     if shutdown:
+        operation_logger.start()
         logger.warn(m18n.n('server_shutdown'))
         subprocess.check_call(['systemctl', 'poweroff'])
 
 
-def tools_reboot(force=False):
+@is_unit_operation()
+def tools_reboot(operation_logger, force=False):
     reboot = force
     if not reboot:
         try:
@@ -732,6 +747,7 @@ def tools_reboot(force=False):
             if i.lower() == 'y' or i.lower() == 'yes':
                 reboot = True
     if reboot:
+        operation_logger.start()
         logger.warn(m18n.n('server_reboot'))
         subprocess.check_call(['systemctl', 'reboot'])
 
@@ -852,12 +868,18 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
 
     # effectively run selected migrations
     for migration in migrations:
+
+        # Start register change on system
+        operation_logger= OperationLogger('tools_migrations_migrate_' + mode)
+        operation_logger.start()
+
         if not skip:
 
             logger.warn(m18n.n('migrations_show_currently_running_migration',
                                number=migration.number, name=migration.name))
 
             try:
+                migration.operation_logger = operation_logger
                 if mode == "forward":
                     migration.migrate()
                 elif mode == "backward":
@@ -867,11 +889,12 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
             except Exception as e:
                 # migration failed, let's stop here but still update state because
                 # we managed to run the previous ones
-                logger.error(m18n.n('migrations_migration_has_failed',
+                msg = m18n.n('migrations_migration_has_failed',
                                     exception=e,
                                     number=migration.number,
-                                    name=migration.name),
+                                    name=migration.name)
-                                    exc_info=1)
+                logger.error(msg, exc_info=1)
+                operation_logger.error(msg)
                 break
 
         else:  # if skip
@@ -885,6 +908,8 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
             "name": migration.name
         }
 
+        operation_logger.success()
+
     # special case where we want to go back from the start
     if target == 0:
         state["last_run_migration"] = None

src/yunohost/user.py

@@ -37,6 +37,7 @@ from moulinette import m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
 from yunohost.service import service_status
+from yunohost.log import is_unit_operation
 
 logger = getActionLogger('yunohost.user')
 
@@ -97,7 +98,8 @@ def user_list(auth, fields=None):
     return {'users': users}
 
 
-def user_create(auth, username, firstname, lastname, mail, password,
+@is_unit_operation([('username', 'user')])
+def user_create(operation_logger, auth, username, firstname, lastname, mail, password,
         mailbox_quota="0"):
     """
     Create user
@@ -132,6 +134,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
                               m18n.n('mail_domain_unknown',
                                      domain=mail.split("@")[1]))
 
+    operation_logger.start()
+
     # Get random UID/GID
     all_uid = {x.pw_uid for x in pwd.getpwall()}
     all_gid = {x.pw_gid for x in pwd.getpwall()}
@@ -217,7 +221,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
     raise MoulinetteError(169, m18n.n('user_creation_failed'))
 
 
-def user_delete(auth, username, purge=False):
+@is_unit_operation([('username', 'user')])
+def user_delete(operation_logger, auth, username, purge=False):
     """
     Delete user
 
@@ -229,6 +234,7 @@ def user_delete(auth, username, purge=False):
     from yunohost.app import app_ssowatconf
     from yunohost.hook import hook_callback
 
+    operation_logger.start()
     if auth.remove('uid=%s,ou=users' % username):
         # Invalidate passwd to take user deletion into account
         subprocess.call(['nscd', '-i', 'passwd'])
@@ -252,7 +258,8 @@ def user_delete(auth, username, purge=False):
     logger.success(m18n.n('user_deleted'))
 
 
-def user_update(auth, username, firstname=None, lastname=None, mail=None,
+@is_unit_operation([('username', 'user')], exclude=['auth', 'change_password'])
+def user_update(operation_logger, auth, username, firstname=None, lastname=None, mail=None,
         change_password=None, add_mailforward=None, remove_mailforward=None,
         add_mailalias=None, remove_mailalias=None, mailbox_quota=None):
     """
@@ -353,6 +360,8 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
     if mailbox_quota is not None:
         new_attr_dict['mailuserquota'] = mailbox_quota
 
+    operation_logger.start()
+
     if auth.update('uid=%s,ou=users' % username, new_attr_dict):
         logger.success(m18n.n('user_updated'))
         app_ssowatconf(auth)

src/yunohost/utils/yunopaste.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+import errno
+
+from moulinette.core import MoulinetteError
+
+def yunopaste(data):
+
+    paste_server = "https://paste.yunohost.org"
+
+    try:
+        r = requests.post("%s/documents" % paste_server, data=data, timeout=30)
+    except Exception as e:
+        raise MoulinetteError(errno.EIO,
+                              "Something wrong happened while trying to paste data on paste.yunohost.org : %s" % str(e))
+
+    if r.status_code != 200:
+        raise MoulinetteError(errno.EIO,
+                              "Something wrong happened while trying to paste data on paste.yunohost.org : %s, %s" % (r.status_code, r.text))
+
+    try:
+        url = json.loads(r.text)["key"]
+    except:
+        raise MoulinetteError(errno.EIO,
+                              "Uhoh, couldn't parse the answer from paste.yunohost.org : %s" % r.text)
+
+    return "%s/raw/%s" % (paste_server, url)