diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 53e6acaef..8509bfb23 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1710,3 +1710,39 @@ hook:
-d:
full: --chdir
help: The directory from where the script will be executed
+
+#############################
+# Log #
+#############################
+log:
+ category_help: Manage debug logs
+ actions:
+
+ ### log_list()
+ list:
+ action_help: List logs
+ api: GET /logs
+ arguments:
+ category:
+ help: Log category to display (default operations), could be operation, history, package, system, access, service or app
+ nargs: "*"
+ -l:
+ full: --limit
+ help: Maximum number of logs
+ type: int
+
+ ### log_display()
+ display:
+ action_help: Display a log content
+ api: GET /logs/display
+ arguments:
+ path:
+ help: Log file which to display the content
+ -n:
+ full: --number
+ help: Number of lines to display
+ default: 50
+ type: int
+ --share:
+ help: Share the full log using yunopaste
+ action: store_true
diff --git a/data/helpers.d/utils b/data/helpers.d/utils
index 07b4d4bb1..595da3c2d 100644
--- a/data/helpers.d/utils
+++ b/data/helpers.d/utils
@@ -257,3 +257,20 @@ ynh_local_curl () {
# Curl the URL
curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url"
}
+
+# Render templates with Jinja2
+#
+# Attention : Variables should be exported before calling this helper to be
+# accessible inside templates.
+#
+# usage: ynh_render_template some_template output_path
+# | arg: some_template - Template file to be rendered
+# | arg: output_path - The path where the output will be redirected to
+ynh_render_template() {
+ local template_path=$1
+ local output_path=$2
+ # Taken from https://stackoverflow.com/a/35009576
+ python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
+ jinja2.Template(sys.stdin.read()
+ ).render(os.environ));' < $template_path > $output_path
+}
diff --git a/data/hooks/conf_regen/19-postfix b/data/hooks/conf_regen/19-postfix
index 3cb5cdf50..a3ad70327 100755
--- a/data/hooks/conf_regen/19-postfix
+++ b/data/hooks/conf_regen/19-postfix
@@ -10,15 +10,25 @@ do_pre_regen() {
postfix_dir="${pending_dir}/etc/postfix"
mkdir -p "$postfix_dir"
+ default_dir="${pending_dir}/etc/default/"
+ mkdir -p "$default_dir"
+
# install plain conf files
cp plain/* "$postfix_dir"
# prepare main.cf conf file
main_domain=$(cat /etc/yunohost/current_host)
+ domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ')
+
cat main.cf \
| sed "s/{{ main_domain }}/${main_domain}/g" \
> "${postfix_dir}/main.cf"
+ cat postsrsd \
+ | sed "s/{{ main_domain }}/${main_domain}/g" \
+ | sed "s/{{ domain_list }}/${domain_list}/g" \
+ > "${default_dir}/postsrsd"
+
# adapt it for IPv4-only hosts
if [ ! -f /proc/net/if_inet6 ]; then
sed -i \
@@ -34,7 +44,8 @@ do_post_regen() {
regen_conf_files=$1
[[ -z "$regen_conf_files" ]] \
- || sudo service postfix restart
+ || { sudo service postfix restart && sudo service postsrsd restart; }
+
}
FORCE=${2:-0}
diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf
index 2cb1d8d72..c38896a3f 100644
--- a/data/templates/postfix/main.cf
+++ b/data/templates/postfix/main.cf
@@ -137,8 +137,10 @@ smtpd_recipient_restrictions =
permit
# SRS
-sender_canonical_maps = regexp:/etc/postfix/sender_canonical
+sender_canonical_maps = tcp:localhost:10001
sender_canonical_classes = envelope_sender
+recipient_canonical_maps = tcp:localhost:10002
+recipient_canonical_classes= envelope_recipient,header_recipient
# Ignore some headers
smtp_header_checks = regexp:/etc/postfix/header_checks
diff --git a/data/templates/postfix/postsrsd b/data/templates/postfix/postsrsd
new file mode 100644
index 000000000..56bfd091e
--- /dev/null
+++ b/data/templates/postfix/postsrsd
@@ -0,0 +1,43 @@
+# Default settings for postsrsd
+
+# Local domain name.
+# Addresses are rewritten to originate from this domain. The default value
+# is taken from `postconf -h mydomain` and probably okay.
+#
+SRS_DOMAIN={{ main_domain }}
+
+# Exclude additional domains.
+# You may list domains which shall not be subjected to address rewriting.
+# If a domain name starts with a dot, it matches all subdomains, but not
+# the domain itself. Separate multiple domains by space or comma.
+# We have to put some "dummy" stuff at start and end... see this comment :
+# https://github.com/roehling/postsrsd/issues/64#issuecomment-284003762
+SRS_EXCLUDE_DOMAINS=dummy {{ domain_list }} dummy
+
+# First separator character after SRS0 or SRS1.
+# Can be one of: -+=
+SRS_SEPARATOR==
+
+# Secret key to sign rewritten addresses.
+# When postsrsd is installed for the first time, a random secret is generated
+# and stored in /etc/postsrsd.secret. For most installations, that's just fine.
+#
+SRS_SECRET=/etc/postsrsd.secret
+
+# Local ports for TCP list.
+# These ports are used to bind the TCP list for postfix. If you change
+# these, you have to modify the postfix settings accordingly. The ports
+# are bound to the loopback interface, and should never be exposed on
+# the internet.
+#
+SRS_FORWARD_PORT=10001
+SRS_REVERSE_PORT=10002
+
+# Drop root privileges and run as another user after initialization.
+# This is highly recommended as postsrsd handles untrusted input.
+#
+RUN_AS=postsrsd
+
+# Jail daemon in chroot environment
+CHROOT=/var/lib/postsrsd
+
diff --git a/debian/control b/debian/control
index 256038598..fae93019b 100644
--- a/debian/control
+++ b/debian/control
@@ -12,13 +12,13 @@ Architecture: all
Depends: ${python:Depends}, ${misc:Depends}
, moulinette (>= 2.7.1), ssowat (>= 2.7.1)
, python-psutil, python-requests, python-dnspython, python-openssl
- , python-apt, python-miniupnpc, python-dbus
+ , python-apt, python-miniupnpc, python-dbus, python-jinja2
, glances
, dnsutils, bind9utils, unzip, git, curl, cron, wget
, ca-certificates, netcat-openbsd, iproute
, mariadb-server, php-mysql | php-mysqlnd
, slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd
- , postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils
+ , postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils, postsrsd
, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved
, dovecot-antispam, fail2ban
, nginx-extras (>=1.6.2), php-fpm, php-ldap, php-intl
diff --git a/locales/en.json b/locales/en.json
index 45f002881..074512311 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -206,6 +206,49 @@
"invalid_url_format": "Invalid URL format",
"ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it",
"iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it",
+ "log_corrupted_md_file": "The yaml metadata file associated with logs is corrupted : '{md_file}'",
+ "log_category_404": "The log category '{category}' does not exist",
+ "log_link_to_log": "Full log of this operation: '{desc}'",
+ "log_help_to_get_log": "To view the log of the operation '{desc}', use the command 'yunohost log display {name}'",
+ "log_link_to_failed_log": "The operation '{desc}' has failed ! To get help, please provide the full log of this operation",
+ "log_help_to_get_failed_log": "The operation '{desc}' has failed ! To get help, please share the full log of this operation using the command 'yunohost log display {name} --share'",
+ "log_category_404": "The log category '{category}' does not exist",
+ "log_does_exists": "There is not operation log with the name '{log}', use 'yunohost log list to see all available operation logs'",
+ "log_operation_unit_unclosed_properly": "Operation unit has not been closed properly",
+ "log_app_addaccess": "Add access to '{}'",
+ "log_app_removeaccess": "Remove access to '{}'",
+ "log_app_clearaccess": "Remove all access to '{}'",
+ "log_app_fetchlist": "Add an application list",
+ "log_app_removelist": "Remove an application list",
+ "log_app_change_url": "Change the url of '{}' application",
+ "log_app_install": "Install '{}' application",
+ "log_app_remove": "Remove '{}' application",
+ "log_app_upgrade": "Upgrade '{}' application",
+ "log_app_makedefault": "Make '{}' as default application",
+ "log_available_on_yunopaste": "This log is now available via {url}",
+ "log_backup_restore_system": "Restore system from a backup archive",
+ "log_backup_restore_app": "Restore '{}' from a backup archive",
+ "log_remove_on_failed_restore": "Remove '{}' after a failed restore from a backup archive",
+ "log_remove_on_failed_install": "Remove '{}' after a failed installation",
+ "log_domain_add": "Add '{}' domain into system configuration",
+ "log_domain_remove": "Remove '{}' domain from system configuration",
+ "log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'",
+ "log_dyndns_update": "Update the ip associated with your YunoHost subdomain '{}'",
+ "log_letsencrypt_cert_install": "Install Let's encrypt certificate on '{}' domain",
+ "log_selfsigned_cert_install": "Install self signed certificate on '{}' domain",
+ "log_letsencrypt_cert_renew": "Renew '{}' Let's encrypt certificate",
+ "log_service_enable": "Enable '{}' service",
+ "log_service_regen_conf": "Regenerate system configurations '{}'",
+ "log_user_create": "Add '{}' user",
+ "log_user_delete": "Delete '{}' user",
+ "log_user_update": "Update information of '{}' user",
+ "log_tools_maindomain": "Make '{}' as main domain",
+ "log_tools_migrations_migrate_forward": "Migrate forward",
+ "log_tools_migrations_migrate_backward": "Migrate backward",
+ "log_tools_postinstall": "Postinstall your YunoHost server",
+ "log_tools_upgrade": "Upgrade debian packages",
+ "log_tools_shutdown": "Shutdown your server",
+ "log_tools_reboot": "Reboot your server",
"ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user",
"ldap_initialized": "LDAP has been initialized",
"license_undefined": "undefined",
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 3e192cc38..1fed09425 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -44,6 +44,7 @@ from moulinette.utils.filesystem import read_json
from yunohost.service import service_log, _run_service_command
from yunohost.utils import packages
+from yunohost.log import is_unit_operation, OperationLogger
logger = getActionLogger('yunohost.app')
@@ -109,10 +110,13 @@ def app_fetchlist(url=None, name=None):
# the fetch only this list
if url is not None:
if name:
+ operation_logger = OperationLogger('app_fetchlist')
+ operation_logger.start()
_register_new_appslist(url, name)
# Refresh the appslists dict
appslists = _read_appslist_list()
appslists_to_be_fetched = [name]
+ operation_logger.success()
else:
raise MoulinetteError(errno.EINVAL,
m18n.n('custom_appslist_name_required'))
@@ -188,7 +192,8 @@ def app_fetchlist(url=None, name=None):
_write_appslist_list(appslists)
-def app_removelist(name):
+@is_unit_operation()
+def app_removelist(operation_logger, name):
"""
Remove list from the repositories
@@ -202,6 +207,8 @@ def app_removelist(name):
if name not in appslists.keys():
raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name))
+ operation_logger.start()
+
# Remove json
json_path = '%s/%s.json' % (REPO_PATH, name)
if os.path.exists(json_path):
@@ -425,7 +432,8 @@ def app_map(app=None, raw=False, user=None):
return result
-def app_change_url(auth, app, domain, path):
+@is_unit_operation()
+def app_change_url(operation_logger, auth, app, domain, path):
"""
Modify the URL at which an application is installed.
@@ -482,6 +490,11 @@ def app_change_url(auth, app, domain, path):
env_dict["YNH_APP_NEW_DOMAIN"] = domain
env_dict["YNH_APP_NEW_PATH"] = path.rstrip("/")
+ if domain != old_domain:
+ operation_logger.related_to.append(('domain', old_domain))
+ operation_logger.extra.update({'env': env_dict})
+ operation_logger.start()
+
if os.path.exists(os.path.join(APP_TMP_FOLDER, "scripts")):
shutil.rmtree(os.path.join(APP_TMP_FOLDER, "scripts"))
@@ -499,16 +512,16 @@ def app_change_url(auth, app, domain, path):
os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts")))
os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url")))
- # XXX journal
if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'),
args=args_list, env=env_dict, user="root") != 0:
- logger.error("Failed to change '%s' url." % app)
+ msg = "Failed to change '%s' url." % app
+ logger.error(msg)
+ operation_logger.error(msg)
# restore values modified by app_checkurl
# see begining of the function
app_setting(app, "domain", value=old_domain)
app_setting(app, "path", value=old_path)
-
return
# this should idealy be done in the change_url script but let's avoid common mistakes
@@ -546,7 +559,6 @@ def app_upgrade(auth, app=[], url=None, file=None):
"""
from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
-
# Retrieve interface
is_api = msettings.get('interface') == 'api'
@@ -617,6 +629,11 @@ def app_upgrade(auth, app=[], url=None, file=None):
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
+ # Start register change on system
+ related_to = [('app', app_instance_name)]
+ operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict)
+ operation_logger.start()
+
# Apply dirty patch to make php5 apps compatible with php7
_patch_php5(extracted_app_folder)
@@ -624,7 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None):
os.system('chown -hR admin: %s' % INSTALL_TMP)
if hook_exec(extracted_app_folder + '/scripts/upgrade',
args=args_list, env=env_dict, user="root") != 0:
- logger.error(m18n.n('app_upgrade_failed', app=app_instance_name))
+ msg = m18n.n('app_upgrade_failed', app=app_instance_name)
+ logger.error(msg)
+ operation_logger.error(msg)
else:
now = int(time.time())
# TODO: Move install_time away from app_setting
@@ -654,7 +673,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
logger.success(m18n.n('app_upgraded', app=app_instance_name))
hook_callback('post_app_upgrade', args=args_list, env=env_dict)
-
+ operation_logger.success()
if not upgraded_apps:
raise MoulinetteError(errno.ENODATA, m18n.n('app_no_upgrade'))
@@ -668,7 +687,8 @@ def app_upgrade(auth, app=[], url=None, file=None):
return {"log": service_log('yunohost-api', number="100").values()[0]}
-def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
+@is_unit_operation()
+def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on_failure=False):
"""
Install apps
@@ -680,6 +700,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
"""
from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
+ from yunohost.log import OperationLogger
+
# Fetch or extract sources
try:
@@ -737,6 +759,12 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
+ # Start register change on system
+ operation_logger.extra.update({'env':env_dict})
+ operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
+ operation_logger.related_to.append(("app", app_id))
+ operation_logger.start()
+
# Create app directory
app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
if os.path.exists(app_setting_path):
@@ -772,13 +800,15 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
try:
install_retcode = hook_exec(
os.path.join(extracted_app_folder, 'scripts/install'),
- args=args_list, env=env_dict, user="root")
+ args=args_list, env=env_dict, user="root"
+ )
except (KeyboardInterrupt, EOFError):
install_retcode = -1
except:
logger.exception(m18n.n('unexpected_error'))
finally:
if install_retcode != 0:
+ error_msg = operation_logger.error(m18n.n('unexpected_error'))
if not no_remove_on_failure:
# Setup environment for remove script
env_dict_remove = {}
@@ -787,12 +817,22 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
# Execute remove script
+ operation_logger_remove = OperationLogger('remove_on_failed_install',
+ [('app', app_instance_name)],
+ env=env_dict_remove)
+ operation_logger_remove.start()
+
remove_retcode = hook_exec(
os.path.join(extracted_app_folder, 'scripts/remove'),
- args=[app_instance_name], env=env_dict_remove, user="root")
+ args=[app_instance_name], env=env_dict_remove, user="root"
+ )
if remove_retcode != 0:
- logger.warning(m18n.n('app_not_properly_removed',
- app=app_instance_name))
+ msg = m18n.n('app_not_properly_removed',
+ app=app_instance_name)
+ logger.warning(msg)
+ operation_logger_remove.error(msg)
+ else:
+ operation_logger_remove.success()
# Clean tmp folders
shutil.rmtree(app_setting_path)
@@ -801,9 +841,10 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
app_ssowatconf(auth)
if install_retcode == -1:
- raise MoulinetteError(errno.EINTR,
- m18n.g('operation_interrupted'))
- raise MoulinetteError(errno.EIO, m18n.n('installation_failed'))
+ msg = m18n.n('operation_interrupted') + " " + error_msg
+ raise MoulinetteError(errno.EINTR, msg)
+ msg = error_msg
+ raise MoulinetteError(errno.EIO, msg)
# Clean hooks and add new ones
hook_remove(app_instance_name)
@@ -828,7 +869,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
hook_callback('post_app_install', args=args_list, env=env_dict)
-def app_remove(auth, app):
+@is_unit_operation()
+def app_remove(operation_logger, auth, app):
"""
Remove app
@@ -837,11 +879,12 @@ def app_remove(auth, app):
"""
from yunohost.hook import hook_exec, hook_remove, hook_callback
-
if not _is_installed(app):
raise MoulinetteError(errno.EINVAL,
m18n.n('app_not_installed', app=app))
+ operation_logger.start()
+
app_setting_path = APPS_SETTING_PATH + app
# TODO: display fail messages from script
@@ -865,6 +908,8 @@ def app_remove(auth, app):
env_dict["YNH_APP_ID"] = app_id
env_dict["YNH_APP_INSTANCE_NAME"] = app
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
+ operation_logger.extra.update({'env': env_dict})
+ operation_logger.flush()
if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list,
env=env_dict, user="root") == 0:
@@ -901,6 +946,8 @@ def app_addaccess(auth, apps, users=[]):
apps = [apps, ]
for app in apps:
+
+
app_settings = _get_app_settings(app)
if not app_settings:
continue
@@ -910,6 +957,12 @@ def app_addaccess(auth, apps, users=[]):
app_settings['mode'] = 'private'
if app_settings['mode'] == 'private':
+
+ # Start register change on system
+ related_to = [('app', app)]
+ operation_logger= OperationLogger('app_addaccess', related_to)
+ operation_logger.start()
+
allowed_users = set()
if 'allowed_users' in app_settings:
allowed_users = set(app_settings['allowed_users'].split(','))
@@ -922,11 +975,15 @@ def app_addaccess(auth, apps, users=[]):
logger.warning(m18n.n('user_unknown', user=allowed_user))
continue
allowed_users.add(allowed_user)
+ operation_logger.related_to.append(('user', allowed_user))
+ operation_logger.flush()
new_users = ','.join(allowed_users)
app_setting(app, 'allowed_users', new_users)
hook_callback('post_app_addaccess', args=[app, new_users])
+ operation_logger.success()
+
result[app] = allowed_users
app_ssowatconf(auth)
@@ -963,6 +1020,12 @@ def app_removeaccess(auth, apps, users=[]):
allowed_users = set()
if app_settings.get('skipped_uris', '') != '/':
+
+ # Start register change on system
+ related_to = [('app', app)]
+ operation_logger= OperationLogger('app_removeaccess', related_to)
+ operation_logger.start()
+
if remove_all:
pass
elif 'allowed_users' in app_settings:
@@ -972,14 +1035,18 @@ def app_removeaccess(auth, apps, users=[]):
else:
for allowed_user in user_list(auth)['users'].keys():
if allowed_user not in users:
- allowed_users.add(allowed_user)
+ allowed_users.append(allowed_user)
+ operation_logger.related_to += [ ('user', x) for x in allowed_users ]
+ operation_logger.flush()
new_users = ','.join(allowed_users)
app_setting(app, 'allowed_users', new_users)
hook_callback('post_app_removeaccess', args=[app, new_users])
result[app] = allowed_users
+ operation_logger.success()
+
app_ssowatconf(auth)
return {'allowed_users': result}
@@ -1003,6 +1070,11 @@ def app_clearaccess(auth, apps):
if not app_settings:
continue
+ # Start register change on system
+ related_to = [('app', app)]
+ operation_logger= OperationLogger('app_clearaccess', related_to)
+ operation_logger.start()
+
if 'mode' in app_settings:
app_setting(app, 'mode', delete=True)
@@ -1011,6 +1083,8 @@ def app_clearaccess(auth, apps):
hook_callback('post_app_clearaccess', args=[app])
+ operation_logger.success()
+
app_ssowatconf(auth)
@@ -1037,7 +1111,8 @@ def app_debug(app):
}
-def app_makedefault(auth, app, domain=None):
+@is_unit_operation()
+def app_makedefault(operation_logger, auth, app, domain=None):
"""
Redirect domain root to an app
@@ -1054,9 +1129,11 @@ def app_makedefault(auth, app, domain=None):
if domain is None:
domain = app_domain
+ operation_logger.related_to.append(('domain',domain))
elif domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
+ operation_logger.start()
if '/' in app_map(raw=True)[domain]:
raise MoulinetteError(errno.EEXIST,
m18n.n('app_make_default_location_already_used',
@@ -2204,7 +2281,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
app_label=app_label,
))
- raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', "\n".join(apps=apps)))
+ raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', apps="\n".join(apps)))
# (We save this normalized path so that the install script have a
# standard path format to deal with no matter what the user inputted)
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index acb7eb574..88959cc2f 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -51,6 +51,7 @@ from yunohost.hook import (
from yunohost.monitor import binary_to_human
from yunohost.tools import tools_postinstall
from yunohost.service import service_regen_conf
+from yunohost.log import OperationLogger
BACKUP_PATH = '/home/yunohost.backup'
ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
@@ -1172,9 +1173,15 @@ class RestoreManager():
if system_targets == []:
return
+ # Start register change on system
+ operation_logger = OperationLogger('backup_restore_system')
+ operation_logger.start()
+
logger.debug(m18n.n('restore_running_hooks'))
env_dict = self._get_env_var()
+ operation_logger.extra['env'] = env_dict
+ operation_logger.flush()
ret = hook_callback('restore',
system_targets,
args=[self.work_dir],
@@ -1184,9 +1191,16 @@ class RestoreManager():
for part in ret['succeed'].keys():
self.targets.set_result("system", part, "Success")
+ error_part = []
for part in ret['failed'].keys():
logger.error(m18n.n('restore_system_part_failed', part=part))
self.targets.set_result("system", part, "Error")
+ error_part.append(part)
+
+ if ret['failed']:
+ operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part)))
+ else:
+ operation_logger.success()
service_regen_conf()
@@ -1234,6 +1248,11 @@ class RestoreManager():
else:
shutil.copy2(s, d)
+ # Start register change on system
+ related_to = [('app', app_instance_name)]
+ operation_logger = OperationLogger('backup_restore_app', related_to)
+ operation_logger.start()
+
# Check if the app is not already installed
if _is_installed(app_instance_name):
logger.error(m18n.n('restore_already_installed_app',
@@ -1283,6 +1302,9 @@ class RestoreManager():
# Prepare env. var. to pass to script
env_dict = self._get_env_var(app_instance_name)
+ operation_logger.extra['env'] = env_dict
+ operation_logger.flush()
+
# Execute app restore script
hook_exec(restore_script,
args=[app_backup_in_archive, app_instance_name],
@@ -1291,8 +1313,10 @@ class RestoreManager():
env=env_dict,
user="root")
except:
- logger.exception(m18n.n('restore_app_failed',
- app=app_instance_name))
+ msg = m18n.n('restore_app_failed',app=app_instance_name)
+ logger.exception(msg)
+ operation_logger.error(msg)
+
self.targets.set_result("apps", app_instance_name, "Error")
remove_script = os.path.join(app_scripts_in_archive, 'remove')
@@ -1304,12 +1328,20 @@ class RestoreManager():
env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
+ operation_logger = OperationLogger('remove_on_failed_restore',
+ [('app', app_instance_name)],
+ env=env_dict_remove)
+ operation_logger.start()
+
# Execute remove script
# TODO: call app_remove instead
if hook_exec(remove_script, args=[app_instance_name],
env=env_dict_remove, user="root") != 0:
- logger.warning(m18n.n('app_not_properly_removed',
- app=app_instance_name))
+ msg = m18n.n('app_not_properly_removed', app=app_instance_name)
+ logger.warning(msg)
+ operation_logger.error(msg)
+ else:
+ operation_logger.success()
# Cleaning app directory
shutil.rmtree(app_settings_new_path, ignore_errors=True)
@@ -1317,6 +1349,7 @@ class RestoreManager():
# TODO Cleaning app hooks
else:
self.targets.set_result("apps", app_instance_name, "Success")
+ operation_logger.success()
finally:
# Cleaning temporary scripts directory
shutil.rmtree(tmp_folder_for_app_restore, ignore_errors=True)
diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py
index 930bc0293..1b80b6b49 100644
--- a/src/yunohost/certificate.py
+++ b/src/yunohost/certificate.py
@@ -45,7 +45,7 @@ from yunohost.utils.network import get_public_ip
from moulinette import m18n
from yunohost.app import app_ssowatconf
from yunohost.service import _run_service_command, service_regen_conf
-
+from yunohost.log import OperationLogger
logger = getActionLogger('yunohost.certmanager')
@@ -160,6 +160,9 @@ def _certificate_install_selfsigned(domain_list, force=False):
for domain in domain_list:
+ operation_logger = OperationLogger('selfsigned_cert_install', [('domain', domain)],
+ args={'force': force})
+
# Paths of files and folder we'll need
date_tag = datetime.now().strftime("%Y%m%d.%H%M%S")
new_cert_folder = "%s/%s-history/%s-selfsigned" % (
@@ -182,6 +185,8 @@ def _certificate_install_selfsigned(domain_list, force=False):
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_attempt_to_replace_valid_cert', domain=domain))
+ operation_logger.start()
+
# Create output folder for new certificate stuff
os.makedirs(new_cert_folder)
@@ -238,9 +243,11 @@ def _certificate_install_selfsigned(domain_list, force=False):
if status and status["CA_type"]["code"] == "self-signed" and status["validity"] > 3648:
logger.success(
m18n.n("certmanager_cert_install_success_selfsigned", domain=domain))
+ operation_logger.success()
else:
- logger.error(
- "Installation of self-signed certificate installation for %s failed !", domain)
+ msg = "Installation of self-signed certificate installation for %s failed !" % (domain)
+ logger.error(msg)
+ operation_logger.error(msg)
def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=False, staging=False):
@@ -281,6 +288,9 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
# Actual install steps
for domain in domain_list:
+ operation_logger = OperationLogger('letsencrypt_cert_install', [('domain', domain)],
+ args={'force': force, 'no_checks': no_checks,
+ 'staging': staging})
logger.info(
"Now attempting install of certificate for domain %s!", domain)
@@ -288,6 +298,8 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
if not no_checks:
_check_domain_is_ready_for_ACME(domain)
+ operation_logger.start()
+
_configure_for_acme_challenge(auth, domain)
_fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
_install_cron()
@@ -295,10 +307,12 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
logger.success(
m18n.n("certmanager_cert_install_success", domain=domain))
+ operation_logger.success()
except Exception as e:
_display_debug_information(domain)
- logger.error("Certificate installation for %s failed !\nException: %s", domain, e)
-
+ msg = "Certificate installation for %s failed !\nException: %s" % (domain, e)
+ logger.error(msg)
+ operation_logger.error(msg)
def certificate_renew(auth, domain_list, force=False, no_checks=False, email=False, staging=False):
"""
@@ -376,6 +390,11 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
# Actual renew steps
for domain in domain_list:
+
+ operation_logger = OperationLogger('letsencrypt_cert_renew', [('domain', domain)],
+ args={'force': force, 'no_checks': no_checks,
+ 'staging': staging, 'email': email})
+
logger.info(
"Now attempting renewing of certificate for domain %s !", domain)
@@ -383,17 +402,23 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
if not no_checks:
_check_domain_is_ready_for_ACME(domain)
+ operation_logger.start()
+
_fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
logger.success(
m18n.n("certmanager_cert_renew_success", domain=domain))
+ operation_logger.success()
+
except Exception as e:
import traceback
from StringIO import StringIO
stack = StringIO()
traceback.print_exc(file=stack)
- logger.error("Certificate renewing for %s failed !", domain)
+ msg = "Certificate renewing for %s failed !" % (domain)
+ logger.error(msg)
+ operation_logger.error(msg)
logger.error(stack.getvalue())
logger.error(str(e))
@@ -401,7 +426,6 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
logger.error("Sending email with details to root ...")
_email_renewing_failed(domain, e, stack.getvalue())
-
###############################################################################
# Back-end stuff #
###############################################################################
diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py
index 08d74185b..560a6fda5 100644
--- a/src/yunohost/domain.py
+++ b/src/yunohost/domain.py
@@ -37,6 +37,7 @@ import yunohost.certificate
from yunohost.service import service_regen_conf
from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.domain')
@@ -61,7 +62,8 @@ def domain_list(auth):
return {'domains': result_list}
-def domain_add(auth, domain, dyndns=False):
+@is_unit_operation()
+def domain_add(operation_logger, auth, domain, dyndns=False):
"""
Create a custom domain
@@ -78,6 +80,8 @@ def domain_add(auth, domain, dyndns=False):
except MoulinetteError:
raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))
+ operation_logger.start()
+
# DynDNS domain
if dyndns:
@@ -110,23 +114,27 @@ def domain_add(auth, domain, dyndns=False):
# Don't regen these conf if we're still in postinstall
if os.path.exists('/etc/yunohost/installed'):
- service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
+ service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
app_ssowatconf(auth)
- except:
+ except Exception, e:
+ from sys import exc_info;
+ t, v, tb = exc_info()
+
# Force domain removal silently
try:
domain_remove(auth, domain, True)
except:
pass
- raise
+ raise t, v, tb
hook_callback('post_domain_add', args=[domain])
logger.success(m18n.n('domain_created'))
-def domain_remove(auth, domain, force=False):
+@is_unit_operation()
+def domain_remove(operation_logger, auth, domain, force=False):
"""
Delete domains
@@ -157,12 +165,13 @@ def domain_remove(auth, domain, force=False):
raise MoulinetteError(errno.EPERM,
m18n.n('domain_uninstall_app_first'))
+ operation_logger.start()
if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
os.system('rm -rf /etc/yunohost/certs/%s' % domain)
else:
raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))
- service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
+ service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
app_ssowatconf(auth)
hook_callback('post_domain_remove', args=[domain])
diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py
index 785b0dd34..88547b4db 100644
--- a/src/yunohost/dyndns.py
+++ b/src/yunohost/dyndns.py
@@ -40,6 +40,7 @@ from moulinette.utils.network import download_json
from yunohost.domain import _get_maindomain, _build_dns_conf
from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.dyndns')
@@ -112,7 +113,8 @@ def _dyndns_available(provider, domain):
return r == u"Domain %s is available" % domain
-def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None):
+@is_unit_operation()
+def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", domain=None, key=None):
"""
Subscribe to a DynDNS service
@@ -124,6 +126,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
"""
if domain is None:
domain = _get_maindomain()
+ operation_logger.related_to.append(('domain', domain))
# Verify if domain is provided by subscribe_host
if not _dyndns_provides(subscribe_host, domain):
@@ -136,6 +139,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
raise MoulinetteError(errno.ENOENT,
m18n.n('dyndns_unavailable', domain=domain))
+ operation_logger.start()
+
if key is None:
if len(glob.glob('/etc/yunohost/dyndns/*.key')) == 0:
if not os.path.exists('/etc/yunohost/dyndns'):
@@ -170,7 +175,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
dyndns_installcron()
-def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
+@is_unit_operation()
+def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, key=None,
ipv4=None, ipv6=None):
"""
Update IP on DynDNS platform
@@ -217,13 +223,17 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
if domain is None:
(domain, key) = _guess_current_dyndns_domain(dyn_host)
# If key is not given, pick the first file we find with the domain given
- elif key is None:
- keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
+ else:
+ if key is None:
+ keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
- if not keys:
- raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
+ if not keys:
+ raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
- key = keys[0]
+ key = keys[0]
+
+ operation_logger.related_to.append(('domain', domain))
+ operation_logger.start()
# This mean that hmac-md5 is used
# (Re?)Trigger the migration to sha256 and return immediately.
diff --git a/src/yunohost/log.py b/src/yunohost/log.py
new file mode 100644
index 000000000..c105b8279
--- /dev/null
+++ b/src/yunohost/log.py
@@ -0,0 +1,462 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+ Copyright (C) 2018 YunoHost
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+""" yunohost_log.py
+
+ Manage debug logs
+"""
+
+import os
+import yaml
+import errno
+import collections
+
+from datetime import datetime
+from logging import FileHandler, getLogger, Formatter
+from sys import exc_info
+
+from moulinette import m18n, msettings
+from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file
+
+CATEGORIES_PATH = '/var/log/yunohost/categories/'
+OPERATIONS_PATH = '/var/log/yunohost/categories/operation/'
+CATEGORIES = ['operation', 'history', 'package', 'system', 'access', 'service',
+ 'app']
+METADATA_FILE_EXT = '.yml'
+LOG_FILE_EXT = '.log'
+RELATED_CATEGORIES = ['app', 'domain', 'service', 'user']
+
+logger = getActionLogger('yunohost.log')
+
+
+def log_list(category=[], limit=None):
+ """
+ List available logs
+
+ Keyword argument:
+ limit -- Maximum number of logs
+ """
+
+ categories = category
+ is_api = msettings.get('interface') == 'api'
+
+ # In cli we just display `operation` logs by default
+ if not categories:
+ categories = ["operation"] if not is_api else CATEGORIES
+
+ result = collections.OrderedDict()
+ for category in categories:
+ result[category] = []
+
+ category_path = os.path.join(CATEGORIES_PATH, category)
+ if not os.path.exists(category_path):
+ logger.debug(m18n.n('log_category_404', category=category))
+
+ continue
+
+ logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
+ os.listdir(category_path))
+ logs = reversed(sorted(logs))
+
+ if limit is not None:
+ logs = logs[:limit]
+
+ for log in logs:
+
+ base_filename = log[:-len(METADATA_FILE_EXT)]
+ md_filename = log
+ md_path = os.path.join(category_path, md_filename)
+
+ log = base_filename.split("-")
+
+ entry = {
+ "name": base_filename,
+ "path": md_path,
+ }
+ entry["description"] = _get_description_from_name(base_filename)
+ try:
+ log_datetime = datetime.strptime(" ".join(log[:2]),
+ "%Y%m%d %H%M%S")
+ except ValueError:
+ pass
+ else:
+ entry["started_at"] = log_datetime
+
+ result[category].append(entry)
+
+ # Reverse the order of log when in cli, more comfortable to read (avoid
+ # unecessary scrolling)
+ if not is_api:
+ for category in result:
+ result[category] = list(reversed(result[category]))
+
+ return result
+
+
+def log_display(path, number=50, share=False):
+ """
+ Display a log file enriched with metadata if any.
+
+ If the file_name is not an absolute path, it will try to search the file in
+ the unit operations log path (see OPERATIONS_PATH).
+
+ Argument:
+ file_name
+ number
+ share
+ """
+
+ # Normalize log/metadata paths and filenames
+ abs_path = path
+ log_path = None
+ if not path.startswith('/'):
+ for category in CATEGORIES:
+ abs_path = os.path.join(CATEGORIES_PATH, category, path)
+ if os.path.exists(abs_path) or os.path.exists(abs_path + METADATA_FILE_EXT):
+ break
+
+ if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT):
+ log_path = abs_path
+
+ if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT):
+ base_path = ''.join(os.path.splitext(abs_path)[:-1])
+ else:
+ base_path = abs_path
+ base_filename = os.path.basename(base_path)
+ md_path = base_path + METADATA_FILE_EXT
+ if log_path is None:
+ log_path = base_path + LOG_FILE_EXT
+
+ if not os.path.exists(md_path) and not os.path.exists(log_path):
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('log_does_exists', log=path))
+
+ infos = {}
+
+ # If it's a unit operation, display the name and the description
+ if base_path.startswith(CATEGORIES_PATH):
+ infos["description"] = _get_description_from_name(base_filename)
+ infos['name'] = base_filename
+
+ if share:
+ from yunohost.utils.yunopaste import yunopaste
+ content = ""
+ if os.path.exists(md_path):
+ content += read_file(md_path)
+ content += "\n============\n\n"
+ if os.path.exists(log_path):
+ content += read_file(log_path)
+
+ url = yunopaste(content)
+
+ logger.info(m18n.n("log_available_on_yunopaste", url=url))
+ if msettings.get('interface') == 'api':
+ return {"url": url}
+ else:
+ return
+
+ # Display metadata if exist
+ if os.path.exists(md_path):
+ with open(md_path, "r") as md_file:
+ try:
+ metadata = yaml.safe_load(md_file)
+ infos['metadata_path'] = md_path
+ infos['metadata'] = metadata
+ if 'log_path' in metadata:
+ log_path = metadata['log_path']
+ except yaml.YAMLError:
+ error = m18n.n('log_corrupted_md_file', file=md_path)
+ if os.path.exists(log_path):
+ logger.warning(error)
+ else:
+ raise MoulinetteError(errno.EINVAL, error)
+
+ # Display logs if exist
+ if os.path.exists(log_path):
+ from yunohost.service import _tail
+ logs = _tail(log_path, int(number))
+ infos['log_path'] = log_path
+ infos['logs'] = logs
+
+ return infos
+
+
+def is_unit_operation(entities=['app', 'domain', 'service', 'user'],
+ exclude=['auth', 'password'], operation_key=None):
+ """
+ Configure quickly a unit operation
+
+ This decorator help you to configure the record of a unit operations.
+
+ Argument:
+ entities A list of entity types related to the unit operation. The entity
+ type is searched inside argument's names of the decorated function. If
+ something match, the argument value is added as related entity. If the
+ argument name is different you can specify it with a tuple
+ (argname, entity_type) instead of just put the entity type.
+
+ exclude Remove some arguments from the context. By default, arguments
+ called 'password' and 'auth' are removed. If an argument is an object, you
+ need to exclude it or create manually the unit operation without this
+ decorator.
+
+ operation_key A key to describe the unit operation log used to create the
+ filename and search a translation. Please ensure that this key prefixed by
+ 'log_' is present in locales/en.json otherwise it won't be translatable.
+
+ """
+ def decorate(func):
+ def func_wrapper(*args, **kwargs):
+ op_key = operation_key
+ if op_key is None:
+ op_key = func.__name__
+
+ # If the function is called directly from an other part of the code
+ # and not by the moulinette framework, we need to complete kwargs
+ # dictionnary with the args list.
+ # Indeed, we use convention naming in this decorator and we need to
+ # know name of each args (so we need to use kwargs instead of args)
+ if len(args) > 0:
+ from inspect import getargspec
+ keys = getargspec(func).args
+ if 'operation_logger' in keys:
+ keys.remove('operation_logger')
+ for k, arg in enumerate(args):
+ kwargs[keys[k]] = arg
+ args = ()
+
+ # Search related entity in arguments of the decorated function
+ related_to = []
+ for entity in entities:
+ if isinstance(entity, tuple):
+ entity_type = entity[1]
+ entity = entity[0]
+ else:
+ entity_type = entity
+
+ if entity in kwargs and kwargs[entity] is not None:
+ if isinstance(kwargs[entity], basestring):
+ related_to.append((entity_type, kwargs[entity]))
+ else:
+ for x in kwargs[entity]:
+ related_to.append((entity_type, x))
+
+ context = kwargs.copy()
+
+ # Exclude unappropriate data from the context
+ for field in exclude:
+ if field in context:
+ context.pop(field, None)
+ operation_logger = OperationLogger(op_key, related_to, args=context)
+
+ try:
+ # Start the actual function, and give the unit operation
+ # in argument to let the developper start the record itself
+ args = (operation_logger,) + args
+ result = func(*args, **kwargs)
+ except Exception as e:
+ operation_logger.error(e)
+ raise
+ else:
+ operation_logger.success()
+ return result
+ return func_wrapper
+ return decorate
+
+
+class OperationLogger(object):
+ """
+ Instances of this class represents unit operation done on the ynh instance.
+
+ Each time an action of the yunohost cli/api change the system, one or
+ several unit operations should be registered.
+
+ This class record logs and metadata like context or start time/end time.
+ """
+
+ def __init__(self, operation, related_to=None, **kwargs):
+ # TODO add a way to not save password on app installation
+ self.operation = operation
+ self.related_to = related_to
+ self.extra = kwargs
+ self.started_at = None
+ self.ended_at = None
+ self.logger = None
+ self._name = None
+
+ self.path = OPERATIONS_PATH
+
+ if not os.path.exists(self.path):
+ os.makedirs(self.path)
+
+ def start(self):
+ """
+ Start to record logs that change the system
+ Until this start method is run, no unit operation will be registered.
+ """
+
+ if self.started_at is None:
+ self.started_at = datetime.now()
+ self.flush()
+ self._register_log()
+
+ def _register_log(self):
+ """
+ Register log with a handler connected on log system
+ """
+
+ # TODO add a way to not save password on app installation
+ filename = os.path.join(self.path, self.name + LOG_FILE_EXT)
+ self.file_handler = FileHandler(filename)
+ self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
+
+ # Listen to the root logger
+ self.logger = getLogger('yunohost')
+ self.logger.addHandler(self.file_handler)
+
+ def flush(self):
+ """
+ Write or rewrite the metadata file with all metadata known
+ """
+
+ filename = os.path.join(self.path, self.name + METADATA_FILE_EXT)
+ with open(filename, 'w') as outfile:
+ yaml.safe_dump(self.metadata, outfile, default_flow_style=False)
+
+ @property
+ def name(self):
+ """
+ Name of the operation
+ This name is used as filename, so don't use space
+ """
+ if self._name is not None:
+ return self._name
+
+ name = [self.started_at.strftime("%Y%m%d-%H%M%S")]
+ name += [self.operation]
+
+ if hasattr(self, "name_parameter_override"):
+ # This is for special cases where the operation is not really
+ # unitary. For instance, the regen conf cannot be logged "per
+ # service" because of the way it's built
+ name.append(self.name_parameter_override)
+ elif self.related_to:
+ # We use the name of the first related thing
+ name.append(self.related_to[0][1])
+
+ self._name = '-'.join(name)
+ return self._name
+
+ @property
+ def metadata(self):
+ """
+ Dictionnary of all metadata collected
+ """
+
+ data = {
+ 'started_at': self.started_at,
+ 'operation': self.operation,
+ }
+ if self.related_to is not None:
+ data['related_to'] = self.related_to
+ if self.ended_at is not None:
+ data['ended_at'] = self.ended_at
+ data['success'] = self._success
+ if self.error is not None:
+ data['error'] = self._error
+ # TODO: detect if 'extra' erase some key of 'data'
+ data.update(self.extra)
+ return data
+
+ def success(self):
+ """
+ Declare the success end of the unit operation
+ """
+ self.close()
+
+ def error(self, error):
+ """
+ Declare the failure of the unit operation
+ """
+ return self.close(error)
+
+ def close(self, error=None):
+ """
+ Close properly the unit operation
+ """
+ if self.ended_at is not None or self.started_at is None:
+ return
+ if error is not None and not isinstance(error, basestring):
+ error = str(error)
+ self.ended_at = datetime.now()
+ self._error = error
+ self._success = error is None
+ if self.logger is not None:
+ self.logger.removeHandler(self.file_handler)
+
+ is_api = msettings.get('interface') == 'api'
+ desc = _get_description_from_name(self.name)
+ if error is None:
+ if is_api:
+ msg = m18n.n('log_link_to_log', name=self.name, desc=desc)
+ else:
+ msg = m18n.n('log_help_to_get_log', name=self.name, desc=desc)
+ logger.debug(msg)
+ else:
+ if is_api:
+ msg = "" + m18n.n('log_link_to_failed_log',
+ name=self.name, desc=desc) + ""
+ else:
+ msg = m18n.n('log_help_to_get_failed_log', name=self.name,
+ desc=desc)
+ logger.info(msg)
+ self.flush()
+ return msg
+
+ def __del__(self):
+ """
+ Try to close the unit operation, if it's missing.
+ The missing of the message below could help to see an electrical
+ shortage.
+ """
+ self.error(m18n.n('log_operation_unit_unclosed_properly'))
+
+
+def _get_description_from_name(name):
+ """
+ Return the translated description from the filename
+ """
+
+ parts = name.split("-", 3)
+ try:
+ try:
+ datetime.strptime(" ".join(parts[:2]), "%Y%m%d %H%M%S")
+ except ValueError:
+ key = "log_" + parts[0]
+ args = parts[1:]
+ else:
+ key = "log_" + parts[2]
+ args = parts[3:]
+ return m18n.n(key, *args)
+ except IndexError:
+ return name
diff --git a/src/yunohost/service.py b/src/yunohost/service.py
index d4912f140..66ae837a9 100644
--- a/src/yunohost/service.py
+++ b/src/yunohost/service.py
@@ -40,6 +40,7 @@ from moulinette.core import MoulinetteError
from moulinette.utils import log, filesystem
from yunohost.hook import hook_callback
+from yunohost.log import is_unit_operation
BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
@@ -150,8 +151,8 @@ def service_stop(names):
logs=_get_journalctl_logs(name)))
logger.debug(m18n.n('service_already_stopped', service=name))
-
-def service_enable(names):
+@is_unit_operation()
+def service_enable(operation_logger, names):
"""
Enable one or more services
@@ -159,6 +160,7 @@ def service_enable(names):
names -- Services name to enable
"""
+ operation_logger.start()
if isinstance(names, str):
names = [names]
for name in names:
@@ -343,7 +345,8 @@ def service_log(name, number=50):
return result
-def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
+@is_unit_operation([('names', 'service')])
+def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
"""
Regenerate the configuration file(s) for a service
@@ -376,6 +379,14 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
return pending_conf
+ if not dry_run:
+ operation_logger.related_to = [('service', x) for x in names]
+ if not names:
+ operation_logger.name_parameter_override = 'all'
+ elif len(names) != 1:
+ operation_logger.name_parameter_override = str(len(operation_logger.related_to))+'_services'
+ operation_logger.start()
+
# Clean pending conf directory
if os.path.isdir(PENDING_CONF_DIR):
if not names:
@@ -414,8 +425,13 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Set the processing method
_regen = _process_regen_conf if not dry_run else lambda *a, **k: True
+ operation_logger.related_to = []
+
# Iterate over services and process pending conf
for service, conf_files in _get_pending_conf(names).items():
+ if not dry_run:
+ operation_logger.related_to.append(('service', service))
+
logger.debug(m18n.n(
'service_regenconf_pending_applying' if not dry_run else
'service_regenconf_dry_pending_applying',
@@ -564,6 +580,8 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
hook_callback('conf_regen', names, pre_callback=_pre_call)
+ operation_logger.success()
+
return result
@@ -691,13 +709,21 @@ def _tail(file, n):
value is a tuple in the form ``(lines, has_more)`` where `has_more` is
an indicator that is `True` if there are more lines in the file.
+ This function works even with splitted logs (gz compression, log rotate...)
"""
avg_line_length = 74
to_read = n
try:
- with open(file, 'r') as f:
- while 1:
+ if file.endswith(".gz"):
+ import gzip
+ f = gzip.open(file)
+ lines = f.read().splitlines()
+ else:
+ f = open(file)
+ pos = 1
+ lines = []
+ while len(lines) < to_read and pos > 0:
try:
f.seek(-(avg_line_length * to_read), 2)
except IOError:
@@ -708,15 +734,48 @@ def _tail(file, n):
pos = f.tell()
lines = f.read().splitlines()
- if len(lines) >= to_read or pos == 0:
+ if len(lines) >= to_read:
return lines[-to_read:]
avg_line_length *= 1.3
+ f.close()
except IOError as e:
logger.warning("Error while tailing file '%s': %s", file, e, exc_info=1)
return []
+ if len(lines) < to_read:
+ previous_log_file = _find_previous_log_file(file)
+ if previous_log_file is not None:
+ lines = _tail(previous_log_file, to_read - len(lines)) + lines
+
+ return lines
+
+
+def _find_previous_log_file(file):
+ """
+ Find the previous log file
+ """
+ import re
+
+ splitext = os.path.splitext(file)
+ if splitext[1] == '.gz':
+ file = splitext[0]
+ splitext = os.path.splitext(file)
+ ext = splitext[1]
+ i = re.findall(r'\.(\d+)', ext)
+ i = int(i[0]) + 1 if len(i) > 0 else 1
+
+ previous_file = file if i == 1 else splitext[0]
+ previous_file = previous_file + '.%d' % (i)
+ if os.path.exists(previous_file):
+ return previous_file
+
+ previous_file = previous_file + ".gz"
+ if os.path.exists(previous_file):
+ return previous_file
+
+ return None
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
"""Compare two files and return the differences
diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py
index 935f8b22d..f9ee14994 100644
--- a/src/yunohost/tools.py
+++ b/src/yunohost/tools.py
@@ -52,6 +52,7 @@ from yunohost.service import service_status, service_regen_conf, service_log, se
from yunohost.monitor import monitor_disk, monitor_system
from yunohost.utils.packages import ynh_packages_version
from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation, OperationLogger
# FIXME this is a duplicate from apps.py
APPS_SETTING_PATH = '/etc/yunohost/apps/'
@@ -138,7 +139,8 @@ def tools_adminpw(auth, new_password):
logger.success(m18n.n('admin_password_changed'))
-def tools_maindomain(auth, new_domain=None):
+@is_unit_operation()
+def tools_maindomain(operation_logger, auth, new_domain=None):
"""
Check the current main domain, or change it
@@ -155,6 +157,9 @@ def tools_maindomain(auth, new_domain=None):
if new_domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
+ operation_logger.related_to.append(('domain', new_domain))
+ operation_logger.start()
+
# Apply changes to ssl certs
ssl_key = "/etc/ssl/private/yunohost_key.pem"
ssl_crt = "/etc/ssl/private/yunohost_crt.pem"
@@ -244,7 +249,8 @@ def _is_inside_container():
return out.split()[0] in container
-def tools_postinstall(domain, password, ignore_dyndns=False):
+@is_unit_operation()
+def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False):
"""
YunoHost post-install
@@ -293,6 +299,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
else:
dyndns = False
+ operation_logger.start()
logger.info(m18n.n('yunohost_installing'))
service_regen_conf(['nslcd', 'nsswitch'], force=True)
@@ -468,7 +475,8 @@ def tools_update(ignore_apps=False, ignore_packages=False):
return {'packages': packages, 'apps': apps}
-def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
+@is_unit_operation()
+def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
@@ -509,6 +517,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
if cache.get_changes():
logger.info(m18n.n('upgrading_packages'))
+ operation_logger.start()
try:
# Apply APT changes
# TODO: Logs output for the API
@@ -518,11 +527,14 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
failure = True
logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
+ operation_logger.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
+ operation_logger.success()
else:
logger.info(m18n.n('packages_no_upgrade'))
+
if not ignore_apps:
try:
app_upgrade(auth)
@@ -703,7 +715,8 @@ def tools_port_available(port):
return False
-def tools_shutdown(force=False):
+@is_unit_operation()
+def tools_shutdown(operation_logger, force=False):
shutdown = force
if not shutdown:
try:
@@ -716,11 +729,13 @@ def tools_shutdown(force=False):
shutdown = True
if shutdown:
+ operation_logger.start()
logger.warn(m18n.n('server_shutdown'))
subprocess.check_call(['systemctl', 'poweroff'])
-def tools_reboot(force=False):
+@is_unit_operation()
+def tools_reboot(operation_logger, force=False):
reboot = force
if not reboot:
try:
@@ -732,6 +747,7 @@ def tools_reboot(force=False):
if i.lower() == 'y' or i.lower() == 'yes':
reboot = True
if reboot:
+ operation_logger.start()
logger.warn(m18n.n('server_reboot'))
subprocess.check_call(['systemctl', 'reboot'])
@@ -852,12 +868,18 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
# effectively run selected migrations
for migration in migrations:
+
+ # Start register change on system
+ operation_logger= OperationLogger('tools_migrations_migrate_' + mode)
+ operation_logger.start()
+
if not skip:
logger.warn(m18n.n('migrations_show_currently_running_migration',
number=migration.number, name=migration.name))
try:
+ migration.operation_logger = operation_logger
if mode == "forward":
migration.migrate()
elif mode == "backward":
@@ -867,11 +889,12 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
except Exception as e:
# migration failed, let's stop here but still update state because
# we managed to run the previous ones
- logger.error(m18n.n('migrations_migration_has_failed',
+ msg = m18n.n('migrations_migration_has_failed',
exception=e,
number=migration.number,
- name=migration.name),
- exc_info=1)
+ name=migration.name)
+ logger.error(msg, exc_info=1)
+ operation_logger.error(msg)
break
else: # if skip
@@ -885,6 +908,8 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
"name": migration.name
}
+ operation_logger.success()
+
# special case where we want to go back from the start
if target == 0:
state["last_run_migration"] = None
diff --git a/src/yunohost/user.py b/src/yunohost/user.py
index bbcecc8d6..48065f70a 100644
--- a/src/yunohost/user.py
+++ b/src/yunohost/user.py
@@ -37,6 +37,7 @@ from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from yunohost.service import service_status
+from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.user')
@@ -97,7 +98,8 @@ def user_list(auth, fields=None):
return {'users': users}
-def user_create(auth, username, firstname, lastname, mail, password,
+@is_unit_operation([('username', 'user')])
+def user_create(operation_logger, auth, username, firstname, lastname, mail, password,
mailbox_quota="0"):
"""
Create user
@@ -132,6 +134,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
m18n.n('mail_domain_unknown',
domain=mail.split("@")[1]))
+ operation_logger.start()
+
# Get random UID/GID
all_uid = {x.pw_uid for x in pwd.getpwall()}
all_gid = {x.pw_gid for x in pwd.getpwall()}
@@ -217,7 +221,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
raise MoulinetteError(169, m18n.n('user_creation_failed'))
-def user_delete(auth, username, purge=False):
+@is_unit_operation([('username', 'user')])
+def user_delete(operation_logger, auth, username, purge=False):
"""
Delete user
@@ -229,6 +234,7 @@ def user_delete(auth, username, purge=False):
from yunohost.app import app_ssowatconf
from yunohost.hook import hook_callback
+ operation_logger.start()
if auth.remove('uid=%s,ou=users' % username):
# Invalidate passwd to take user deletion into account
subprocess.call(['nscd', '-i', 'passwd'])
@@ -252,7 +258,8 @@ def user_delete(auth, username, purge=False):
logger.success(m18n.n('user_deleted'))
-def user_update(auth, username, firstname=None, lastname=None, mail=None,
+@is_unit_operation([('username', 'user')], exclude=['auth', 'change_password'])
+def user_update(operation_logger, auth, username, firstname=None, lastname=None, mail=None,
change_password=None, add_mailforward=None, remove_mailforward=None,
add_mailalias=None, remove_mailalias=None, mailbox_quota=None):
"""
@@ -353,6 +360,8 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
if mailbox_quota is not None:
new_attr_dict['mailuserquota'] = mailbox_quota
+ operation_logger.start()
+
if auth.update('uid=%s,ou=users' % username, new_attr_dict):
logger.success(m18n.n('user_updated'))
app_ssowatconf(auth)
diff --git a/src/yunohost/utils/yunopaste.py b/src/yunohost/utils/yunopaste.py
new file mode 100644
index 000000000..2b53062d1
--- /dev/null
+++ b/src/yunohost/utils/yunopaste.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+import errno
+
+from moulinette.core import MoulinetteError
+
+def yunopaste(data):
+
+ paste_server = "https://paste.yunohost.org"
+
+ try:
+ r = requests.post("%s/documents" % paste_server, data=data, timeout=30)
+ except Exception as e:
+ raise MoulinetteError(errno.EIO,
+ "Something wrong happened while trying to paste data on paste.yunohost.org : %s" % str(e))
+
+ if r.status_code != 200:
+ raise MoulinetteError(errno.EIO,
+ "Something wrong happened while trying to paste data on paste.yunohost.org : %s, %s" % (r.status_code, r.text))
+
+ try:
+ url = json.loads(r.text)["key"]
+ except:
+ raise MoulinetteError(errno.EIO,
+ "Uhoh, couldn't parse the answer from paste.yunohost.org : %s" % r.text)
+
+ return "%s/raw/%s" % (paste_server, url)