YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'stretch-unstable' into app-stdinfo

Browse source
This commit is contained in:
Alexandre Aubin 2018-08-23 21:25:29 +02:00 • committed by GitHub
commit 0eaee300c6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 955 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
data/actionsmap/yunohost.yml
View file

@ -1710,3 +1710,39 @@ hook:
-d:
full: --chdir
help: The directory from where the script will be executed
#############################
# Log #
#############################
log:
category_help: Manage debug logs
actions:
### log_list()
list:
action_help: List logs
api: GET /logs
arguments:
category:
help: Log category to display (default operations), could be operation, history, package, system, access, service or app
nargs: "*"
-l:
full: --limit
help: Maximum number of logs
type: int
### log_display()
display:
action_help: Display a log content
api: GET /logs/display
arguments:
path:
help: Log file which to display the content
-n:
full: --number
help: Number of lines to display
default: 50
type: int
--share:
help: Share the full log using yunopaste
action: store_true

17
data/helpers.d/utils
View file

@ -257,3 +257,20 @@ ynh_local_curl () {
# Curl the URL
curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url"
}
# Render templates with Jinja2
#
# Attention : Variables should be exported before calling this helper to be
# accessible inside templates.
#
# usage: ynh_render_template some_template output_path
# | arg: some_template - Template file to be rendered
# | arg: output_path - The path where the output will be redirected to
ynh_render_template() {
local template_path=$1
local output_path=$2
# Taken from https://stackoverflow.com/a/35009576
python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
jinja2.Template(sys.stdin.read()
).render(os.environ));' < $template_path > $output_path
}

13
data/hooks/conf_regen/19-postfix
View file

@ -10,15 +10,25 @@ do_pre_regen() {
postfix_dir="${pending_dir}/etc/postfix"
mkdir -p "$postfix_dir"
default_dir="${pending_dir}/etc/default/"
mkdir -p "$default_dir"
# install plain conf files
cp plain/* "$postfix_dir"
# prepare main.cf conf file
main_domain=$(cat /etc/yunohost/current_host)
domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ')
cat main.cf \
| sed "s/{{ main_domain }}/${main_domain}/g" \
> "${postfix_dir}/main.cf"
cat postsrsd \
| sed "s/{{ main_domain }}/${main_domain}/g" \
| sed "s/{{ domain_list }}/${domain_list}/g" \
> "${default_dir}/postsrsd"
# adapt it for IPv4-only hosts
if [ ! -f /proc/net/if_inet6 ]; then
sed -i \
@ -34,7 +44,8 @@ do_post_regen() {
regen_conf_files=$1
[[ -z "$regen_conf_files" ]] \
|| sudo service postfix restart
|| { sudo service postfix restart && sudo service postsrsd restart; }
}
FORCE=${2:-0}

4
data/templates/postfix/main.cf
View file

@ -137,8 +137,10 @@ smtpd_recipient_restrictions =
permit
# SRS
sender_canonical_maps = regexp:/etc/postfix/sender_canonical
sender_canonical_maps = tcp:localhost:10001
sender_canonical_classes = envelope_sender
recipient_canonical_maps = tcp:localhost:10002
recipient_canonical_classes= envelope_recipient,header_recipient
# Ignore some headers
smtp_header_checks = regexp:/etc/postfix/header_checks

43
data/templates/postfix/postsrsd Normal file
View file

@ -0,0 +1,43 @@
# Default settings for postsrsd
# Local domain name.
# Addresses are rewritten to originate from this domain. The default value
# is taken from `postconf -h mydomain` and probably okay.
#
SRS_DOMAIN={{ main_domain }}
# Exclude additional domains.
# You may list domains which shall not be subjected to address rewriting.
# If a domain name starts with a dot, it matches all subdomains, but not
# the domain itself. Separate multiple domains by space or comma.
# We have to put some "dummy" stuff at start and end... see this comment :
# https://github.com/roehling/postsrsd/issues/64#issuecomment-284003762
SRS_EXCLUDE_DOMAINS=dummy {{ domain_list }} dummy
# First separator character after SRS0 or SRS1.
# Can be one of: -+=
SRS_SEPARATOR==
# Secret key to sign rewritten addresses.
# When postsrsd is installed for the first time, a random secret is generated
# and stored in /etc/postsrsd.secret. For most installations, that's just fine.
#
SRS_SECRET=/etc/postsrsd.secret
# Local ports for TCP list.
# These ports are used to bind the TCP list for postfix. If you change
# these, you have to modify the postfix settings accordingly. The ports
# are bound to the loopback interface, and should never be exposed on
# the internet.
#
SRS_FORWARD_PORT=10001
SRS_REVERSE_PORT=10002
# Drop root privileges and run as another user after initialization.
# This is highly recommended as postsrsd handles untrusted input.
#
RUN_AS=postsrsd
# Jail daemon in chroot environment
CHROOT=/var/lib/postsrsd

4
debian/control vendored
View file

@ -12,13 +12,13 @@ Architecture: all
Depends: ${python:Depends}, ${misc:Depends}
, moulinette (>= 2.7.1), ssowat (>= 2.7.1)
, python-psutil, python-requests, python-dnspython, python-openssl
, python-apt, python-miniupnpc, python-dbus
, python-apt, python-miniupnpc, python-dbus, python-jinja2
, glances
, dnsutils, bind9utils, unzip, git, curl, cron, wget
, ca-certificates, netcat-openbsd, iproute
, mariadb-server, php-mysql | php-mysqlnd
, slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd
, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils
, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils, postsrsd
, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved
, dovecot-antispam, fail2ban
, nginx-extras (>=1.6.2), php-fpm, php-ldap, php-intl

43
locales/en.json
View file

@ -206,6 +206,49 @@
"invalid_url_format": "Invalid URL format",
"ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it",
"iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it",
"log_corrupted_md_file": "The yaml metadata file associated with logs is corrupted : '{md_file}'",
"log_category_404": "The log category '{category}' does not exist",
"log_link_to_log": "Full log of this operation: '<a href=\"#/tools/logs/{name}\" style=\"text-decoration:underline\">{desc}</a>'",
"log_help_to_get_log": "To view the log of the operation '{desc}', use the command 'yunohost log display {name}'",
"log_link_to_failed_log": "The operation '{desc}' has failed ! To get help, please <a href=\"#/tools/logs/{name}\">provide the full log of this operation</a>",
"log_help_to_get_failed_log": "The operation '{desc}' has failed ! To get help, please share the full log of this operation using the command 'yunohost log display {name} --share'",
"log_category_404": "The log category '{category}' does not exist",
"log_does_exists": "There is not operation log with the name '{log}', use 'yunohost log list to see all available operation logs'",
"log_operation_unit_unclosed_properly": "Operation unit has not been closed properly",
"log_app_addaccess": "Add access to '{}'",
"log_app_removeaccess": "Remove access to '{}'",
"log_app_clearaccess": "Remove all access to '{}'",
"log_app_fetchlist": "Add an application list",
"log_app_removelist": "Remove an application list",
"log_app_change_url": "Change the url of '{}' application",
"log_app_install": "Install '{}' application",
"log_app_remove": "Remove '{}' application",
"log_app_upgrade": "Upgrade '{}' application",
"log_app_makedefault": "Make '{}' as default application",
"log_available_on_yunopaste": "This log is now available via {url}",
"log_backup_restore_system": "Restore system from a backup archive",
"log_backup_restore_app": "Restore '{}' from a backup archive",
"log_remove_on_failed_restore": "Remove '{}' after a failed restore from a backup archive",
"log_remove_on_failed_install": "Remove '{}' after a failed installation",
"log_domain_add": "Add '{}' domain into system configuration",
"log_domain_remove": "Remove '{}' domain from system configuration",
"log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'",
"log_dyndns_update": "Update the ip associated with your YunoHost subdomain '{}'",
"log_letsencrypt_cert_install": "Install Let's encrypt certificate on '{}' domain",
"log_selfsigned_cert_install": "Install self signed certificate on '{}' domain",
"log_letsencrypt_cert_renew": "Renew '{}' Let's encrypt certificate",
"log_service_enable": "Enable '{}' service",
"log_service_regen_conf": "Regenerate system configurations '{}'",
"log_user_create": "Add '{}' user",
"log_user_delete": "Delete '{}' user",
"log_user_update": "Update information of '{}' user",
"log_tools_maindomain": "Make '{}' as main domain",
"log_tools_migrations_migrate_forward": "Migrate forward",
"log_tools_migrations_migrate_backward": "Migrate backward",
"log_tools_postinstall": "Postinstall your YunoHost server",
"log_tools_upgrade": "Upgrade debian packages",
"log_tools_shutdown": "Shutdown your server",
"log_tools_reboot": "Reboot your server",
"ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user",
"ldap_initialized": "LDAP has been initialized",
"license_undefined": "undefined",

119
src/yunohost/app.py
View file

@ -44,6 +44,7 @@ from moulinette.utils.filesystem import read_json
from yunohost.service import service_log, _run_service_command
from yunohost.utils import packages
from yunohost.log import is_unit_operation, OperationLogger
logger = getActionLogger('yunohost.app')
@ -109,10 +110,13 @@ def app_fetchlist(url=None, name=None):
# the fetch only this list
if url is not None:
if name:
operation_logger = OperationLogger('app_fetchlist')
operation_logger.start()
_register_new_appslist(url, name)
# Refresh the appslists dict
appslists = _read_appslist_list()
appslists_to_be_fetched = [name]
operation_logger.success()
else:
raise MoulinetteError(errno.EINVAL,
m18n.n('custom_appslist_name_required'))
@ -188,7 +192,8 @@ def app_fetchlist(url=None, name=None):
_write_appslist_list(appslists)
def app_removelist(name):
@is_unit_operation()
def app_removelist(operation_logger, name):
"""
Remove list from the repositories
@ -202,6 +207,8 @@ def app_removelist(name):
if name not in appslists.keys():
raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name))
operation_logger.start()
# Remove json
json_path = '%s/%s.json' % (REPO_PATH, name)
if os.path.exists(json_path):
@ -425,7 +432,8 @@ def app_map(app=None, raw=False, user=None):
return result
def app_change_url(auth, app, domain, path):
@is_unit_operation()
def app_change_url(operation_logger, auth, app, domain, path):
"""
Modify the URL at which an application is installed.
@ -482,6 +490,11 @@ def app_change_url(auth, app, domain, path):
env_dict["YNH_APP_NEW_DOMAIN"] = domain
env_dict["YNH_APP_NEW_PATH"] = path.rstrip("/")
if domain != old_domain:
operation_logger.related_to.append(('domain', old_domain))
operation_logger.extra.update({'env': env_dict})
operation_logger.start()
if os.path.exists(os.path.join(APP_TMP_FOLDER, "scripts")):
shutil.rmtree(os.path.join(APP_TMP_FOLDER, "scripts"))
@ -499,16 +512,16 @@ def app_change_url(auth, app, domain, path):
os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts")))
os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url")))
# XXX journal
if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'),
args=args_list, env=env_dict, user="root") != 0:
logger.error("Failed to change '%s' url." % app)
msg = "Failed to change '%s' url." % app
logger.error(msg)
operation_logger.error(msg)
# restore values modified by app_checkurl
# see begining of the function
app_setting(app, "domain", value=old_domain)
app_setting(app, "path", value=old_path)
return
# this should idealy be done in the change_url script but let's avoid common mistakes
@ -546,7 +559,6 @@ def app_upgrade(auth, app=[], url=None, file=None):
"""
from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
# Retrieve interface
is_api = msettings.get('interface') == 'api'
@ -617,6 +629,11 @@ def app_upgrade(auth, app=[], url=None, file=None):
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
# Start register change on system
related_to = [('app', app_instance_name)]
operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict)
operation_logger.start()
# Apply dirty patch to make php5 apps compatible with php7
_patch_php5(extracted_app_folder)
@ -624,7 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None):
os.system('chown -hR admin: %s' % INSTALL_TMP)
if hook_exec(extracted_app_folder + '/scripts/upgrade',
args=args_list, env=env_dict, user="root") != 0:
logger.error(m18n.n('app_upgrade_failed', app=app_instance_name))
msg = m18n.n('app_upgrade_failed', app=app_instance_name)
logger.error(msg)
operation_logger.error(msg)
else:
now = int(time.time())
# TODO: Move install_time away from app_setting
@ -654,7 +673,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
logger.success(m18n.n('app_upgraded', app=app_instance_name))
hook_callback('post_app_upgrade', args=args_list, env=env_dict)
operation_logger.success()
if not upgraded_apps:
raise MoulinetteError(errno.ENODATA, m18n.n('app_no_upgrade'))
@ -668,7 +687,8 @@ def app_upgrade(auth, app=[], url=None, file=None):
return {"log": service_log('yunohost-api', number="100").values()[0]}
def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
@is_unit_operation()
def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on_failure=False):
"""
Install apps
@ -680,6 +700,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
"""
from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
from yunohost.log import OperationLogger
# Fetch or extract sources
try:
@ -737,6 +759,12 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
# Start register change on system
operation_logger.extra.update({'env':env_dict})
operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
operation_logger.related_to.append(("app", app_id))
operation_logger.start()
# Create app directory
app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
if os.path.exists(app_setting_path):
@ -772,13 +800,15 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
try:
install_retcode = hook_exec(
os.path.join(extracted_app_folder, 'scripts/install'),
args=args_list, env=env_dict, user="root")
args=args_list, env=env_dict, user="root"
)
except (KeyboardInterrupt, EOFError):
install_retcode = -1
except:
logger.exception(m18n.n('unexpected_error'))
finally:
if install_retcode != 0:
error_msg = operation_logger.error(m18n.n('unexpected_error'))
if not no_remove_on_failure:
# Setup environment for remove script
env_dict_remove = {}
@ -787,12 +817,22 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
# Execute remove script
operation_logger_remove = OperationLogger('remove_on_failed_install',
[('app', app_instance_name)],
env=env_dict_remove)
operation_logger_remove.start()
remove_retcode = hook_exec(
os.path.join(extracted_app_folder, 'scripts/remove'),
args=[app_instance_name], env=env_dict_remove, user="root")
args=[app_instance_name], env=env_dict_remove, user="root"
)
if remove_retcode != 0:
logger.warning(m18n.n('app_not_properly_removed',
app=app_instance_name))
msg = m18n.n('app_not_properly_removed',
app=app_instance_name)
logger.warning(msg)
operation_logger_remove.error(msg)
else:
operation_logger_remove.success()
# Clean tmp folders
shutil.rmtree(app_setting_path)
@ -801,9 +841,10 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
app_ssowatconf(auth)
if install_retcode == -1:
raise MoulinetteError(errno.EINTR,
m18n.g('operation_interrupted'))
raise MoulinetteError(errno.EIO, m18n.n('installation_failed'))
msg = m18n.n('operation_interrupted') + " " + error_msg
raise MoulinetteError(errno.EINTR, msg)
msg = error_msg
raise MoulinetteError(errno.EIO, msg)
# Clean hooks and add new ones
hook_remove(app_instance_name)
@ -828,7 +869,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False):
hook_callback('post_app_install', args=args_list, env=env_dict)
def app_remove(auth, app):
@is_unit_operation()
def app_remove(operation_logger, auth, app):
"""
Remove app
@ -837,11 +879,12 @@ def app_remove(auth, app):
"""
from yunohost.hook import hook_exec, hook_remove, hook_callback
if not _is_installed(app):
raise MoulinetteError(errno.EINVAL,
m18n.n('app_not_installed', app=app))
operation_logger.start()
app_setting_path = APPS_SETTING_PATH + app
# TODO: display fail messages from script
@ -865,6 +908,8 @@ def app_remove(auth, app):
env_dict["YNH_APP_ID"] = app_id
env_dict["YNH_APP_INSTANCE_NAME"] = app
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
operation_logger.extra.update({'env': env_dict})
operation_logger.flush()
if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list,
env=env_dict, user="root") == 0:
@ -901,6 +946,8 @@ def app_addaccess(auth, apps, users=[]):
apps = [apps, ]
for app in apps:
app_settings = _get_app_settings(app)
if not app_settings:
continue
@ -910,6 +957,12 @@ def app_addaccess(auth, apps, users=[]):
app_settings['mode'] = 'private'
if app_settings['mode'] == 'private':
# Start register change on system
related_to = [('app', app)]
operation_logger= OperationLogger('app_addaccess', related_to)
operation_logger.start()
allowed_users = set()
if 'allowed_users' in app_settings:
allowed_users = set(app_settings['allowed_users'].split(','))
@ -922,11 +975,15 @@ def app_addaccess(auth, apps, users=[]):
logger.warning(m18n.n('user_unknown', user=allowed_user))
continue
allowed_users.add(allowed_user)
operation_logger.related_to.append(('user', allowed_user))
operation_logger.flush()
new_users = ','.join(allowed_users)
app_setting(app, 'allowed_users', new_users)
hook_callback('post_app_addaccess', args=[app, new_users])
operation_logger.success()
result[app] = allowed_users
app_ssowatconf(auth)
@ -963,6 +1020,12 @@ def app_removeaccess(auth, apps, users=[]):
allowed_users = set()
if app_settings.get('skipped_uris', '') != '/':
# Start register change on system
related_to = [('app', app)]
operation_logger= OperationLogger('app_removeaccess', related_to)
operation_logger.start()
if remove_all:
pass
elif 'allowed_users' in app_settings:
@ -972,14 +1035,18 @@ def app_removeaccess(auth, apps, users=[]):
else:
for allowed_user in user_list(auth)['users'].keys():
if allowed_user not in users:
allowed_users.add(allowed_user)
allowed_users.append(allowed_user)
operation_logger.related_to += [ ('user', x) for x in allowed_users ]
operation_logger.flush()
new_users = ','.join(allowed_users)
app_setting(app, 'allowed_users', new_users)
hook_callback('post_app_removeaccess', args=[app, new_users])
result[app] = allowed_users
operation_logger.success()
app_ssowatconf(auth)
return {'allowed_users': result}
@ -1003,6 +1070,11 @@ def app_clearaccess(auth, apps):
if not app_settings:
continue
# Start register change on system
related_to = [('app', app)]
operation_logger= OperationLogger('app_clearaccess', related_to)
operation_logger.start()
if 'mode' in app_settings:
app_setting(app, 'mode', delete=True)
@ -1011,6 +1083,8 @@ def app_clearaccess(auth, apps):
hook_callback('post_app_clearaccess', args=[app])
operation_logger.success()
app_ssowatconf(auth)
@ -1037,7 +1111,8 @@ def app_debug(app):
}
def app_makedefault(auth, app, domain=None):
@is_unit_operation()
def app_makedefault(operation_logger, auth, app, domain=None):
"""
Redirect domain root to an app
@ -1054,9 +1129,11 @@ def app_makedefault(auth, app, domain=None):
if domain is None:
domain = app_domain
operation_logger.related_to.append(('domain',domain))
elif domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
operation_logger.start()
if '/' in app_map(raw=True)[domain]:
raise MoulinetteError(errno.EEXIST,
m18n.n('app_make_default_location_already_used',
@ -2204,7 +2281,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
app_label=app_label,
))
raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', "\n".join(apps=apps)))
raise MoulinetteError(errno.EINVAL, m18n.n('app_location_unavailable', apps="\n".join(apps)))
# (We save this normalized path so that the install script have a
# standard path format to deal with no matter what the user inputted)

41
src/yunohost/backup.py
View file

@ -51,6 +51,7 @@ from yunohost.hook import (
from yunohost.monitor import binary_to_human
from yunohost.tools import tools_postinstall
from yunohost.service import service_regen_conf
from yunohost.log import OperationLogger
BACKUP_PATH = '/home/yunohost.backup'
ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
@ -1172,9 +1173,15 @@ class RestoreManager():
if system_targets == []:
return
# Start register change on system
operation_logger = OperationLogger('backup_restore_system')
operation_logger.start()
logger.debug(m18n.n('restore_running_hooks'))
env_dict = self._get_env_var()
operation_logger.extra['env'] = env_dict
operation_logger.flush()
ret = hook_callback('restore',
system_targets,
args=[self.work_dir],
@ -1184,9 +1191,16 @@ class RestoreManager():
for part in ret['succeed'].keys():
self.targets.set_result("system", part, "Success")
error_part = []
for part in ret['failed'].keys():
logger.error(m18n.n('restore_system_part_failed', part=part))
self.targets.set_result("system", part, "Error")
error_part.append(part)
if ret['failed']:
operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part)))
else:
operation_logger.success()
service_regen_conf()
@ -1234,6 +1248,11 @@ class RestoreManager():
else:
shutil.copy2(s, d)
# Start register change on system
related_to = [('app', app_instance_name)]
operation_logger = OperationLogger('backup_restore_app', related_to)
operation_logger.start()
# Check if the app is not already installed
if _is_installed(app_instance_name):
logger.error(m18n.n('restore_already_installed_app',
@ -1283,6 +1302,9 @@ class RestoreManager():
# Prepare env. var. to pass to script
env_dict = self._get_env_var(app_instance_name)
operation_logger.extra['env'] = env_dict
operation_logger.flush()
# Execute app restore script
hook_exec(restore_script,
args=[app_backup_in_archive, app_instance_name],
@ -1291,8 +1313,10 @@ class RestoreManager():
env=env_dict,
user="root")
except:
logger.exception(m18n.n('restore_app_failed',
app=app_instance_name))
msg = m18n.n('restore_app_failed',app=app_instance_name)
logger.exception(msg)
operation_logger.error(msg)
self.targets.set_result("apps", app_instance_name, "Error")
remove_script = os.path.join(app_scripts_in_archive, 'remove')
@ -1304,12 +1328,20 @@ class RestoreManager():
env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
operation_logger = OperationLogger('remove_on_failed_restore',
[('app', app_instance_name)],
env=env_dict_remove)
operation_logger.start()
# Execute remove script
# TODO: call app_remove instead
if hook_exec(remove_script, args=[app_instance_name],
env=env_dict_remove, user="root") != 0:
logger.warning(m18n.n('app_not_properly_removed',
app=app_instance_name))
msg = m18n.n('app_not_properly_removed', app=app_instance_name)
logger.warning(msg)
operation_logger.error(msg)
else:
operation_logger.success()
# Cleaning app directory
shutil.rmtree(app_settings_new_path, ignore_errors=True)
@ -1317,6 +1349,7 @@ class RestoreManager():
# TODO Cleaning app hooks
else:
self.targets.set_result("apps", app_instance_name, "Success")
operation_logger.success()
finally:
# Cleaning temporary scripts directory
shutil.rmtree(tmp_folder_for_app_restore, ignore_errors=True)

38
src/yunohost/certificate.py
View file

@ -45,7 +45,7 @@ from yunohost.utils.network import get_public_ip
from moulinette import m18n
from yunohost.app import app_ssowatconf
from yunohost.service import _run_service_command, service_regen_conf
from yunohost.log import OperationLogger
logger = getActionLogger('yunohost.certmanager')
@ -160,6 +160,9 @@ def _certificate_install_selfsigned(domain_list, force=False):
for domain in domain_list:
operation_logger = OperationLogger('selfsigned_cert_install', [('domain', domain)],
args={'force': force})
# Paths of files and folder we'll need
date_tag = datetime.now().strftime("%Y%m%d.%H%M%S")
new_cert_folder = "%s/%s-history/%s-selfsigned" % (
@ -182,6 +185,8 @@ def _certificate_install_selfsigned(domain_list, force=False):
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_attempt_to_replace_valid_cert', domain=domain))
operation_logger.start()
# Create output folder for new certificate stuff
os.makedirs(new_cert_folder)
@ -238,9 +243,11 @@ def _certificate_install_selfsigned(domain_list, force=False):
if status and status["CA_type"]["code"] == "self-signed" and status["validity"] > 3648:
logger.success(
m18n.n("certmanager_cert_install_success_selfsigned", domain=domain))
operation_logger.success()
else:
logger.error(
"Installation of self-signed certificate installation for %s failed !", domain)
msg = "Installation of self-signed certificate installation for %s failed !" % (domain)
logger.error(msg)
operation_logger.error(msg)
def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=False, staging=False):
@ -281,6 +288,9 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
# Actual install steps
for domain in domain_list:
operation_logger = OperationLogger('letsencrypt_cert_install', [('domain', domain)],
args={'force': force, 'no_checks': no_checks,
'staging': staging})
logger.info(
"Now attempting install of certificate for domain %s!", domain)
@ -288,6 +298,8 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
if not no_checks:
_check_domain_is_ready_for_ACME(domain)
operation_logger.start()
_configure_for_acme_challenge(auth, domain)
_fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
_install_cron()
@ -295,10 +307,12 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
logger.success(
m18n.n("certmanager_cert_install_success", domain=domain))
operation_logger.success()
except Exception as e:
_display_debug_information(domain)
logger.error("Certificate installation for %s failed !\nException: %s", domain, e)
msg = "Certificate installation for %s failed !\nException: %s" % (domain, e)
logger.error(msg)
operation_logger.error(msg)
def certificate_renew(auth, domain_list, force=False, no_checks=False, email=False, staging=False):
"""
@ -376,6 +390,11 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
# Actual renew steps
for domain in domain_list:
operation_logger = OperationLogger('letsencrypt_cert_renew', [('domain', domain)],
args={'force': force, 'no_checks': no_checks,
'staging': staging, 'email': email})
logger.info(
"Now attempting renewing of certificate for domain %s !", domain)
@ -383,17 +402,23 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
if not no_checks:
_check_domain_is_ready_for_ACME(domain)
operation_logger.start()
_fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
logger.success(
m18n.n("certmanager_cert_renew_success", domain=domain))
operation_logger.success()
except Exception as e:
import traceback
from StringIO import StringIO
stack = StringIO()
traceback.print_exc(file=stack)
logger.error("Certificate renewing for %s failed !", domain)
msg = "Certificate renewing for %s failed !" % (domain)
logger.error(msg)
operation_logger.error(msg)
logger.error(stack.getvalue())
logger.error(str(e))
@ -401,7 +426,6 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
logger.error("Sending email with details to root ...")
_email_renewing_failed(domain, e, stack.getvalue())
###############################################################################
# Back-end stuff #
###############################################################################

21
src/yunohost/domain.py
View file

@ -37,6 +37,7 @@ import yunohost.certificate
from yunohost.service import service_regen_conf
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.domain')
@ -61,7 +62,8 @@ def domain_list(auth):
return {'domains': result_list}
def domain_add(auth, domain, dyndns=False):
@is_unit_operation()
def domain_add(operation_logger, auth, domain, dyndns=False):
"""
Create a custom domain
@ -78,6 +80,8 @@ def domain_add(auth, domain, dyndns=False):
except MoulinetteError:
raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))
operation_logger.start()
# DynDNS domain
if dyndns:
@ -110,23 +114,27 @@ def domain_add(auth, domain, dyndns=False):
# Don't regen these conf if we're still in postinstall
if os.path.exists('/etc/yunohost/installed'):
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
app_ssowatconf(auth)
except:
except Exception, e:
from sys import exc_info;
t, v, tb = exc_info()
# Force domain removal silently
try:
domain_remove(auth, domain, True)
except:
pass
raise
raise t, v, tb
hook_callback('post_domain_add', args=[domain])
logger.success(m18n.n('domain_created'))
def domain_remove(auth, domain, force=False):
@is_unit_operation()
def domain_remove(operation_logger, auth, domain, force=False):
"""
Delete domains
@ -157,12 +165,13 @@ def domain_remove(auth, domain, force=False):
raise MoulinetteError(errno.EPERM,
m18n.n('domain_uninstall_app_first'))
operation_logger.start()
if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
os.system('rm -rf /etc/yunohost/certs/%s' % domain)
else:
raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
app_ssowatconf(auth)
hook_callback('post_domain_remove', args=[domain])

24
src/yunohost/dyndns.py
View file

@ -40,6 +40,7 @@ from moulinette.utils.network import download_json
from yunohost.domain import _get_maindomain, _build_dns_conf
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.dyndns')
@ -112,7 +113,8 @@ def _dyndns_available(provider, domain):
return r == u"Domain %s is available" % domain
def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None):
@is_unit_operation()
def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", domain=None, key=None):
"""
Subscribe to a DynDNS service
@ -124,6 +126,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
"""
if domain is None:
domain = _get_maindomain()
operation_logger.related_to.append(('domain', domain))
# Verify if domain is provided by subscribe_host
if not _dyndns_provides(subscribe_host, domain):
@ -136,6 +139,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
raise MoulinetteError(errno.ENOENT,
m18n.n('dyndns_unavailable', domain=domain))
operation_logger.start()
if key is None:
if len(glob.glob('/etc/yunohost/dyndns/*.key')) == 0:
if not os.path.exists('/etc/yunohost/dyndns'):
@ -170,7 +175,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
dyndns_installcron()
def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
@is_unit_operation()
def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, key=None,
ipv4=None, ipv6=None):
"""
Update IP on DynDNS platform
@ -217,13 +223,17 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
if domain is None:
(domain, key) = _guess_current_dyndns_domain(dyn_host)
# If key is not given, pick the first file we find with the domain given
elif key is None:
keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
else:
if key is None:
keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
if not keys:
raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
if not keys:
raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
key = keys[0]
key = keys[0]
operation_logger.related_to.append(('domain', domain))
operation_logger.start()
# This mean that hmac-md5 is used
# (Re?)Trigger the migration to sha256 and return immediately.

462
src/yunohost/log.py Normal file
View file

@ -0,0 +1,462 @@
# -*- coding: utf-8 -*-
""" License
Copyright (C) 2018 YunoHost
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, see http://www.gnu.org/licenses
"""
""" yunohost_log.py
Manage debug logs
"""
import os
import yaml
import errno
import collections
from datetime import datetime
from logging import FileHandler, getLogger, Formatter
from sys import exc_info
from moulinette import m18n, msettings
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file
CATEGORIES_PATH = '/var/log/yunohost/categories/'
OPERATIONS_PATH = '/var/log/yunohost/categories/operation/'
CATEGORIES = ['operation', 'history', 'package', 'system', 'access', 'service',
'app']
METADATA_FILE_EXT = '.yml'
LOG_FILE_EXT = '.log'
RELATED_CATEGORIES = ['app', 'domain', 'service', 'user']
logger = getActionLogger('yunohost.log')
def log_list(category=[], limit=None):
"""
List available logs
Keyword argument:
limit -- Maximum number of logs
"""
categories = category
is_api = msettings.get('interface') == 'api'
# In cli we just display `operation` logs by default
if not categories:
categories = ["operation"] if not is_api else CATEGORIES
result = collections.OrderedDict()
for category in categories:
result[category] = []
category_path = os.path.join(CATEGORIES_PATH, category)
if not os.path.exists(category_path):
logger.debug(m18n.n('log_category_404', category=category))
continue
logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
os.listdir(category_path))
logs = reversed(sorted(logs))
if limit is not None:
logs = logs[:limit]
for log in logs:
base_filename = log[:-len(METADATA_FILE_EXT)]
md_filename = log
md_path = os.path.join(category_path, md_filename)
log = base_filename.split("-")
entry = {
"name": base_filename,
"path": md_path,
}
entry["description"] = _get_description_from_name(base_filename)
try:
log_datetime = datetime.strptime(" ".join(log[:2]),
"%Y%m%d %H%M%S")
except ValueError:
pass
else:
entry["started_at"] = log_datetime
result[category].append(entry)
# Reverse the order of log when in cli, more comfortable to read (avoid
# unecessary scrolling)
if not is_api:
for category in result:
result[category] = list(reversed(result[category]))
return result
def log_display(path, number=50, share=False):
"""
Display a log file enriched with metadata if any.
If the file_name is not an absolute path, it will try to search the file in
the unit operations log path (see OPERATIONS_PATH).
Argument:
file_name
number
share
"""
# Normalize log/metadata paths and filenames
abs_path = path
log_path = None
if not path.startswith('/'):
for category in CATEGORIES:
abs_path = os.path.join(CATEGORIES_PATH, category, path)
if os.path.exists(abs_path) or os.path.exists(abs_path + METADATA_FILE_EXT):
break
if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT):
log_path = abs_path
if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT):
base_path = ''.join(os.path.splitext(abs_path)[:-1])
else:
base_path = abs_path
base_filename = os.path.basename(base_path)
md_path = base_path + METADATA_FILE_EXT
if log_path is None:
log_path = base_path + LOG_FILE_EXT
if not os.path.exists(md_path) and not os.path.exists(log_path):
raise MoulinetteError(errno.EINVAL,
m18n.n('log_does_exists', log=path))
infos = {}
# If it's a unit operation, display the name and the description
if base_path.startswith(CATEGORIES_PATH):
infos["description"] = _get_description_from_name(base_filename)
infos['name'] = base_filename
if share:
from yunohost.utils.yunopaste import yunopaste
content = ""
if os.path.exists(md_path):
content += read_file(md_path)
content += "\n============\n\n"
if os.path.exists(log_path):
content += read_file(log_path)
url = yunopaste(content)
logger.info(m18n.n("log_available_on_yunopaste", url=url))
if msettings.get('interface') == 'api':
return {"url": url}
else:
return
# Display metadata if exist
if os.path.exists(md_path):
with open(md_path, "r") as md_file:
try:
metadata = yaml.safe_load(md_file)
infos['metadata_path'] = md_path
infos['metadata'] = metadata
if 'log_path' in metadata:
log_path = metadata['log_path']
except yaml.YAMLError:
error = m18n.n('log_corrupted_md_file', file=md_path)
if os.path.exists(log_path):
logger.warning(error)
else:
raise MoulinetteError(errno.EINVAL, error)
# Display logs if exist
if os.path.exists(log_path):
from yunohost.service import _tail
logs = _tail(log_path, int(number))
infos['log_path'] = log_path
infos['logs'] = logs
return infos
def is_unit_operation(entities=['app', 'domain', 'service', 'user'],
exclude=['auth', 'password'], operation_key=None):
"""
Configure quickly a unit operation
This decorator help you to configure the record of a unit operations.
Argument:
entities A list of entity types related to the unit operation. The entity
type is searched inside argument's names of the decorated function. If
something match, the argument value is added as related entity. If the
argument name is different you can specify it with a tuple
(argname, entity_type) instead of just put the entity type.
exclude Remove some arguments from the context. By default, arguments
called 'password' and 'auth' are removed. If an argument is an object, you
need to exclude it or create manually the unit operation without this
decorator.
operation_key A key to describe the unit operation log used to create the
filename and search a translation. Please ensure that this key prefixed by
'log_' is present in locales/en.json otherwise it won't be translatable.
"""
def decorate(func):
def func_wrapper(*args, **kwargs):
op_key = operation_key
if op_key is None:
op_key = func.__name__
# If the function is called directly from an other part of the code
# and not by the moulinette framework, we need to complete kwargs
# dictionnary with the args list.
# Indeed, we use convention naming in this decorator and we need to
# know name of each args (so we need to use kwargs instead of args)
if len(args) > 0:
from inspect import getargspec
keys = getargspec(func).args
if 'operation_logger' in keys:
keys.remove('operation_logger')
for k, arg in enumerate(args):
kwargs[keys[k]] = arg
args = ()
# Search related entity in arguments of the decorated function
related_to = []
for entity in entities:
if isinstance(entity, tuple):
entity_type = entity[1]
entity = entity[0]
else:
entity_type = entity
if entity in kwargs and kwargs[entity] is not None:
if isinstance(kwargs[entity], basestring):
related_to.append((entity_type, kwargs[entity]))
else:
for x in kwargs[entity]:
related_to.append((entity_type, x))
context = kwargs.copy()
# Exclude unappropriate data from the context
for field in exclude:
if field in context:
context.pop(field, None)
operation_logger = OperationLogger(op_key, related_to, args=context)
try:
# Start the actual function, and give the unit operation
# in argument to let the developper start the record itself
args = (operation_logger,) + args
result = func(*args, **kwargs)
except Exception as e:
operation_logger.error(e)
raise
else:
operation_logger.success()
return result
return func_wrapper
return decorate
class OperationLogger(object):
"""
Instances of this class represents unit operation done on the ynh instance.
Each time an action of the yunohost cli/api change the system, one or
several unit operations should be registered.
This class record logs and metadata like context or start time/end time.
"""
def __init__(self, operation, related_to=None, **kwargs):
# TODO add a way to not save password on app installation
self.operation = operation
self.related_to = related_to
self.extra = kwargs
self.started_at = None
self.ended_at = None
self.logger = None
self._name = None
self.path = OPERATIONS_PATH
if not os.path.exists(self.path):
os.makedirs(self.path)
def start(self):
"""
Start to record logs that change the system
Until this start method is run, no unit operation will be registered.
"""
if self.started_at is None:
self.started_at = datetime.now()
self.flush()
self._register_log()
def _register_log(self):
"""
Register log with a handler connected on log system
"""
# TODO add a way to not save password on app installation
filename = os.path.join(self.path, self.name + LOG_FILE_EXT)
self.file_handler = FileHandler(filename)
self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
# Listen to the root logger
self.logger = getLogger('yunohost')
self.logger.addHandler(self.file_handler)
def flush(self):
"""
Write or rewrite the metadata file with all metadata known
"""
filename = os.path.join(self.path, self.name + METADATA_FILE_EXT)
with open(filename, 'w') as outfile:
yaml.safe_dump(self.metadata, outfile, default_flow_style=False)
@property
def name(self):
"""
Name of the operation
This name is used as filename, so don't use space
"""
if self._name is not None:
return self._name
name = [self.started_at.strftime("%Y%m%d-%H%M%S")]
name += [self.operation]
if hasattr(self, "name_parameter_override"):
# This is for special cases where the operation is not really
# unitary. For instance, the regen conf cannot be logged "per
# service" because of the way it's built
name.append(self.name_parameter_override)
elif self.related_to:
# We use the name of the first related thing
name.append(self.related_to[0][1])
self._name = '-'.join(name)
return self._name
@property
def metadata(self):
"""
Dictionnary of all metadata collected
"""
data = {
'started_at': self.started_at,
'operation': self.operation,
}
if self.related_to is not None:
data['related_to'] = self.related_to
if self.ended_at is not None:
data['ended_at'] = self.ended_at
data['success'] = self._success
if self.error is not None:
data['error'] = self._error
# TODO: detect if 'extra' erase some key of 'data'
data.update(self.extra)
return data
def success(self):
"""
Declare the success end of the unit operation
"""
self.close()
def error(self, error):
"""
Declare the failure of the unit operation
"""
return self.close(error)
def close(self, error=None):
"""
Close properly the unit operation
"""
if self.ended_at is not None or self.started_at is None:
return
if error is not None and not isinstance(error, basestring):
error = str(error)
self.ended_at = datetime.now()
self._error = error
self._success = error is None
if self.logger is not None:
self.logger.removeHandler(self.file_handler)
is_api = msettings.get('interface') == 'api'
desc = _get_description_from_name(self.name)
if error is None:
if is_api:
msg = m18n.n('log_link_to_log', name=self.name, desc=desc)
else:
msg = m18n.n('log_help_to_get_log', name=self.name, desc=desc)
logger.debug(msg)
else:
if is_api:
msg = "<strong>" + m18n.n('log_link_to_failed_log',
name=self.name, desc=desc) + "</strong>"
else:
msg = m18n.n('log_help_to_get_failed_log', name=self.name,
desc=desc)
logger.info(msg)
self.flush()
return msg
def __del__(self):
"""
Try to close the unit operation, if it's missing.
The missing of the message below could help to see an electrical
shortage.
"""
self.error(m18n.n('log_operation_unit_unclosed_properly'))
def _get_description_from_name(name):
"""
Return the translated description from the filename
"""
parts = name.split("-", 3)
try:
try:
datetime.strptime(" ".join(parts[:2]), "%Y%m%d %H%M%S")
except ValueError:
key = "log_" + parts[0]
args = parts[1:]
else:
key = "log_" + parts[2]
args = parts[3:]
return m18n.n(key, *args)
except IndexError:
return name

71
src/yunohost/service.py
View file

@ -40,6 +40,7 @@ from moulinette.core import MoulinetteError
from moulinette.utils import log, filesystem
from yunohost.hook import hook_callback
from yunohost.log import is_unit_operation
BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
@ -150,8 +151,8 @@ def service_stop(names):
logs=_get_journalctl_logs(name)))
logger.debug(m18n.n('service_already_stopped', service=name))
def service_enable(names):
@is_unit_operation()
def service_enable(operation_logger, names):
"""
Enable one or more services
@ -159,6 +160,7 @@ def service_enable(names):
names -- Services name to enable
"""
operation_logger.start()
if isinstance(names, str):
names = [names]
for name in names:
@ -343,7 +345,8 @@ def service_log(name, number=50):
return result
def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
@is_unit_operation([('names', 'service')])
def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
"""
Regenerate the configuration file(s) for a service
@ -376,6 +379,14 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
return pending_conf
if not dry_run:
operation_logger.related_to = [('service', x) for x in names]
if not names:
operation_logger.name_parameter_override = 'all'
elif len(names) != 1:
operation_logger.name_parameter_override = str(len(operation_logger.related_to))+'_services'
operation_logger.start()
# Clean pending conf directory
if os.path.isdir(PENDING_CONF_DIR):
if not names:
@ -414,8 +425,13 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Set the processing method
_regen = _process_regen_conf if not dry_run else lambda *a, **k: True
operation_logger.related_to = []
# Iterate over services and process pending conf
for service, conf_files in _get_pending_conf(names).items():
if not dry_run:
operation_logger.related_to.append(('service', service))
logger.debug(m18n.n(
'service_regenconf_pending_applying' if not dry_run else
'service_regenconf_dry_pending_applying',
@ -564,6 +580,8 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
hook_callback('conf_regen', names, pre_callback=_pre_call)
operation_logger.success()
return result
@ -691,13 +709,21 @@ def _tail(file, n):
value is a tuple in the form ``(lines, has_more)`` where `has_more` is
an indicator that is `True` if there are more lines in the file.
This function works even with splitted logs (gz compression, log rotate...)
"""
avg_line_length = 74
to_read = n
try:
with open(file, 'r') as f:
while 1:
if file.endswith(".gz"):
import gzip
f = gzip.open(file)
lines = f.read().splitlines()
else:
f = open(file)
pos = 1
lines = []
while len(lines) < to_read and pos > 0:
try:
f.seek(-(avg_line_length * to_read), 2)
except IOError:
@ -708,15 +734,48 @@ def _tail(file, n):
pos = f.tell()
lines = f.read().splitlines()
if len(lines) >= to_read or pos == 0:
if len(lines) >= to_read:
return lines[-to_read:]
avg_line_length *= 1.3
f.close()
except IOError as e:
logger.warning("Error while tailing file '%s': %s", file, e, exc_info=1)
return []
if len(lines) < to_read:
previous_log_file = _find_previous_log_file(file)
if previous_log_file is not None:
lines = _tail(previous_log_file, to_read - len(lines)) + lines
return lines
def _find_previous_log_file(file):
"""
Find the previous log file
"""
import re
splitext = os.path.splitext(file)
if splitext[1] == '.gz':
file = splitext[0]
splitext = os.path.splitext(file)
ext = splitext[1]
i = re.findall(r'\.(\d+)', ext)
i = int(i[0]) + 1 if len(i) > 0 else 1
previous_file = file if i == 1 else splitext[0]
previous_file = previous_file + '.%d' % (i)
if os.path.exists(previous_file):
return previous_file
previous_file = previous_file + ".gz"
if os.path.exists(previous_file):
return previous_file
return None
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
"""Compare two files and return the differences

41
src/yunohost/tools.py
View file

@ -52,6 +52,7 @@ from yunohost.service import service_status, service_regen_conf, service_log, se
from yunohost.monitor import monitor_disk, monitor_system
from yunohost.utils.packages import ynh_packages_version
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation, OperationLogger
# FIXME this is a duplicate from apps.py
APPS_SETTING_PATH = '/etc/yunohost/apps/'
@ -138,7 +139,8 @@ def tools_adminpw(auth, new_password):
logger.success(m18n.n('admin_password_changed'))
def tools_maindomain(auth, new_domain=None):
@is_unit_operation()
def tools_maindomain(operation_logger, auth, new_domain=None):
"""
Check the current main domain, or change it
@ -155,6 +157,9 @@ def tools_maindomain(auth, new_domain=None):
if new_domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
operation_logger.related_to.append(('domain', new_domain))
operation_logger.start()
# Apply changes to ssl certs
ssl_key = "/etc/ssl/private/yunohost_key.pem"
ssl_crt = "/etc/ssl/private/yunohost_crt.pem"
@ -244,7 +249,8 @@ def _is_inside_container():
return out.split()[0] in container
def tools_postinstall(domain, password, ignore_dyndns=False):
@is_unit_operation()
def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False):
"""
YunoHost post-install
@ -293,6 +299,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
else:
dyndns = False
operation_logger.start()
logger.info(m18n.n('yunohost_installing'))
service_regen_conf(['nslcd', 'nsswitch'], force=True)
@ -468,7 +475,8 @@ def tools_update(ignore_apps=False, ignore_packages=False):
return {'packages': packages, 'apps': apps}
def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
@is_unit_operation()
def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
@ -509,6 +517,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
if cache.get_changes():
logger.info(m18n.n('upgrading_packages'))
operation_logger.start()
try:
# Apply APT changes
# TODO: Logs output for the API
@ -518,11 +527,14 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
failure = True
logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
operation_logger.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
operation_logger.success()
else:
logger.info(m18n.n('packages_no_upgrade'))
if not ignore_apps:
try:
app_upgrade(auth)
@ -703,7 +715,8 @@ def tools_port_available(port):
return False
def tools_shutdown(force=False):
@is_unit_operation()
def tools_shutdown(operation_logger, force=False):
shutdown = force
if not shutdown:
try:
@ -716,11 +729,13 @@ def tools_shutdown(force=False):
shutdown = True
if shutdown:
operation_logger.start()
logger.warn(m18n.n('server_shutdown'))
subprocess.check_call(['systemctl', 'poweroff'])
def tools_reboot(force=False):
@is_unit_operation()
def tools_reboot(operation_logger, force=False):
reboot = force
if not reboot:
try:
@ -732,6 +747,7 @@ def tools_reboot(force=False):
if i.lower() == 'y' or i.lower() == 'yes':
reboot = True
if reboot:
operation_logger.start()
logger.warn(m18n.n('server_reboot'))
subprocess.check_call(['systemctl', 'reboot'])
@ -852,12 +868,18 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
# effectively run selected migrations
for migration in migrations:
# Start register change on system
operation_logger= OperationLogger('tools_migrations_migrate_' + mode)
operation_logger.start()
if not skip:
logger.warn(m18n.n('migrations_show_currently_running_migration',
number=migration.number, name=migration.name))
try:
migration.operation_logger = operation_logger
if mode == "forward":
migration.migrate()
elif mode == "backward":
@ -867,11 +889,12 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
except Exception as e:
# migration failed, let's stop here but still update state because
# we managed to run the previous ones
logger.error(m18n.n('migrations_migration_has_failed',
msg = m18n.n('migrations_migration_has_failed',
exception=e,
number=migration.number,
name=migration.name),
exc_info=1)
name=migration.name)
logger.error(msg, exc_info=1)
operation_logger.error(msg)
break
else: # if skip
@ -885,6 +908,8 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai
"name": migration.name
}
operation_logger.success()
# special case where we want to go back from the start
if target == 0:
state["last_run_migration"] = None

15
src/yunohost/user.py
View file

@ -37,6 +37,7 @@ from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from yunohost.service import service_status
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.user')
@ -97,7 +98,8 @@ def user_list(auth, fields=None):
return {'users': users}
def user_create(auth, username, firstname, lastname, mail, password,
@is_unit_operation([('username', 'user')])
def user_create(operation_logger, auth, username, firstname, lastname, mail, password,
mailbox_quota="0"):
"""
Create user
@ -132,6 +134,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
m18n.n('mail_domain_unknown',
domain=mail.split("@")[1]))
operation_logger.start()
# Get random UID/GID
all_uid = {x.pw_uid for x in pwd.getpwall()}
all_gid = {x.pw_gid for x in pwd.getpwall()}
@ -217,7 +221,8 @@ def user_create(auth, username, firstname, lastname, mail, password,
raise MoulinetteError(169, m18n.n('user_creation_failed'))
def user_delete(auth, username, purge=False):
@is_unit_operation([('username', 'user')])
def user_delete(operation_logger, auth, username, purge=False):
"""
Delete user
@ -229,6 +234,7 @@ def user_delete(auth, username, purge=False):
from yunohost.app import app_ssowatconf
from yunohost.hook import hook_callback
operation_logger.start()
if auth.remove('uid=%s,ou=users' % username):
# Invalidate passwd to take user deletion into account
subprocess.call(['nscd', '-i', 'passwd'])
@ -252,7 +258,8 @@ def user_delete(auth, username, purge=False):
logger.success(m18n.n('user_deleted'))
def user_update(auth, username, firstname=None, lastname=None, mail=None,
@is_unit_operation([('username', 'user')], exclude=['auth', 'change_password'])
def user_update(operation_logger, auth, username, firstname=None, lastname=None, mail=None,
change_password=None, add_mailforward=None, remove_mailforward=None,
add_mailalias=None, remove_mailalias=None, mailbox_quota=None):
"""
@ -353,6 +360,8 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
if mailbox_quota is not None:
new_attr_dict['mailuserquota'] = mailbox_quota
operation_logger.start()
if auth.update('uid=%s,ou=users' % username, new_attr_dict):
logger.success(m18n.n('user_updated'))
app_ssowatconf(auth)

29
src/yunohost/utils/yunopaste.py Normal file
View file

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
import requests
import json
import errno
from moulinette.core import MoulinetteError
def yunopaste(data):
paste_server = "https://paste.yunohost.org"
try:
r = requests.post("%s/documents" % paste_server, data=data, timeout=30)
except Exception as e:
raise MoulinetteError(errno.EIO,
"Something wrong happened while trying to paste data on paste.yunohost.org : %s" % str(e))
if r.status_code != 200:
raise MoulinetteError(errno.EIO,
"Something wrong happened while trying to paste data on paste.yunohost.org : %s, %s" % (r.status_code, r.text))
try:
url = json.loads(r.text)["key"]
except:
raise MoulinetteError(errno.EIO,
"Uhoh, couldn't parse the answer from paste.yunohost.org : %s" % r.text)
return "%s/raw/%s" % (paste_server, url)