[enh] Add related info

This commit is contained in:
ljf 2018-04-12 21:22:17 +02:00
parent b81d89f93a
commit 79ee0396d0
8 changed files with 200 additions and 121 deletions

View file

@ -206,6 +206,7 @@
"ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it",
"iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it",
"log_does_exists": "There is not operation log with the name '{log}', use 'yunohost log list to see all available operation logs'",
"log_operation_unit_unclosed_properly": "Operation unit has not been closed properly",
"ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user",
"ldap_initialized": "LDAP has been initialized",
"license_undefined": "undefined",

View file

@ -188,6 +188,7 @@ def app_fetchlist(url=None, name=None):
_write_appslist_list(appslists)
@is_unit_operation()
def app_removelist(name):
"""
Remove list from the repositories
@ -423,8 +424,8 @@ def app_map(app=None, raw=False, user=None):
return result
@is_unit_operation()
def app_change_url(auth, app, domain, path):
@is_unit_operation(lazy=True)
def app_change_url(uo, auth, app, domain, path):
"""
Modify the URL at which an application is installed.
@ -481,6 +482,9 @@ def app_change_url(auth, app, domain, path):
env_dict["YNH_APP_NEW_DOMAIN"] = domain
env_dict["YNH_APP_NEW_PATH"] = path.rstrip("/")
uo.extra.update({'env': env_dict})
uo.start()
if os.path.exists(os.path.join(APP_TMP_FOLDER, "scripts")):
shutil.rmtree(os.path.join(APP_TMP_FOLDER, "scripts"))
@ -499,13 +503,14 @@ def app_change_url(auth, app, domain, path):
os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url")))
if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'), args=args_list, env=env_dict, user="root") != 0:
logger.error("Failed to change '%s' url." % app)
msg = "Failed to change '%s' url." % app
logger.error(msg)
uo.error(msg)
# restore values modified by app_checkurl
# see begining of the function
app_setting(app, "domain", value=old_domain)
app_setting(app, "path", value=old_path)
return
# this should idealy be done in the change_url script but let's avoid common mistakes
@ -531,7 +536,6 @@ def app_change_url(auth, app, domain, path):
hook_callback('post_app_change_url', args=args_list, env=env_dict)
@is_unit_operation()
def app_upgrade(auth, app=[], url=None, file=None):
"""
Upgrade app
@ -614,10 +618,15 @@ def app_upgrade(auth, app=[], url=None, file=None):
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
# Start register change on system
uo = UnitOperation('app_upgrade', 'app', app_instance_name, env=env_dict)
# Execute App upgrade script
os.system('chown -hR admin: %s' % INSTALL_TMP)
if hook_exec(extracted_app_folder + '/scripts/upgrade', args=args_list, env=env_dict, user="root") != 0:
logger.error(m18n.n('app_upgrade_failed', app=app_instance_name))
msg = m18n.n('app_upgrade_failed', app=app_instance_name)
logger.error(msg)
uo.error(msg)
else:
now = int(time.time())
# TODO: Move install_time away from app_setting
@ -646,7 +655,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
logger.success(m18n.n('app_upgraded', app=app_instance_name))
hook_callback('post_app_upgrade', args=args_list, env=env_dict)
uo.success()
if not upgraded_apps:
raise MoulinetteError(errno.ENODATA, m18n.n('app_no_upgrade'))
@ -660,7 +669,8 @@ def app_upgrade(auth, app=[], url=None, file=None):
return {"log": service_log('yunohost-api', number="100").values()[0]}
def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **kwargs):
@is_unit_operation(lazy=True)
def app_install(uo, auth, app, label=None, args=None, no_remove_on_failure=False):
"""
Install apps
@ -672,9 +682,8 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **
"""
from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
from yunohost.log import UnitOperationHandler
from yunohost.log import UnitOperation
uo_install = UnitOperationHandler('app_install', 'app', args=kwargs)
# Fetch or extract sources
try:
@ -732,6 +741,10 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
# Start register change on system
uo.extra.update({'env':env_dict})
uo.start()
# Create app directory
app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
if os.path.exists(app_setting_path):
@ -771,7 +784,7 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **
logger.exception(m18n.n('unexpected_error'))
finally:
if install_retcode != 0:
uo_install.close()
uo.error(m18n.n('unexpected_error'))
if not no_remove_on_failure:
# Setup environment for remove script
env_dict_remove = {}
@ -780,18 +793,21 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
# Execute remove script
uo_remove = UnitOperationHandler('remove_on_failed_install',
'app', args=env_dict_remove)
uo_remove = UnitOperation('remove_on_failed_install',
'app', app_instance_name,
env=env_dict_remove)
remove_retcode = hook_exec(
os.path.join(extracted_app_folder, 'scripts/remove'),
args=[app_instance_name], env=env_dict_remove, user="root"
)
if remove_retcode != 0:
logger.warning(m18n.n('app_not_properly_removed',
app=app_instance_name))
uo_remove.close()
msg = m18n.n('app_not_properly_removed',
app=app_instance_name)
logger.warning(msg)
uo_remove.error(msg)
else:
uo_remove.success()
# Clean tmp folders
shutil.rmtree(app_setting_path)
@ -826,11 +842,9 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False, **
hook_callback('post_app_install', args=args_list, env=env_dict)
uo_install.close()
@is_unit_operation()
def app_remove(auth, app):
@is_unit_operation(lazy=True)
def app_remove(uo, auth, app):
"""
Remove app
@ -839,11 +853,12 @@ def app_remove(auth, app):
"""
from yunohost.hook import hook_exec, hook_remove, hook_callback
if not _is_installed(app):
raise MoulinetteError(errno.EINVAL,
m18n.n('app_not_installed', app=app))
uo.start()
app_setting_path = APPS_SETTING_PATH + app
# TODO: display fail messages from script
@ -863,6 +878,8 @@ def app_remove(auth, app):
env_dict["YNH_APP_ID"] = app_id
env_dict["YNH_APP_INSTANCE_NAME"] = app
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
uo.extra.update({'env': env_dict})
uo.flush()
if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list, env=env_dict, user="root") == 0:
logger.success(m18n.n('app_removed', app=app))
@ -1034,7 +1051,8 @@ def app_debug(app):
}
def app_makedefault(auth, app, domain=None):
@is_unit_operation(lazy=True)
def app_makedefault(uo, auth, app, domain=None):
"""
Redirect domain root to an app
@ -1051,9 +1069,11 @@ def app_makedefault(auth, app, domain=None):
if domain is None:
domain = app_domain
uo.related_to['domain']=[domain]
elif domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
uo.start()
if '/' in app_map(raw=True)[domain]:
raise MoulinetteError(errno.EEXIST,
m18n.n('app_make_default_location_already_used',

View file

@ -49,7 +49,7 @@ from yunohost.utils.network import get_public_ip
from moulinette import m18n
from yunohost.app import app_ssowatconf
from yunohost.service import _run_service_command, service_regen_conf
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.certmanager')

View file

@ -38,6 +38,7 @@ import yunohost.certificate
from yunohost.service import service_regen_conf
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.domain')
@ -62,6 +63,7 @@ def domain_list(auth):
return {'domains': result_list}
@is_unit_operation()
def domain_add(auth, domain, dyndns=False):
"""
Create a custom domain
@ -127,6 +129,7 @@ def domain_add(auth, domain, dyndns=False):
logger.success(m18n.n('domain_created'))
@is_unit_operation()
def domain_remove(auth, domain, force=False):
"""
Delete domains

View file

@ -41,6 +41,7 @@ from moulinette.utils.network import download_json
from yunohost.domain import _get_maindomain, _build_dns_conf
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.dyndns')
@ -113,7 +114,8 @@ def _dyndns_available(provider, domain):
return r == u"Domain %s is available" % domain
def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None):
@is_unit_operation('domain', lazy=True)
def dyndns_subscribe(uo, subscribe_host="dyndns.yunohost.org", domain=None, key=None):
"""
Subscribe to a DynDNS service
@ -126,6 +128,10 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
if domain is None:
domain = _get_maindomain()
uo.on = [domain]
uo.related_to['domain'] = [domain]
uo.start()
# Verify if domain is provided by subscribe_host
if not _dyndns_provides(subscribe_host, domain):
raise MoulinetteError(errno.ENOENT,
@ -170,7 +176,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
dyndns_installcron()
def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
@is_unit_operation('domain',lazy=True)
def dyndns_update(uo, dyn_host="dyndns.yunohost.org", domain=None, key=None,
ipv4=None, ipv6=None):
"""
Update IP on DynDNS platform
@ -212,6 +219,9 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
return
else:
logger.info("Updated needed, going on...")
uo.on = [domain]
uo.related_to['domain'] = [domain]
uo.start()
# If domain is not given, try to guess it from keys available...
if domain is None:

View file

@ -27,10 +27,9 @@
import os
import yaml
import errno
import logging
from datetime import datetime
from logging import StreamHandler, getLogger, Formatter
from logging import FileHandler, getLogger, Formatter
from sys import exc_info
from moulinette import m18n
@ -38,7 +37,9 @@ from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
OPERATIONS_PATH = '/var/log/yunohost/operation/'
OPERATION_FILE_EXT = '.yml'
METADATA_FILE_EXT = '.yml'
LOG_FILE_EXT = '.log'
RELATED_CATEGORIES = ['app', 'domain', 'service', 'user']
logger = getActionLogger('yunohost.log')
@ -57,11 +58,11 @@ def log_list(limit=None):
for category in sorted(os.listdir(OPERATIONS_PATH)):
result["categories"].append({"name": category, "operations": []})
for operation in filter(lambda x: x.endswith(OPERATION_FILE_EXT), os.listdir(os.path.join(OPERATIONS_PATH, category))):
for operation in filter(lambda x: x.endswith(METADATA_FILE_EXT), os.listdir(os.path.join(OPERATIONS_PATH, category))):
file_name = operation
operation = operation[:-len(OPERATION_FILE_EXT)]
operation = operation[:-len(METADATA_FILE_EXT)]
operation = operation.split("_")
operation_datetime = datetime.strptime(" ".join(operation[:2]), "%Y-%m-%d %H-%M-%S")
@ -96,31 +97,27 @@ def log_display(file_name_list):
result = {"operations": []}
for category in os.listdir(OPERATIONS_PATH):
for operation in filter(lambda x: x.endswith(OPERATION_FILE_EXT), os.listdir(os.path.join(OPERATIONS_PATH, category))):
for operation in filter(lambda x: x.endswith(METADATA_FILE_EXT), os.listdir(os.path.join(OPERATIONS_PATH, category))):
if operation not in file_name_list and file_name_list:
continue
file_name = operation
file_name = operation[:-len(METADATA_FILE_EXT)]
operation = file_name.split("_")
with open(os.path.join(OPERATIONS_PATH, category, file_name), "r") as content:
content = content.read()
with open(os.path.join(OPERATIONS_PATH, category, file_name + METADATA_FILE_EXT), "r") as md_file:
try:
infos = yaml.safe_load(md_file)
except yaml.YAMLError as exc:
print(exc)
operation = operation[:-len(OPERATION_FILE_EXT)]
operation = operation.split("_")
operation_datetime = datetime.strptime(" ".join(operation[:2]), "%Y-%m-%d %H-%M-%S")
infos, logs = content.split("\n---\n", 1)
infos = yaml.safe_load(infos)
with open(os.path.join(OPERATIONS_PATH, category, file_name + LOG_FILE_EXT), "r") as content:
logs = content.read()
logs = [{"datetime": x.split(": ", 1)[0].replace("_", " "), "line": x.split(": ", 1)[1]} for x in logs.split("\n") if x]
result['operations'].append({
"started_at": operation_datetime,
"name": " ".join(operation[-2:]),
"file_name": file_name,
"path": os.path.join(OPERATIONS_PATH, category, file_name),
"metadata": infos,
"logs": logs,
})
infos['logs'] = logs
infos['name'] = " ".join(operation[-2:])
infos['file_name'] = file_name + METADATA_FILE_EXT
infos['path'] = os.path.join(OPERATIONS_PATH, category, file_name)
result['operations'].append(infos)
if len(file_name_list) > 0 and len(result['operations']) < len(file_name_list):
logger.error(m18n.n('log_does_exists', log="', '".join(file_name_list)))
@ -129,105 +126,133 @@ def log_display(file_name_list):
result['operations'] = sorted(result['operations'], key=lambda operation: operation['started_at'])
return result
def is_unit_operation(categorie=None, description_key=None):
def is_unit_operation(categorie=None, operation_key=None, lazy=False):
def decorate(func):
def func_wrapper(*args, **kwargs):
cat = categorie
desc_key = description_key
op_key = operation_key
on = None
related_to = {}
inject = lazy
to_start = not lazy
if cat is None:
cat = func.__module__.split('.')[1]
if desc_key is None:
desc_key = func.__name__
uo = UnitOperationHandler(desc_key, cat, args=kwargs)
if op_key is None:
op_key = func.__name__
if cat in kwargs:
on = kwargs[cat]
for r_category in RELATED_CATEGORIES:
if r_category in kwargs and kwargs[r_category] is not None:
if r_category not in related_to:
related_to[r_category] = []
if isinstance(kwargs[r_category], basestring):
related_to[r_category] += [kwargs[r_category]]
else:
related_to[r_category] += kwargs[r_category]
context = kwargs.copy()
if 'auth' in context:
context.pop('auth', None)
uo = UnitOperation(op_key, cat, on, related_to, args=context)
if to_start:
uo.start()
try:
if inject:
args = (uo,) + args
result = func(*args, **kwargs)
finally:
uo.close(exc_info()[0])
if uo.started_at is not None:
uo.close(exc_info()[0])
return result
return func_wrapper
return decorate
class UnitOperationHandler(StreamHandler):
def __init__(self, name, category, **kwargs):
class UnitOperation(object):
def __init__(self, operation, category, on=None, related_to=None, **kwargs):
# TODO add a way to not save password on app installation
self._name = name
self.operation = operation
self.category = category
self.first_write = True
self.closed = False
self.on = on
if isinstance(self.on, basestring):
self.on = [self.on]
# this help uniformise file name and avoir threads concurrency errors
self.started_at = datetime.now()
self.related_to = related_to
if related_to is None:
if self.category in RELATED_CATEGORIES:
self.related_to = {self.category: self.on}
self.extra = kwargs
self.started_at = None
self.ended_at = None
self.logger = None
self.path = os.path.join(OPERATIONS_PATH, category)
if not os.path.exists(self.path):
os.makedirs(self.path)
self.filename = "%s_%s" % (self.started_at.strftime("%F_%X").replace(":", "-"), self._name if isinstance(self._name, basestring) else "_".join(self._name))
self.filename += OPERATION_FILE_EXT
def start(self):
if self.started_at is None:
self.started_at = datetime.now()
self.flush()
self._register_log()
self.additional_information = kwargs
logging.StreamHandler.__init__(self, self._open())
self.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
if self.stream is None:
self.stream = self._open()
def _register_log(self):
# TODO add a way to not save password on app installation
filename = os.path.join(self.path, self.name + LOG_FILE_EXT)
self.file_handler = FileHandler(filename)
self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
# Listen to the root logger
self.logger = getLogger('yunohost')
self.logger.addHandler(self)
self.logger.addHandler(self.file_handler)
def flush(self):
filename = os.path.join(self.path, self.name + METADATA_FILE_EXT)
with open(filename, 'w') as outfile:
yaml.safe_dump(self.metadata, outfile, default_flow_style=False)
def _open(self):
stream = open(os.path.join(self.path, self.filename), "w")
return stream
@property
def name(self):
name = [self.started_at.strftime("%F_%X").replace(":", "-")]
name += [self.operation]
if self.on is not None:
name += self.on
return '_'.join(name)
def close(self, error=None):
"""
Closes the stream.
"""
if self.closed:
return
self.acquire()
#self.ended_at = datetime.now()
#self.error = error
#self.stream.seek(0)
#context = {
# 'ended_at': datetime.now()
#}
#if error is not None:
# context['error'] = error
#self.stream.write(yaml.safe_dump(context))
self.logger.removeHandler(self)
try:
if self.stream:
try:
self.flush()
finally:
stream = self.stream
self.stream = None
if hasattr(stream, "close"):
stream.close()
finally:
self.release()
self.closed = True
@property
def metadata(self):
data = {
'started_at': self.started_at,
'operation': self.operation,
'related_to': self.related_to
}
if self.on is not None:
data['on'] = self.on
if self.ended_at is not None:
data['ended_at'] = self.ended_at
data['success'] = self._success
if self.error is not None:
data['error'] = self._error
# TODO: detect if 'extra' erase some key of 'data'
data.update(self.extra)
return data
def __del__(self):
def success(self):
self.close()
def emit(self, record):
if self.first_write:
self._do_first_write()
self.first_write = False
def error(self, error):
self.close(error)
StreamHandler.emit(self, record)
def close(self, error=None):
if self.ended_at is not None or self.started_at is None:
return
self.ended_at = datetime.now()
self._error = error
self._success = error is None
if self.logger is not None:
self.logger.removeHandler(self.file_handler)
self.flush()
def _do_first_write(self):
def __del__(self):
self.error(m18n.n('log_operation_unit_unclosed_properly'))
serialized_additional_information = yaml.safe_dump(self.additional_information, default_flow_style=False)
self.stream.write(serialized_additional_information)
self.stream.write("\n---\n")

View file

@ -52,6 +52,7 @@ from yunohost.service import service_status, service_regen_conf, service_log, se
from yunohost.monitor import monitor_disk, monitor_system
from yunohost.utils.packages import ynh_packages_version
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
# FIXME this is a duplicate from apps.py
APPS_SETTING_PATH = '/etc/yunohost/apps/'
@ -138,7 +139,8 @@ def tools_adminpw(auth, new_password):
logger.success(m18n.n('admin_password_changed'))
def tools_maindomain(auth, new_domain=None):
@is_unit_operation('domain', lazy=True)
def tools_maindomain(uo, auth, new_domain=None):
"""
Check the current main domain, or change it
@ -155,6 +157,10 @@ def tools_maindomain(auth, new_domain=None):
if new_domain not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
uo.on = [new_domain]
uo.related_to['domain'] = [new_domain]
uo.start()
# Apply changes to ssl certs
ssl_key = "/etc/ssl/private/yunohost_key.pem"
ssl_crt = "/etc/ssl/private/yunohost_crt.pem"
@ -244,6 +250,7 @@ def _is_inside_container():
return out.split()[1] != "(1,"
@is_unit_operation()
def tools_postinstall(domain, password, ignore_dyndns=False):
"""
YunoHost post-install
@ -464,7 +471,8 @@ def tools_update(ignore_apps=False, ignore_packages=False):
return {'packages': packages, 'apps': apps}
def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
@is_unit_operation(lazy=True)
def tools_upgrade(uo, auth, ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
@ -505,6 +513,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
if cache.get_changes():
logger.info(m18n.n('upgrading_packages'))
uo.start()
try:
# Apply APT changes
# TODO: Logs output for the API
@ -514,11 +523,14 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
failure = True
logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
uo.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
uo.success()
else:
logger.info(m18n.n('packages_no_upgrade'))
if not ignore_apps:
try:
app_upgrade(auth)
@ -699,7 +711,8 @@ def tools_port_available(port):
return False
def tools_shutdown(force=False):
@is_unit_operation(lazy=True)
def tools_shutdown(uo, force=False):
shutdown = force
if not shutdown:
try:
@ -712,11 +725,13 @@ def tools_shutdown(force=False):
shutdown = True
if shutdown:
uo.start()
logger.warn(m18n.n('server_shutdown'))
subprocess.check_call(['systemctl', 'poweroff'])
def tools_reboot(force=False):
@is_unit_operation(lazy=True)
def tools_reboot(uo, force=False):
reboot = force
if not reboot:
try:
@ -728,6 +743,7 @@ def tools_reboot(force=False):
if i.lower() == 'y' or i.lower() == 'yes':
reboot = True
if reboot:
uo.start()
logger.warn(m18n.n('server_reboot'))
subprocess.check_call(['systemctl', 'reboot'])

View file

@ -36,6 +36,7 @@ from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from yunohost.service import service_status
from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.user')
@ -89,6 +90,7 @@ def user_list(auth, fields=None):
return {'users': users}
@is_unit_operation()
def user_create(auth, username, firstname, lastname, mail, password,
mailbox_quota="0"):
"""
@ -210,6 +212,7 @@ def user_create(auth, username, firstname, lastname, mail, password,
raise MoulinetteError(169, m18n.n('user_creation_failed'))
@is_unit_operation()
def user_delete(auth, username, purge=False):
"""
Delete user
@ -245,6 +248,7 @@ def user_delete(auth, username, purge=False):
logger.success(m18n.n('user_deleted'))
@is_unit_operation()
def user_update(auth, username, firstname=None, lastname=None, mail=None,
change_password=None, add_mailforward=None, remove_mailforward=None,
add_mailalias=None, remove_mailalias=None, mailbox_quota=None):