From 94d0e253f7ca56b5fb742e18716c5587d14077b1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 7 Apr 2020 20:24:04 +0200 Subject: [PATCH 01/11] Clean usr/bin/yunohost and yunohost-api ... --- bin/yunohost | 100 +++++++++++------------------------------ bin/yunohost-api | 115 +++++++++++------------------------------------ 2 files changed, 52 insertions(+), 163 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index b640c8c52..29a97e016 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -5,36 +5,13 @@ import os import sys import argparse -# Either we are in a development environment or not -IN_DEVEL = False - -# Level for which loggers will log -LOGGERS_LEVEL = 'DEBUG' -TTY_LOG_LEVEL = 'INFO' - -# Handlers that will be used by loggers -# - file: log to the file LOG_DIR/LOG_FILE -# - tty: log to current tty -LOGGERS_HANDLERS = ['file', 'tty'] - -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-cli.log' - -# Check and load - as needed - development environment -if not __file__.startswith('/usr/'): - IN_DEVEL = True -if IN_DEVEL: - basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) - if os.path.isdir(os.path.join(basedir, 'moulinette')): - sys.path.insert(0, basedir) - LOG_DIR = os.path.join(basedir, 'log') - - import moulinette from moulinette.actionsmap import ActionsMap from moulinette.interfaces.cli import colorize, get_locale +# Directory and file to be used by logging +LOG_DIR = '/var/log/yunohost' +LOG_FILE = 'yunohost-cli.log' # Initialization & helpers functions ----------------------------------- @@ -46,10 +23,6 @@ def _die(message, title='Error:'): def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('--no-cache', - action='store_false', default=True, dest='use_cache', - help="Don't use actions map cache", - ) parser.add_argument('--output-as', choices=['json', 'plain', 'none'], default=None, help="Output result in another format", @@ -90,22 +63,13 @@ def _parse_cli_args(): def _init_moulinette(debug=False, quiet=False): """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(LOGGERS_HANDLERS) - if quiet and 'tty' in handlers: - handlers.remove('tty') - elif 'tty' not in handlers: - handlers.append('tty') - root_handlers = set(handlers) - if not debug and 'tty' in root_handlers: - root_handlers.remove('tty') - - # Define loggers level - level = LOGGERS_LEVEL - tty_level = TTY_LOG_LEVEL - if debug: - tty_level = 'DEBUG' + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) # Custom logging configuration logging = { @@ -126,7 +90,7 @@ def _init_moulinette(debug=False, quiet=False): }, 'handlers': { 'tty': { - 'level': tty_level, + 'level': 'DEBUG' if debug else 'INFO', 'class': 'moulinette.interfaces.cli.TTYHandler', 'formatter': 'tty-debug' if debug else '', }, @@ -139,45 +103,34 @@ def _init_moulinette(debug=False, quiet=False): }, 'loggers': { 'yunohost': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], 'propagate': False, }, 'moulinette': { - 'level': level, + 'level': 'DEBUG', 'handlers': [], 'propagate': True, }, 'moulinette.interface': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], 'propagate': False, }, }, 'root': { - 'level': level, - 'handlers': root_handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if debug else ['file'], }, } - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=IN_DEVEL) + moulinette.init(logging_config=logging) def _retrieve_namespaces(): """Return the list of namespaces to load""" - ret = ['yunohost'] - for n in ActionsMap.get_namespaces(): - # Append YunoHost modules - if n.startswith('ynh_'): - ret.append(n) - return ret + extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] + return ['yunohost'] + extensions # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -197,10 +150,9 @@ if __name__ == '__main__': _init_moulinette(opts.debug, opts.quiet) # Check that YunoHost is installed + allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \ - args[0] +' '+ args[1] != 'backup restore' and \ - args[0] +' '+ args[1] != 'log display')): + (len(args) < 2 or (args[0] +' '+ args[1] not in allowed_if_not_installed)): from moulinette import m18n # Init i18n @@ -212,9 +164,11 @@ if __name__ == '__main__': # Execute the action ret = moulinette.cli( - _retrieve_namespaces(), args, - use_cache=opts.use_cache, output_as=opts.output_as, - password=opts.password, parser_kwargs={'top_parser': parser}, + _retrieve_namespaces(), + args, + output_as=opts.output_as, + password=opts.password, timeout=opts.timeout, + parser_kwargs={'top_parser': parser}, ) sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index e518c34b0..7503d28ad 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -5,42 +5,18 @@ import os import sys import argparse -# Either we are in a development environment or not -IN_DEVEL = False +import moulinette +from moulinette.actionsmap import ActionsMap +from moulinette.interfaces.cli import colorize # Default server configuration DEFAULT_HOST = 'localhost' DEFAULT_PORT = 6787 -# Level for which loggers will log -LOGGERS_LEVEL = 'DEBUG' -API_LOGGER_LEVEL = 'INFO' - -# Handlers that will be used by loggers -# - file: log to the file LOG_DIR/LOG_FILE -# - api: serve logs through the api -# - console: log to stderr -LOGGERS_HANDLERS = ['file', 'api'] - # Directory and file to be used by logging LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Check and load - as needed - development environment -if not __file__.startswith('/usr/'): - IN_DEVEL = True -if IN_DEVEL: - basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) - if os.path.isdir(os.path.join(basedir, 'moulinette')): - sys.path.insert(0, basedir) - LOG_DIR = os.path.join(basedir, 'log') - - -import moulinette -from moulinette.actionsmap import ActionsMap -from moulinette.interfaces.cli import colorize - - # Initialization & helpers functions ----------------------------------- def _die(message, title='Error:'): @@ -62,46 +38,26 @@ def _parse_api_args(): action='store', default=DEFAULT_PORT, type=int, help="Port to listen on (default: %d)" % DEFAULT_PORT, ) - srv_group.add_argument('--no-websocket', - action='store_true', default=True, dest='use_websocket', - help="Serve without WebSocket support, used to handle " - "asynchronous responses such as the messages", - ) glob_group = parser.add_argument_group('global arguments') - glob_group.add_argument('--no-cache', - action='store_false', default=True, dest='use_cache', - help="Don't use actions map cache", - ) glob_group.add_argument('--debug', action='store_true', default=False, help="Set log level to DEBUG", ) - glob_group.add_argument('--verbose', - action='store_true', default=False, - help="Be verbose in the output", - ) glob_group.add_argument('--help', action='help', help="Show this help message and exit", ) return parser.parse_args() -def _init_moulinette(use_websocket=True, debug=False, verbose=False): +def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(LOGGERS_HANDLERS) - if not use_websocket and 'api' in handlers: - handlers.remove('api') - if verbose and 'console' not in handlers: - handlers.add('console') - root_handlers = handlers - set(['api']) - # Define loggers level - level = LOGGERS_LEVEL - api_level = API_LOGGER_LEVEL - if debug: - level = 'DEBUG' - api_level = 'DEBUG' + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) # Custom logging configuration logging = { @@ -122,7 +78,7 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): }, 'handlers': { 'api': { - 'level': api_level, + 'level': 'DEBUG' if debug else 'INFO', 'class': 'moulinette.interfaces.api.APIQueueHandler', }, 'file': { @@ -140,58 +96,36 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): }, 'loggers': { 'yunohost': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'api'] + ['console'] if debug else [], 'propagate': False, }, 'moulinette': { - 'level': level, + 'level': 'DEBUG', 'handlers': [], 'propagate': True, }, - 'gnupg': { - 'level': 'INFO', - 'handlers': [], - 'propagate': False, - }, }, 'root': { - 'level': level, - 'handlers': root_handlers, + 'level': 'DEBUG', + 'handlers': ['file'] + ['console'] if debug else [], }, } - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=IN_DEVEL) + moulinette.init(logging_config=logging) def _retrieve_namespaces(): """Return the list of namespaces to load""" - ret = ['yunohost'] - for n in ActionsMap.get_namespaces(): - # Append YunoHost modules - if n.startswith('ynh_'): - ret.append(n) - return ret + extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] + return ['yunohost'] + extensions # Callbacks for additional routes -------------------------------------- def is_installed(): - """ - Check whether YunoHost is installed or not - - """ - installed = False - if os.path.isfile('/etc/yunohost/installed'): - installed = True - return { 'installed': installed } + """ Check whether YunoHost is installed or not """ + return { 'installed': os.path.isfile('/etc/yunohost/installed') } # Main action ---------------------------------------------------------- @@ -203,8 +137,9 @@ if __name__ == '__main__': # Run the server ret = moulinette.api( _retrieve_namespaces(), - host=opts.host, port=opts.port, routes={ - ('GET', '/installed'): is_installed, - }, use_cache=opts.use_cache, use_websocket=opts.use_websocket + host=opts.host, + port=opts.port, + routes={ ('GET', '/installed'): is_installed, }, + use_websocket=True ) sys.exit(ret) From 31e5f7e8b599fb25e27ebdd9b4a4e90f4a1fd62b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 8 Apr 2020 02:58:42 +0200 Subject: [PATCH 02/11] This arg ain't meaningful anymore? --- bin/yunohost | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 29a97e016..4e0ece1e4 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -39,10 +39,6 @@ def _parse_cli_args(): type=int, default=None, help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock", ) - parser.add_argument('--admin-password', - default=None, dest='password', metavar='PASSWORD', - help="The admin password to use to authenticate", - ) # deprecated arguments parser.add_argument('--plain', action='store_true', default=False, help=argparse.SUPPRESS @@ -167,7 +163,6 @@ if __name__ == '__main__': _retrieve_namespaces(), args, output_as=opts.output_as, - password=opts.password, timeout=opts.timeout, parser_kwargs={'top_parser': parser}, ) From 68c9244492166c94570fcc06e4c79c75b2d5186b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 04:34:49 +0200 Subject: [PATCH 03/11] Remove _die, simplify creation of log dir, make linter a bit happier --- bin/yunohost | 24 ++++++++++-------------- bin/yunohost-api | 16 ++++------------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 4e0ece1e4..b56666eb4 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -13,12 +13,12 @@ from moulinette.interfaces.cli import colorize, get_locale LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-cli.log' +# Create log directory +if not os.path.isdir(LOG_DIR): + os.makedirs(LOG_DIR, 0750) + # Initialization & helpers functions ----------------------------------- -def _die(message, title='Error:'): - """Print error message and exit""" - print('%s %s' % (colorize(title, 'red'), message)) - sys.exit(1) def _parse_cli_args(): """Parse additional arguments for the cli""" @@ -57,16 +57,10 @@ def _parse_cli_args(): return (parser, opts, args) + def _init_moulinette(debug=False, quiet=False): """Configure logging and initialize the moulinette""" - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Custom logging configuration logging = { 'version': 1, @@ -123,6 +117,7 @@ def _init_moulinette(debug=False, quiet=False): # Initialize moulinette moulinette.init(logging_config=logging) + def _retrieve_namespaces(): """Return the list of namespaces to load""" extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] @@ -138,7 +133,7 @@ if os.environ["PATH"] != default_path: if __name__ == '__main__': if os.geteuid() != 0: # since moulinette isn't initialized, we can't use m18n here - sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " \ + sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " "run as root or with sudo.\n") sys.exit(1) @@ -148,7 +143,7 @@ if __name__ == '__main__': # Check that YunoHost is installed allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] +' '+ args[1] not in allowed_if_not_installed)): + (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): from moulinette import m18n # Init i18n @@ -156,7 +151,8 @@ if __name__ == '__main__': m18n.set_locale(get_locale()) # Print error and exit - _die(m18n.n('yunohost_not_installed'), m18n.g('error')) + print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) + sys.exit(1) # Execute the action ret = moulinette.cli( diff --git a/bin/yunohost-api b/bin/yunohost-api index 7503d28ad..3185738f6 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -17,12 +17,11 @@ DEFAULT_PORT = 6787 LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Initialization & helpers functions ----------------------------------- +# Create log directory +if not os.path.isdir(LOG_DIR): + os.makedirs(LOG_DIR, 0750) -def _die(message, title='Error:'): - """Print error message and exit""" - print('%s %s' % (colorize(title, 'red'), message)) - sys.exit(1) +# Initialization & helpers functions ----------------------------------- def _parse_api_args(): """Parse main arguments for the api""" @@ -52,13 +51,6 @@ def _parse_api_args(): def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Custom logging configuration logging = { 'version': 1, From f5c16737ebd44748225e7d17f60faf67c05d97db Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 05:15:08 +0200 Subject: [PATCH 04/11] More stuff to reduce complexity --- bin/yunohost | 7 +++++-- bin/yunohost-api | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index b56666eb4..173cbc1cb 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,9 +4,9 @@ import os import sys import argparse +import glob import moulinette -from moulinette.actionsmap import ActionsMap from moulinette.interfaces.cli import colorize, get_locale # Directory and file to be used by logging @@ -123,6 +123,7 @@ def _retrieve_namespaces(): extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] return ['yunohost'] + extensions + # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" if os.environ["PATH"] != default_path: @@ -154,9 +155,11 @@ if __name__ == '__main__': print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) sys.exit(1) + extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] + # Execute the action ret = moulinette.cli( - _retrieve_namespaces(), + ['yunohost'] + extensions, args, output_as=opts.output_as, timeout=opts.timeout, diff --git a/bin/yunohost-api b/bin/yunohost-api index 3185738f6..7a2119a08 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -4,10 +4,9 @@ import os import sys import argparse +import glob import moulinette -from moulinette.actionsmap import ActionsMap -from moulinette.interfaces.cli import colorize # Default server configuration DEFAULT_HOST = 'localhost' @@ -23,6 +22,7 @@ if not os.path.isdir(LOG_DIR): # Initialization & helpers functions ----------------------------------- + def _parse_api_args(): """Parse main arguments for the api""" parser = argparse.ArgumentParser(add_help=False, @@ -48,6 +48,7 @@ def _parse_api_args(): return parser.parse_args() + def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" @@ -107,31 +108,28 @@ def _init_moulinette(debug=False): # Initialize moulinette moulinette.init(logging_config=logging) -def _retrieve_namespaces(): - """Return the list of namespaces to load""" - extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] - return ['yunohost'] + extensions - - # Callbacks for additional routes -------------------------------------- + def is_installed(): """ Check whether YunoHost is installed or not """ - return { 'installed': os.path.isfile('/etc/yunohost/installed') } + return {'installed': os.path.isfile('/etc/yunohost/installed')} # Main action ---------------------------------------------------------- if __name__ == '__main__': opts = _parse_api_args() - _init_moulinette(opts.use_websocket, opts.debug, opts.verbose) + _init_moulinette(opts.debug) + + extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] # Run the server ret = moulinette.api( - _retrieve_namespaces(), + ['yunohost'] + extensions, host=opts.host, port=opts.port, - routes={ ('GET', '/installed'): is_installed, }, + routes={('GET', '/installed'): is_installed}, use_websocket=True ) sys.exit(ret) From 233c962710aec0e98f77936bde476fe662ca062f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 16:49:35 +0200 Subject: [PATCH 05/11] Hmpf idk another iteration to cleaning attempt --- bin/yunohost | 26 +++++++++++--------------- bin/yunohost-api | 23 +++++++++-------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 173cbc1cb..8c4b10d8d 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -7,19 +7,13 @@ import argparse import glob import moulinette -from moulinette.interfaces.cli import colorize, get_locale # Directory and file to be used by logging LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-cli.log' -# Create log directory -if not os.path.isdir(LOG_DIR): - os.makedirs(LOG_DIR, 0750) - # Initialization & helpers functions ----------------------------------- - def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) @@ -58,11 +52,13 @@ def _parse_cli_args(): return (parser, opts, args) -def _init_moulinette(debug=False, quiet=False): - """Configure logging and initialize the moulinette""" +def init(debug=False, quiet=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - # Custom logging configuration - logging = { + logdir = os.path.dirname(logfile) + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + moulinette.init(logging_config={ 'version': 1, 'disable_existing_loggers': True, 'formatters': { @@ -87,7 +83,7 @@ def _init_moulinette(debug=False, quiet=False): 'file': { 'class': 'logging.FileHandler', 'formatter': 'precise', - 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + 'filename': logfile, 'filters': ['action'], }, }, @@ -112,10 +108,8 @@ def _init_moulinette(debug=False, quiet=False): 'level': 'DEBUG', 'handlers': ['file', 'tty'] if debug else ['file'], }, - } + }) - # Initialize moulinette - moulinette.init(logging_config=logging) def _retrieve_namespaces(): @@ -139,7 +133,7 @@ if __name__ == '__main__': sys.exit(1) parser, opts, args = _parse_cli_args() - _init_moulinette(opts.debug, opts.quiet) + init(debug=opts.debug, quiet=opts.quiet) # Check that YunoHost is installed allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] @@ -147,6 +141,8 @@ if __name__ == '__main__': (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): from moulinette import m18n + from moulinette.interfaces.cli import colorize, get_locale + # Init i18n m18n.load_namespace('yunohost') m18n.set_locale(get_locale()) diff --git a/bin/yunohost-api b/bin/yunohost-api index 7a2119a08..0b6961a90 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -16,10 +16,6 @@ DEFAULT_PORT = 6787 LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Create log directory -if not os.path.isdir(LOG_DIR): - os.makedirs(LOG_DIR, 0750) - # Initialization & helpers functions ----------------------------------- @@ -49,11 +45,13 @@ def _parse_api_args(): return parser.parse_args() -def _init_moulinette(debug=False): - """Configure logging and initialize the moulinette""" +def init_api(debug=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - # Custom logging configuration - logging = { + logdir = os.path.dirname(logfile) + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + moulinette.init(logging_config={ 'version': 1, 'disable_existing_loggers': True, 'formatters': { @@ -77,7 +75,7 @@ def _init_moulinette(debug=False): 'file': { 'class': 'logging.handlers.WatchedFileHandler', 'formatter': 'precise', - 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + 'filename': logfile, 'filters': ['action'], }, 'console': { @@ -103,10 +101,7 @@ def _init_moulinette(debug=False): 'level': 'DEBUG', 'handlers': ['file'] + ['console'] if debug else [], }, - } - - # Initialize moulinette - moulinette.init(logging_config=logging) + }) # Callbacks for additional routes -------------------------------------- @@ -120,7 +115,7 @@ def is_installed(): if __name__ == '__main__': opts = _parse_api_args() - _init_moulinette(opts.debug) + init_api(opts.debug) extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] From dee08a16feb5358122bb424f0da3458952f03e85 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 18:41:54 +0200 Subject: [PATCH 06/11] Move moulinette initialization and other stuff to src/yunohost/__init__.py --- bin/yunohost | 113 +++------------------- bin/yunohost-api | 92 +----------------- src/yunohost/__init__.py | 202 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 191 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 8c4b10d8d..546d2d913 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,34 +4,29 @@ import os import sys import argparse -import glob -import moulinette +sys.path.insert(0, "/usr/lib/moulinette/") +import yunohost -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-cli.log' - -# Initialization & helpers functions ----------------------------------- def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--output-as', choices=['json', 'plain', 'none'], default=None, - help="Output result in another format", + help="Output result in another format" ) parser.add_argument('--debug', action='store_true', default=False, - help="Log and print debug messages", + help="Log and print debug messages" ) parser.add_argument('--quiet', action='store_true', default=False, - help="Don't produce any output", + help="Don't produce any output" ) parser.add_argument('--timeout', type=int, default=None, - help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock", + help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock" ) # deprecated arguments parser.add_argument('--plain', @@ -52,72 +47,6 @@ def _parse_cli_args(): return (parser, opts, args) -def init(debug=False, quiet=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - - logdir = os.path.dirname(logfile) - if not os.path.isdir(logdir): - os.makedirs(logdir, 0750) - - moulinette.init(logging_config={ - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'tty-debug': { - 'format': '%(relativeCreated)-4d %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'tty': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.cli.TTYHandler', - 'formatter': 'tty-debug' if debug else '', - }, - 'file': { - 'class': 'logging.FileHandler', - 'formatter': 'precise', - 'filename': logfile, - 'filters': ['action'], - }, - }, - 'loggers': { - 'yunohost': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if not quiet else ['file'], - 'propagate': False, - }, - 'moulinette': { - 'level': 'DEBUG', - 'handlers': [], - 'propagate': True, - }, - 'moulinette.interface': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if not quiet else ['file'], - 'propagate': False, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if debug else ['file'], - }, - }) - - - -def _retrieve_namespaces(): - """Return the list of namespaces to load""" - extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] - return ['yunohost'] + extensions - - # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" if os.environ["PATH"] != default_path: @@ -127,38 +56,18 @@ if os.environ["PATH"] != default_path: if __name__ == '__main__': if os.geteuid() != 0: - # since moulinette isn't initialized, we can't use m18n here sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " "run as root or with sudo.\n") sys.exit(1) parser, opts, args = _parse_cli_args() - init(debug=opts.debug, quiet=opts.quiet) - - # Check that YunoHost is installed - allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] - if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): - - from moulinette import m18n - from moulinette.interfaces.cli import colorize, get_locale - - # Init i18n - m18n.load_namespace('yunohost') - m18n.set_locale(get_locale()) - - # Print error and exit - print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) - sys.exit(1) - - extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] # Execute the action - ret = moulinette.cli( - ['yunohost'] + extensions, - args, + yunohost.cli( + debug=opts.debug, + quiet=opts.quiet, output_as=opts.output_as, timeout=opts.timeout, - parser_kwargs={'top_parser': parser}, + args=args, + parser=parser ) - sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index 0b6961a90..cc849590a 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -1,23 +1,16 @@ #! /usr/bin/python # -*- coding: utf-8 -*- -import os import sys import argparse -import glob -import moulinette +sys.path.insert(0, "/usr/lib/moulinette/") +import yunohost # Default server configuration DEFAULT_HOST = 'localhost' DEFAULT_PORT = 6787 -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-api.log' - -# Initialization & helpers functions ----------------------------------- - def _parse_api_args(): """Parse main arguments for the api""" @@ -45,86 +38,7 @@ def _parse_api_args(): return parser.parse_args() -def init_api(debug=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - - logdir = os.path.dirname(logfile) - if not os.path.isdir(logdir): - os.makedirs(logdir, 0750) - - moulinette.init(logging_config={ - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'console': { - 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'api': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.api.APIQueueHandler', - }, - 'file': { - 'class': 'logging.handlers.WatchedFileHandler', - 'formatter': 'precise', - 'filename': logfile, - 'filters': ['action'], - }, - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'console', - 'stream': 'ext://sys.stdout', - 'filters': ['action'], - }, - }, - 'loggers': { - 'yunohost': { - 'level': 'DEBUG', - 'handlers': ['file', 'api'] + ['console'] if debug else [], - 'propagate': False, - }, - 'moulinette': { - 'level': 'DEBUG', - 'handlers': [], - 'propagate': True, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['file'] + ['console'] if debug else [], - }, - }) - -# Callbacks for additional routes -------------------------------------- - - -def is_installed(): - """ Check whether YunoHost is installed or not """ - return {'installed': os.path.isfile('/etc/yunohost/installed')} - - -# Main action ---------------------------------------------------------- - if __name__ == '__main__': opts = _parse_api_args() - init_api(opts.debug) - - extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] - # Run the server - ret = moulinette.api( - ['yunohost'] + extensions, - host=opts.host, - port=opts.port, - routes={('GET', '/installed'): is_installed}, - use_websocket=True - ) - sys.exit(ret) + yunohost.api(debug=opts.debug, host=opts.host, port=opts.port) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index e69de29bb..06e3d773d 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -0,0 +1,202 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import glob + +import moulinette +from moulinette.utils.log import configure_logging + + +def is_installed(): + return os.path.isfile('/etc/yunohost/installed') + + +def cli(debug, quiet, output_as, timeout, args, parser): + + init_logging(interface="cli", debug=debug, quiet=quiet) + + # Check that YunoHost is installed + if not is_installed(): + check_command_is_valid_before_postinstall(args) + + ret = moulinette.cli( + ['yunohost'] + extensions(), + args, + output_as=output_as, + timeout=timeout, + parser_kwargs={'top_parser': parser}, + ) + sys.exit(ret) + + +def api(debug, host, port): + + init_logging(debug=debug) + + def is_installed_api(): + return {'installed': is_installed()} + + # FIXME : someday, maybe find a way to disable route /postinstall if + # postinstall already done ... + + ret = moulinette.api( + ['yunohost'] + extensions(), + host=host, + port=port, + routes={('GET', '/installed'): is_installed_api}, + use_websocket=True + ) + sys.exit(ret) + + +def extensions(): + # This is probably not used anywhere, but the actionsmap and code can be + # extended by creating such files that contain bits of actionmap... + return [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] + + +def check_command_is_valid_before_postinstall(args): + + allowed_if_not_postinstalled = ['tools postinstall', + 'tools versions', + 'backup list', + 'backup restore', + 'log display'] + + if (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_postinstalled)): + + # This function is called before m18n is initialized, so we only initialized + # the specific bit to be able to call m18n.n/g()... + from moulinette import m18n + from moulinette.interfaces.cli import colorize, get_locale + + # Init i18n + m18n.load_namespace('yunohost') + m18n.set_locale(get_locale()) + + print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) + sys.exit(1) + + +def init_logging(interface="cli", + debug=False, + quiet=False, + logdir="/var/log/yunohost"): + + logfile = os.path.join(logdir, "yunohost-%s.log" % interface) + + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + # ####################################################################### # + # Logging configuration for CLI (or any other interface than api...) # + # ####################################################################### # + if interface != "api": + configure_logging({ + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'tty-debug': { + 'format': '%(relativeCreated)-4d %(fmessage)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + }, + 'filters': { + 'action': { + '()': 'moulinette.utils.log.ActionFilter', + }, + }, + 'handlers': { + 'tty': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'moulinette.interfaces.cli.TTYHandler', + 'formatter': 'tty-debug' if debug else '', + }, + 'file': { + 'class': 'logging.FileHandler', + 'formatter': 'precise', + 'filename': logfile, + 'filters': ['action'], + }, + }, + 'loggers': { + 'yunohost': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], + 'propagate': False, + }, + 'moulinette': { + 'level': 'DEBUG', + 'handlers': [], + 'propagate': True, + }, + 'moulinette.interface': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], + 'propagate': False, + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if debug else ['file'], + }, + }) + # ####################################################################### # + # Logging configuration for API # + # ####################################################################### # + else: + configure_logging({ + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'console': { + 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + }, + 'filters': { + 'action': { + '()': 'moulinette.utils.log.ActionFilter', + }, + }, + 'handlers': { + 'api': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'moulinette.interfaces.api.APIQueueHandler', + }, + 'file': { + 'class': 'logging.handlers.WatchedFileHandler', + 'formatter': 'precise', + 'filename': logfile, + 'filters': ['action'], + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', + 'stream': 'ext://sys.stdout', + 'filters': ['action'], + }, + }, + 'loggers': { + 'yunohost': { + 'level': 'DEBUG', + 'handlers': ['file', 'api'] + ['console'] if debug else [], + 'propagate': False, + }, + 'moulinette': { + 'level': 'DEBUG', + 'handlers': [], + 'propagate': True, + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['file'] + ['console'] if debug else [], + }, + }) From 69a520657c13aba8964c659963413cd8a1e6a951 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 18:59:44 +0200 Subject: [PATCH 07/11] Add init_i18n for convenience when using yunohost as a lib... --- src/yunohost/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 06e3d773d..8a23f8e00 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -6,8 +6,9 @@ import sys import glob import moulinette +from moulinette import m18n from moulinette.utils.log import configure_logging - +from moulinette.interfaces.cli import colorize, get_locale def is_installed(): return os.path.isfile('/etc/yunohost/installed') @@ -66,20 +67,18 @@ def check_command_is_valid_before_postinstall(args): 'log display'] if (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_postinstalled)): - - # This function is called before m18n is initialized, so we only initialized - # the specific bit to be able to call m18n.n/g()... - from moulinette import m18n - from moulinette.interfaces.cli import colorize, get_locale - - # Init i18n - m18n.load_namespace('yunohost') - m18n.set_locale(get_locale()) - + init_i18n() print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) sys.exit(1) +def init_i18n(): + # This should only be called when not willing to go through moulinette.cli + # or moulinette.api but still willing to call m18n.n/g... + m18n.load_namespace('yunohost') + m18n.set_locale(get_locale()) + + def init_logging(interface="cli", debug=False, quiet=False, From 7df8e8421df39bbed03f1fc7cb53102876a400a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Apr 2020 04:03:02 +0200 Subject: [PATCH 08/11] Forgot to set interface as api for init logging --- src/yunohost/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 8a23f8e00..e2821f558 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -34,7 +34,7 @@ def cli(debug, quiet, output_as, timeout, args, parser): def api(debug, host, port): - init_logging(debug=debug) + init_logging(interface="api", debug=debug) def is_installed_api(): return {'installed': is_installed()} From 84e39a416a69c7974e36cf32af1d4e40e376bf03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 04:43:04 +0200 Subject: [PATCH 09/11] Add an 'init' helper for scripts/tests to initialize everything needed... --- src/yunohost/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index e2821f558..3e7ddb496 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -72,6 +72,19 @@ def check_command_is_valid_before_postinstall(args): sys.exit(1) +def init(interface="cli", debug=False, quiet=False, logdir="/var/log/yunohost"): + """ + This is a small util function ONLY meant to be used to initialize a Yunohost + context when ran from tests or from scripts. + """ + init_logging(interface=interface, debug=debug, quiet=quiet, logdir=logdir) + init_i18n() + from moulinette.core import MoulinetteLock + lock = MoulinetteLock("yunohost", timeout=30) + lock.acquire() + return lock + + def init_i18n(): # This should only be called when not willing to go through moulinette.cli # or moulinette.api but still willing to call m18n.n/g... From 96e115c6091317d4d77e324e4f40f9e659648e03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 04:43:29 +0200 Subject: [PATCH 10/11] Use 'init' helper from tests! --- src/yunohost/tests/conftest.py | 67 ++-------------------------------- 1 file changed, 4 insertions(+), 63 deletions(-) diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 073c880f8..69f8bcfed 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -1,12 +1,11 @@ import os import pytest import sys -import moulinette +import moulinette from moulinette import m18n from yunohost.utils.error import YunohostError from contextlib import contextmanager - sys.path.append("..") @@ -68,65 +67,7 @@ moulinette.core.Moulinette18n.n = new_m18nn def pytest_cmdline_main(config): - """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(['tty']) - root_handlers = set(handlers) - # Define loggers level - level = 'DEBUG' - if config.option.yunodebug: - tty_level = 'DEBUG' - else: - tty_level = 'INFO' - - # Custom logging configuration - logging = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'tty-debug': { - 'format': '%(relativeCreated)-4d %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'tty': { - 'level': tty_level, - 'class': 'moulinette.interfaces.cli.TTYHandler', - 'formatter': '', - }, - }, - 'loggers': { - 'yunohost': { - 'level': level, - 'handlers': handlers, - 'propagate': False, - }, - 'moulinette': { - 'level': level, - 'handlers': [], - 'propagate': True, - }, - 'moulinette.interface': { - 'level': level, - 'handlers': handlers, - 'propagate': False, - }, - }, - 'root': { - 'level': level, - 'handlers': root_handlers, - }, - } - - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=False) - moulinette.m18n.load_namespace('yunohost') + sys.path.insert(0, "/usr/lib/moulinette/") + import yunohost + yunohost.init(debug=config.option.yunodebug) From d8be90165c2345ec956e24f789a7215f923679dd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 May 2020 06:11:53 +0200 Subject: [PATCH 11/11] Propagate changes from moulinette/simplify-interface-init --- src/yunohost/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 3e7ddb496..810d6127a 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -3,13 +3,13 @@ import os import sys -import glob import moulinette from moulinette import m18n from moulinette.utils.log import configure_logging from moulinette.interfaces.cli import colorize, get_locale + def is_installed(): return os.path.isfile('/etc/yunohost/installed') @@ -23,11 +23,10 @@ def cli(debug, quiet, output_as, timeout, args, parser): check_command_is_valid_before_postinstall(args) ret = moulinette.cli( - ['yunohost'] + extensions(), args, output_as=output_as, timeout=timeout, - parser_kwargs={'top_parser': parser}, + top_parser=parser ) sys.exit(ret) @@ -43,21 +42,13 @@ def api(debug, host, port): # postinstall already done ... ret = moulinette.api( - ['yunohost'] + extensions(), host=host, port=port, routes={('GET', '/installed'): is_installed_api}, - use_websocket=True ) sys.exit(ret) -def extensions(): - # This is probably not used anywhere, but the actionsmap and code can be - # extended by creating such files that contain bits of actionmap... - return [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] - - def check_command_is_valid_before_postinstall(args): allowed_if_not_postinstalled = ['tools postinstall',