From c7dd8817740a7af4bf2b267eb7fc3d6667775857 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 May 2020 21:41:54 +0200 Subject: [PATCH 001/138] Default 'ask' questions for common app manifest args --- locales/en.json | 5 ++++ src/yunohost/app.py | 59 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index 25712e8cd..486bc053c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -27,6 +27,11 @@ "app_make_default_location_already_used": "Can't make the app '{app}' the default on the domain, '{domain}' is already in use by the other app '{other_app}'", "app_location_unavailable": "This URL is either unavailable, or conflicts with the already installed app(s):\n{apps:s}", "app_manifest_invalid": "Something is wrong with the app manifest: {error}", + "app_manifest_install_ask_domain": "Choose the domain where this app should be installed", + "app_manifest_install_ask_path": "Choose the path where this app should be installed", + "app_manifest_install_ask_password": "Choose an administration password for this app", + "app_manifest_install_ask_admin": "Choose an administrator user for this app", + "app_manifest_install_ask_is_public": "Should this app be exposed to anonymous visitors?", "app_not_upgraded": "The app '{failed_app}' failed to upgrade, and as a consequence the following apps' upgrades have been cancelled: {apps}", "app_not_correctly_installed": "{app:s} seems to be incorrectly installed", "app_not_installed": "Could not find the app '{app:s}' in the list of installed apps: {all_apps}", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e2df6ba78..4a42a0484 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -91,6 +91,8 @@ def app_catalog(full=False, with_categories=False): "description": infos['manifest']['description'], "level": infos["level"], } + else: + infos["manifest"]["arguments"] = _set_default_ask_questions(infos["manifest"]["arguments"]) # Trim info for categories if not using --full for category in catalog["categories"]: @@ -110,7 +112,6 @@ def app_catalog(full=False, with_categories=False): return {"apps": catalog["apps"], "categories": catalog["categories"]} - # Old legacy function... def app_fetchlist(): logger.warning("'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead") @@ -170,6 +171,7 @@ def app_info(app, full=False): return ret ret["manifest"] = local_manifest + ret["manifest"]["arguments"] = _set_default_ask_questions(ret["manifest"]["arguments"]) ret['settings'] = settings absolute_app_name = app if "__" not in app else app[:app.index('__')] # idk this is the name of the app even for multiinstance apps (so wordpress__2 -> wordpress) @@ -2071,12 +2073,63 @@ def _get_manifest_of_app(path): manifest["arguments"]["install"] = install_arguments - return manifest elif os.path.exists(os.path.join(path, "manifest.json")): - return read_json(os.path.join(path, "manifest.json")) + manifest = read_json(os.path.join(path, "manifest.json")) else: raise YunohostError("There doesn't seem to be any manifest file in %s ... It looks like an app was not correctly installed/removed." % path, raw_msg=True) + manifest["arguments"] = _set_default_ask_questions(manifest["arguments"]) + return manifest + + +def _set_default_ask_questions(arguments): + + # arguments is something like + # { "install": [ + # { "name": "domain", + # "type": "domain", + # .... + # }, + # { "name": "path", + # "type": "path" + # ... + # }, + # ... + # ], + # "upgrade": [ ... ] + # } + + # We set a default for any question with these matching (type, name) + # type namei + # N.B. : this is only for install script ... should be reworked for other + # scripts if we supports args for other scripts in the future... + questions_with_default = [("domain", "domain"), + ("path", "path"), + ("password", "password"), + ("user", "admin"), + ("boolean", "is_public")] + + for script_name, arg_list in arguments.items(): + + # We only support questions for install so far, and for other + if script_name != "install": + continue + + for arg in arg_list: + + # Do not override 'ask' field if provided by app ?... Or shall we ? + #if "ask" in arg: + # continue + + # If this arg corresponds to a question with default ask message... + if any((arg.get("type"), arg["name"]) == question for question in questions_with_default): + # The key is for example "app_manifest_install_ask_domain" + key = "app_manifest_%s_ask_%s" % (script_name, arg["name"]) + arg["ask"] = m18n.n(key) + + return arguments + + def _get_git_last_commit_hash(repository, reference='HEAD'): """ From 49c4324ee1af0c02f927afe3d326c0aca961b147 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 May 2020 22:17:39 +0200 Subject: [PATCH 002/138] During app installs, set default answer for user-type args to main user --- src/yunohost/app.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e2df6ba78..289a03bd2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2439,9 +2439,16 @@ def _parse_args_in_yunohost_format(args, action_args): elif arg_type == 'user': msignals.display(m18n.n('users_available')) - for user in user_list()['users'].keys(): + users = user_list()['users'] + for user in users.keys(): msignals.display("- {}".format(user)) + root_mail = "root@%s" % _get_maindomain() + for user in users.keys(): + if root_mail in user_info(user)["mail-aliases"]: + arg_default = user + ask_string += ' (default: {0})'.format(arg_default) + elif arg_type == 'password': msignals.display(m18n.n('good_practices_about_user_password')) From 882b003bd70718a181347971d1262d6167e25082 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 May 2020 00:07:10 +0200 Subject: [PATCH 003/138] Fix i18n string test --- src/yunohost/app.py | 13 ++++++------- tests/test_i18n_keys.py | 4 ++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4a42a0484..0647e17d4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2103,11 +2103,11 @@ def _set_default_ask_questions(arguments): # type namei # N.B. : this is only for install script ... should be reworked for other # scripts if we supports args for other scripts in the future... - questions_with_default = [("domain", "domain"), - ("path", "path"), - ("password", "password"), - ("user", "admin"), - ("boolean", "is_public")] + questions_with_default = [("domain", "domain"), # i18n: app_manifest_install_ask_domain + ("path", "path"), # i18n: app_manifest_install_ask_path + ("password", "password"), # i18n: app_manifest_install_ask_password + ("user", "admin"), # i18n: app_manifest_install_ask_admin + ("boolean", "is_public")] # i18n: app_manifest_install_ask_is_public for script_name, arg_list in arguments.items(): @@ -2118,7 +2118,7 @@ def _set_default_ask_questions(arguments): for arg in arg_list: # Do not override 'ask' field if provided by app ?... Or shall we ? - #if "ask" in arg: + # if "ask" in arg: # continue # If this arg corresponds to a question with default ask message... @@ -2130,7 +2130,6 @@ def _set_default_ask_questions(arguments): return arguments - def _get_git_last_commit_hash(repository, reference='HEAD'): """ Attempt to retrieve the last commit hash of a git repository diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 874794e11..799db3de2 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -23,8 +23,10 @@ def find_expected_string_keys(): # Try to find : # m18n.n( "foo" # YunohostError("foo" + # # i18n: foo p1 = re.compile(r'm18n\.n\(\s*[\"\'](\w+)[\"\']') p2 = re.compile(r'YunohostError\([\'\"](\w+)[\'\"]') + p3 = re.compile(r'# i18n: [\'\"]?(\w+)[\'\"]?') python_files = glob.glob("src/yunohost/*.py") python_files.extend(glob.glob("src/yunohost/utils/*.py")) @@ -42,6 +44,8 @@ def find_expected_string_keys(): if m.endswith("_"): continue yield m + for m in p3.findall(content): + yield m # For each diagnosis, try to find strings like "diagnosis_stuff_foo" (c.f. diagnosis summaries) # Also we expect to have "diagnosis_description_" for each diagnosis From 94d0e253f7ca56b5fb742e18716c5587d14077b1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 7 Apr 2020 20:24:04 +0200 Subject: [PATCH 004/138] Clean usr/bin/yunohost and yunohost-api ... --- bin/yunohost | 100 +++++++++++------------------------------ bin/yunohost-api | 115 +++++++++++------------------------------------ 2 files changed, 52 insertions(+), 163 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index b640c8c52..29a97e016 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -5,36 +5,13 @@ import os import sys import argparse -# Either we are in a development environment or not -IN_DEVEL = False - -# Level for which loggers will log -LOGGERS_LEVEL = 'DEBUG' -TTY_LOG_LEVEL = 'INFO' - -# Handlers that will be used by loggers -# - file: log to the file LOG_DIR/LOG_FILE -# - tty: log to current tty -LOGGERS_HANDLERS = ['file', 'tty'] - -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-cli.log' - -# Check and load - as needed - development environment -if not __file__.startswith('/usr/'): - IN_DEVEL = True -if IN_DEVEL: - basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) - if os.path.isdir(os.path.join(basedir, 'moulinette')): - sys.path.insert(0, basedir) - LOG_DIR = os.path.join(basedir, 'log') - - import moulinette from moulinette.actionsmap import ActionsMap from moulinette.interfaces.cli import colorize, get_locale +# Directory and file to be used by logging +LOG_DIR = '/var/log/yunohost' +LOG_FILE = 'yunohost-cli.log' # Initialization & helpers functions ----------------------------------- @@ -46,10 +23,6 @@ def _die(message, title='Error:'): def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('--no-cache', - action='store_false', default=True, dest='use_cache', - help="Don't use actions map cache", - ) parser.add_argument('--output-as', choices=['json', 'plain', 'none'], default=None, help="Output result in another format", @@ -90,22 +63,13 @@ def _parse_cli_args(): def _init_moulinette(debug=False, quiet=False): """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(LOGGERS_HANDLERS) - if quiet and 'tty' in handlers: - handlers.remove('tty') - elif 'tty' not in handlers: - handlers.append('tty') - root_handlers = set(handlers) - if not debug and 'tty' in root_handlers: - root_handlers.remove('tty') - - # Define loggers level - level = LOGGERS_LEVEL - tty_level = TTY_LOG_LEVEL - if debug: - tty_level = 'DEBUG' + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) # Custom logging configuration logging = { @@ -126,7 +90,7 @@ def _init_moulinette(debug=False, quiet=False): }, 'handlers': { 'tty': { - 'level': tty_level, + 'level': 'DEBUG' if debug else 'INFO', 'class': 'moulinette.interfaces.cli.TTYHandler', 'formatter': 'tty-debug' if debug else '', }, @@ -139,45 +103,34 @@ def _init_moulinette(debug=False, quiet=False): }, 'loggers': { 'yunohost': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], 'propagate': False, }, 'moulinette': { - 'level': level, + 'level': 'DEBUG', 'handlers': [], 'propagate': True, }, 'moulinette.interface': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], 'propagate': False, }, }, 'root': { - 'level': level, - 'handlers': root_handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if debug else ['file'], }, } - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=IN_DEVEL) + moulinette.init(logging_config=logging) def _retrieve_namespaces(): """Return the list of namespaces to load""" - ret = ['yunohost'] - for n in ActionsMap.get_namespaces(): - # Append YunoHost modules - if n.startswith('ynh_'): - ret.append(n) - return ret + extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] + return ['yunohost'] + extensions # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -197,10 +150,9 @@ if __name__ == '__main__': _init_moulinette(opts.debug, opts.quiet) # Check that YunoHost is installed + allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \ - args[0] +' '+ args[1] != 'backup restore' and \ - args[0] +' '+ args[1] != 'log display')): + (len(args) < 2 or (args[0] +' '+ args[1] not in allowed_if_not_installed)): from moulinette import m18n # Init i18n @@ -212,9 +164,11 @@ if __name__ == '__main__': # Execute the action ret = moulinette.cli( - _retrieve_namespaces(), args, - use_cache=opts.use_cache, output_as=opts.output_as, - password=opts.password, parser_kwargs={'top_parser': parser}, + _retrieve_namespaces(), + args, + output_as=opts.output_as, + password=opts.password, timeout=opts.timeout, + parser_kwargs={'top_parser': parser}, ) sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index e518c34b0..7503d28ad 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -5,42 +5,18 @@ import os import sys import argparse -# Either we are in a development environment or not -IN_DEVEL = False +import moulinette +from moulinette.actionsmap import ActionsMap +from moulinette.interfaces.cli import colorize # Default server configuration DEFAULT_HOST = 'localhost' DEFAULT_PORT = 6787 -# Level for which loggers will log -LOGGERS_LEVEL = 'DEBUG' -API_LOGGER_LEVEL = 'INFO' - -# Handlers that will be used by loggers -# - file: log to the file LOG_DIR/LOG_FILE -# - api: serve logs through the api -# - console: log to stderr -LOGGERS_HANDLERS = ['file', 'api'] - # Directory and file to be used by logging LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Check and load - as needed - development environment -if not __file__.startswith('/usr/'): - IN_DEVEL = True -if IN_DEVEL: - basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) - if os.path.isdir(os.path.join(basedir, 'moulinette')): - sys.path.insert(0, basedir) - LOG_DIR = os.path.join(basedir, 'log') - - -import moulinette -from moulinette.actionsmap import ActionsMap -from moulinette.interfaces.cli import colorize - - # Initialization & helpers functions ----------------------------------- def _die(message, title='Error:'): @@ -62,46 +38,26 @@ def _parse_api_args(): action='store', default=DEFAULT_PORT, type=int, help="Port to listen on (default: %d)" % DEFAULT_PORT, ) - srv_group.add_argument('--no-websocket', - action='store_true', default=True, dest='use_websocket', - help="Serve without WebSocket support, used to handle " - "asynchronous responses such as the messages", - ) glob_group = parser.add_argument_group('global arguments') - glob_group.add_argument('--no-cache', - action='store_false', default=True, dest='use_cache', - help="Don't use actions map cache", - ) glob_group.add_argument('--debug', action='store_true', default=False, help="Set log level to DEBUG", ) - glob_group.add_argument('--verbose', - action='store_true', default=False, - help="Be verbose in the output", - ) glob_group.add_argument('--help', action='help', help="Show this help message and exit", ) return parser.parse_args() -def _init_moulinette(use_websocket=True, debug=False, verbose=False): +def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(LOGGERS_HANDLERS) - if not use_websocket and 'api' in handlers: - handlers.remove('api') - if verbose and 'console' not in handlers: - handlers.add('console') - root_handlers = handlers - set(['api']) - # Define loggers level - level = LOGGERS_LEVEL - api_level = API_LOGGER_LEVEL - if debug: - level = 'DEBUG' - api_level = 'DEBUG' + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) # Custom logging configuration logging = { @@ -122,7 +78,7 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): }, 'handlers': { 'api': { - 'level': api_level, + 'level': 'DEBUG' if debug else 'INFO', 'class': 'moulinette.interfaces.api.APIQueueHandler', }, 'file': { @@ -140,58 +96,36 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): }, 'loggers': { 'yunohost': { - 'level': level, - 'handlers': handlers, + 'level': 'DEBUG', + 'handlers': ['file', 'api'] + ['console'] if debug else [], 'propagate': False, }, 'moulinette': { - 'level': level, + 'level': 'DEBUG', 'handlers': [], 'propagate': True, }, - 'gnupg': { - 'level': 'INFO', - 'handlers': [], - 'propagate': False, - }, }, 'root': { - 'level': level, - 'handlers': root_handlers, + 'level': 'DEBUG', + 'handlers': ['file'] + ['console'] if debug else [], }, } - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=IN_DEVEL) + moulinette.init(logging_config=logging) def _retrieve_namespaces(): """Return the list of namespaces to load""" - ret = ['yunohost'] - for n in ActionsMap.get_namespaces(): - # Append YunoHost modules - if n.startswith('ynh_'): - ret.append(n) - return ret + extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] + return ['yunohost'] + extensions # Callbacks for additional routes -------------------------------------- def is_installed(): - """ - Check whether YunoHost is installed or not - - """ - installed = False - if os.path.isfile('/etc/yunohost/installed'): - installed = True - return { 'installed': installed } + """ Check whether YunoHost is installed or not """ + return { 'installed': os.path.isfile('/etc/yunohost/installed') } # Main action ---------------------------------------------------------- @@ -203,8 +137,9 @@ if __name__ == '__main__': # Run the server ret = moulinette.api( _retrieve_namespaces(), - host=opts.host, port=opts.port, routes={ - ('GET', '/installed'): is_installed, - }, use_cache=opts.use_cache, use_websocket=opts.use_websocket + host=opts.host, + port=opts.port, + routes={ ('GET', '/installed'): is_installed, }, + use_websocket=True ) sys.exit(ret) From 31e5f7e8b599fb25e27ebdd9b4a4e90f4a1fd62b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 8 Apr 2020 02:58:42 +0200 Subject: [PATCH 005/138] This arg ain't meaningful anymore? --- bin/yunohost | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 29a97e016..4e0ece1e4 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -39,10 +39,6 @@ def _parse_cli_args(): type=int, default=None, help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock", ) - parser.add_argument('--admin-password', - default=None, dest='password', metavar='PASSWORD', - help="The admin password to use to authenticate", - ) # deprecated arguments parser.add_argument('--plain', action='store_true', default=False, help=argparse.SUPPRESS @@ -167,7 +163,6 @@ if __name__ == '__main__': _retrieve_namespaces(), args, output_as=opts.output_as, - password=opts.password, timeout=opts.timeout, parser_kwargs={'top_parser': parser}, ) From 68c9244492166c94570fcc06e4c79c75b2d5186b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 04:34:49 +0200 Subject: [PATCH 006/138] Remove _die, simplify creation of log dir, make linter a bit happier --- bin/yunohost | 24 ++++++++++-------------- bin/yunohost-api | 16 ++++------------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 4e0ece1e4..b56666eb4 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -13,12 +13,12 @@ from moulinette.interfaces.cli import colorize, get_locale LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-cli.log' +# Create log directory +if not os.path.isdir(LOG_DIR): + os.makedirs(LOG_DIR, 0750) + # Initialization & helpers functions ----------------------------------- -def _die(message, title='Error:'): - """Print error message and exit""" - print('%s %s' % (colorize(title, 'red'), message)) - sys.exit(1) def _parse_cli_args(): """Parse additional arguments for the cli""" @@ -57,16 +57,10 @@ def _parse_cli_args(): return (parser, opts, args) + def _init_moulinette(debug=False, quiet=False): """Configure logging and initialize the moulinette""" - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Custom logging configuration logging = { 'version': 1, @@ -123,6 +117,7 @@ def _init_moulinette(debug=False, quiet=False): # Initialize moulinette moulinette.init(logging_config=logging) + def _retrieve_namespaces(): """Return the list of namespaces to load""" extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] @@ -138,7 +133,7 @@ if os.environ["PATH"] != default_path: if __name__ == '__main__': if os.geteuid() != 0: # since moulinette isn't initialized, we can't use m18n here - sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " \ + sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " "run as root or with sudo.\n") sys.exit(1) @@ -148,7 +143,7 @@ if __name__ == '__main__': # Check that YunoHost is installed allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] +' '+ args[1] not in allowed_if_not_installed)): + (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): from moulinette import m18n # Init i18n @@ -156,7 +151,8 @@ if __name__ == '__main__': m18n.set_locale(get_locale()) # Print error and exit - _die(m18n.n('yunohost_not_installed'), m18n.g('error')) + print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) + sys.exit(1) # Execute the action ret = moulinette.cli( diff --git a/bin/yunohost-api b/bin/yunohost-api index 7503d28ad..3185738f6 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -17,12 +17,11 @@ DEFAULT_PORT = 6787 LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Initialization & helpers functions ----------------------------------- +# Create log directory +if not os.path.isdir(LOG_DIR): + os.makedirs(LOG_DIR, 0750) -def _die(message, title='Error:'): - """Print error message and exit""" - print('%s %s' % (colorize(title, 'red'), message)) - sys.exit(1) +# Initialization & helpers functions ----------------------------------- def _parse_api_args(): """Parse main arguments for the api""" @@ -52,13 +51,6 @@ def _parse_api_args(): def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" - # Create log directory - if not os.path.isdir(LOG_DIR): - try: - os.makedirs(LOG_DIR, 0750) - except os.error as e: - _die(str(e)) - # Custom logging configuration logging = { 'version': 1, From f5c16737ebd44748225e7d17f60faf67c05d97db Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 05:15:08 +0200 Subject: [PATCH 007/138] More stuff to reduce complexity --- bin/yunohost | 7 +++++-- bin/yunohost-api | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index b56666eb4..173cbc1cb 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,9 +4,9 @@ import os import sys import argparse +import glob import moulinette -from moulinette.actionsmap import ActionsMap from moulinette.interfaces.cli import colorize, get_locale # Directory and file to be used by logging @@ -123,6 +123,7 @@ def _retrieve_namespaces(): extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] return ['yunohost'] + extensions + # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" if os.environ["PATH"] != default_path: @@ -154,9 +155,11 @@ if __name__ == '__main__': print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) sys.exit(1) + extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] + # Execute the action ret = moulinette.cli( - _retrieve_namespaces(), + ['yunohost'] + extensions, args, output_as=opts.output_as, timeout=opts.timeout, diff --git a/bin/yunohost-api b/bin/yunohost-api index 3185738f6..7a2119a08 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -4,10 +4,9 @@ import os import sys import argparse +import glob import moulinette -from moulinette.actionsmap import ActionsMap -from moulinette.interfaces.cli import colorize # Default server configuration DEFAULT_HOST = 'localhost' @@ -23,6 +22,7 @@ if not os.path.isdir(LOG_DIR): # Initialization & helpers functions ----------------------------------- + def _parse_api_args(): """Parse main arguments for the api""" parser = argparse.ArgumentParser(add_help=False, @@ -48,6 +48,7 @@ def _parse_api_args(): return parser.parse_args() + def _init_moulinette(debug=False): """Configure logging and initialize the moulinette""" @@ -107,31 +108,28 @@ def _init_moulinette(debug=False): # Initialize moulinette moulinette.init(logging_config=logging) -def _retrieve_namespaces(): - """Return the list of namespaces to load""" - extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] - return ['yunohost'] + extensions - - # Callbacks for additional routes -------------------------------------- + def is_installed(): """ Check whether YunoHost is installed or not """ - return { 'installed': os.path.isfile('/etc/yunohost/installed') } + return {'installed': os.path.isfile('/etc/yunohost/installed')} # Main action ---------------------------------------------------------- if __name__ == '__main__': opts = _parse_api_args() - _init_moulinette(opts.use_websocket, opts.debug, opts.verbose) + _init_moulinette(opts.debug) + + extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] # Run the server ret = moulinette.api( - _retrieve_namespaces(), + ['yunohost'] + extensions, host=opts.host, port=opts.port, - routes={ ('GET', '/installed'): is_installed, }, + routes={('GET', '/installed'): is_installed}, use_websocket=True ) sys.exit(ret) From 233c962710aec0e98f77936bde476fe662ca062f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 16:49:35 +0200 Subject: [PATCH 008/138] Hmpf idk another iteration to cleaning attempt --- bin/yunohost | 26 +++++++++++--------------- bin/yunohost-api | 23 +++++++++-------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 173cbc1cb..8c4b10d8d 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -7,19 +7,13 @@ import argparse import glob import moulinette -from moulinette.interfaces.cli import colorize, get_locale # Directory and file to be used by logging LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-cli.log' -# Create log directory -if not os.path.isdir(LOG_DIR): - os.makedirs(LOG_DIR, 0750) - # Initialization & helpers functions ----------------------------------- - def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) @@ -58,11 +52,13 @@ def _parse_cli_args(): return (parser, opts, args) -def _init_moulinette(debug=False, quiet=False): - """Configure logging and initialize the moulinette""" +def init(debug=False, quiet=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - # Custom logging configuration - logging = { + logdir = os.path.dirname(logfile) + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + moulinette.init(logging_config={ 'version': 1, 'disable_existing_loggers': True, 'formatters': { @@ -87,7 +83,7 @@ def _init_moulinette(debug=False, quiet=False): 'file': { 'class': 'logging.FileHandler', 'formatter': 'precise', - 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + 'filename': logfile, 'filters': ['action'], }, }, @@ -112,10 +108,8 @@ def _init_moulinette(debug=False, quiet=False): 'level': 'DEBUG', 'handlers': ['file', 'tty'] if debug else ['file'], }, - } + }) - # Initialize moulinette - moulinette.init(logging_config=logging) def _retrieve_namespaces(): @@ -139,7 +133,7 @@ if __name__ == '__main__': sys.exit(1) parser, opts, args = _parse_cli_args() - _init_moulinette(opts.debug, opts.quiet) + init(debug=opts.debug, quiet=opts.quiet) # Check that YunoHost is installed allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] @@ -147,6 +141,8 @@ if __name__ == '__main__': (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): from moulinette import m18n + from moulinette.interfaces.cli import colorize, get_locale + # Init i18n m18n.load_namespace('yunohost') m18n.set_locale(get_locale()) diff --git a/bin/yunohost-api b/bin/yunohost-api index 7a2119a08..0b6961a90 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -16,10 +16,6 @@ DEFAULT_PORT = 6787 LOG_DIR = '/var/log/yunohost' LOG_FILE = 'yunohost-api.log' -# Create log directory -if not os.path.isdir(LOG_DIR): - os.makedirs(LOG_DIR, 0750) - # Initialization & helpers functions ----------------------------------- @@ -49,11 +45,13 @@ def _parse_api_args(): return parser.parse_args() -def _init_moulinette(debug=False): - """Configure logging and initialize the moulinette""" +def init_api(debug=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - # Custom logging configuration - logging = { + logdir = os.path.dirname(logfile) + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + moulinette.init(logging_config={ 'version': 1, 'disable_existing_loggers': True, 'formatters': { @@ -77,7 +75,7 @@ def _init_moulinette(debug=False): 'file': { 'class': 'logging.handlers.WatchedFileHandler', 'formatter': 'precise', - 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + 'filename': logfile, 'filters': ['action'], }, 'console': { @@ -103,10 +101,7 @@ def _init_moulinette(debug=False): 'level': 'DEBUG', 'handlers': ['file'] + ['console'] if debug else [], }, - } - - # Initialize moulinette - moulinette.init(logging_config=logging) + }) # Callbacks for additional routes -------------------------------------- @@ -120,7 +115,7 @@ def is_installed(): if __name__ == '__main__': opts = _parse_api_args() - _init_moulinette(opts.debug) + init_api(opts.debug) extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] From dee08a16feb5358122bb424f0da3458952f03e85 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 18:41:54 +0200 Subject: [PATCH 009/138] Move moulinette initialization and other stuff to src/yunohost/__init__.py --- bin/yunohost | 113 +++------------------- bin/yunohost-api | 92 +----------------- src/yunohost/__init__.py | 202 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 191 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 8c4b10d8d..546d2d913 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,34 +4,29 @@ import os import sys import argparse -import glob -import moulinette +sys.path.insert(0, "/usr/lib/moulinette/") +import yunohost -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-cli.log' - -# Initialization & helpers functions ----------------------------------- def _parse_cli_args(): """Parse additional arguments for the cli""" parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--output-as', choices=['json', 'plain', 'none'], default=None, - help="Output result in another format", + help="Output result in another format" ) parser.add_argument('--debug', action='store_true', default=False, - help="Log and print debug messages", + help="Log and print debug messages" ) parser.add_argument('--quiet', action='store_true', default=False, - help="Don't produce any output", + help="Don't produce any output" ) parser.add_argument('--timeout', type=int, default=None, - help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock", + help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock" ) # deprecated arguments parser.add_argument('--plain', @@ -52,72 +47,6 @@ def _parse_cli_args(): return (parser, opts, args) -def init(debug=False, quiet=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - - logdir = os.path.dirname(logfile) - if not os.path.isdir(logdir): - os.makedirs(logdir, 0750) - - moulinette.init(logging_config={ - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'tty-debug': { - 'format': '%(relativeCreated)-4d %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'tty': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.cli.TTYHandler', - 'formatter': 'tty-debug' if debug else '', - }, - 'file': { - 'class': 'logging.FileHandler', - 'formatter': 'precise', - 'filename': logfile, - 'filters': ['action'], - }, - }, - 'loggers': { - 'yunohost': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if not quiet else ['file'], - 'propagate': False, - }, - 'moulinette': { - 'level': 'DEBUG', - 'handlers': [], - 'propagate': True, - }, - 'moulinette.interface': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if not quiet else ['file'], - 'propagate': False, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['file', 'tty'] if debug else ['file'], - }, - }) - - - -def _retrieve_namespaces(): - """Return the list of namespaces to load""" - extensions = [n for n in ActionsMap.get_namespaces() if n.startswith('ynh_')] - return ['yunohost'] + extensions - - # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" if os.environ["PATH"] != default_path: @@ -127,38 +56,18 @@ if os.environ["PATH"] != default_path: if __name__ == '__main__': if os.geteuid() != 0: - # since moulinette isn't initialized, we can't use m18n here sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " "run as root or with sudo.\n") sys.exit(1) parser, opts, args = _parse_cli_args() - init(debug=opts.debug, quiet=opts.quiet) - - # Check that YunoHost is installed - allowed_if_not_installed = ['tools postinstall', 'backup restore', 'log display'] - if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_installed)): - - from moulinette import m18n - from moulinette.interfaces.cli import colorize, get_locale - - # Init i18n - m18n.load_namespace('yunohost') - m18n.set_locale(get_locale()) - - # Print error and exit - print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) - sys.exit(1) - - extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] # Execute the action - ret = moulinette.cli( - ['yunohost'] + extensions, - args, + yunohost.cli( + debug=opts.debug, + quiet=opts.quiet, output_as=opts.output_as, timeout=opts.timeout, - parser_kwargs={'top_parser': parser}, + args=args, + parser=parser ) - sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index 0b6961a90..cc849590a 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -1,23 +1,16 @@ #! /usr/bin/python # -*- coding: utf-8 -*- -import os import sys import argparse -import glob -import moulinette +sys.path.insert(0, "/usr/lib/moulinette/") +import yunohost # Default server configuration DEFAULT_HOST = 'localhost' DEFAULT_PORT = 6787 -# Directory and file to be used by logging -LOG_DIR = '/var/log/yunohost' -LOG_FILE = 'yunohost-api.log' - -# Initialization & helpers functions ----------------------------------- - def _parse_api_args(): """Parse main arguments for the api""" @@ -45,86 +38,7 @@ def _parse_api_args(): return parser.parse_args() -def init_api(debug=False, logfile='%s/%s' % (LOG_DIR, LOG_FILE)): - - logdir = os.path.dirname(logfile) - if not os.path.isdir(logdir): - os.makedirs(logdir, 0750) - - moulinette.init(logging_config={ - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'console': { - 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'api': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.api.APIQueueHandler', - }, - 'file': { - 'class': 'logging.handlers.WatchedFileHandler', - 'formatter': 'precise', - 'filename': logfile, - 'filters': ['action'], - }, - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'console', - 'stream': 'ext://sys.stdout', - 'filters': ['action'], - }, - }, - 'loggers': { - 'yunohost': { - 'level': 'DEBUG', - 'handlers': ['file', 'api'] + ['console'] if debug else [], - 'propagate': False, - }, - 'moulinette': { - 'level': 'DEBUG', - 'handlers': [], - 'propagate': True, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['file'] + ['console'] if debug else [], - }, - }) - -# Callbacks for additional routes -------------------------------------- - - -def is_installed(): - """ Check whether YunoHost is installed or not """ - return {'installed': os.path.isfile('/etc/yunohost/installed')} - - -# Main action ---------------------------------------------------------- - if __name__ == '__main__': opts = _parse_api_args() - init_api(opts.debug) - - extensions = [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] - # Run the server - ret = moulinette.api( - ['yunohost'] + extensions, - host=opts.host, - port=opts.port, - routes={('GET', '/installed'): is_installed}, - use_websocket=True - ) - sys.exit(ret) + yunohost.api(debug=opts.debug, host=opts.host, port=opts.port) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index e69de29bb..06e3d773d 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -0,0 +1,202 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import glob + +import moulinette +from moulinette.utils.log import configure_logging + + +def is_installed(): + return os.path.isfile('/etc/yunohost/installed') + + +def cli(debug, quiet, output_as, timeout, args, parser): + + init_logging(interface="cli", debug=debug, quiet=quiet) + + # Check that YunoHost is installed + if not is_installed(): + check_command_is_valid_before_postinstall(args) + + ret = moulinette.cli( + ['yunohost'] + extensions(), + args, + output_as=output_as, + timeout=timeout, + parser_kwargs={'top_parser': parser}, + ) + sys.exit(ret) + + +def api(debug, host, port): + + init_logging(debug=debug) + + def is_installed_api(): + return {'installed': is_installed()} + + # FIXME : someday, maybe find a way to disable route /postinstall if + # postinstall already done ... + + ret = moulinette.api( + ['yunohost'] + extensions(), + host=host, + port=port, + routes={('GET', '/installed'): is_installed_api}, + use_websocket=True + ) + sys.exit(ret) + + +def extensions(): + # This is probably not used anywhere, but the actionsmap and code can be + # extended by creating such files that contain bits of actionmap... + return [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] + + +def check_command_is_valid_before_postinstall(args): + + allowed_if_not_postinstalled = ['tools postinstall', + 'tools versions', + 'backup list', + 'backup restore', + 'log display'] + + if (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_postinstalled)): + + # This function is called before m18n is initialized, so we only initialized + # the specific bit to be able to call m18n.n/g()... + from moulinette import m18n + from moulinette.interfaces.cli import colorize, get_locale + + # Init i18n + m18n.load_namespace('yunohost') + m18n.set_locale(get_locale()) + + print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) + sys.exit(1) + + +def init_logging(interface="cli", + debug=False, + quiet=False, + logdir="/var/log/yunohost"): + + logfile = os.path.join(logdir, "yunohost-%s.log" % interface) + + if not os.path.isdir(logdir): + os.makedirs(logdir, 0750) + + # ####################################################################### # + # Logging configuration for CLI (or any other interface than api...) # + # ####################################################################### # + if interface != "api": + configure_logging({ + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'tty-debug': { + 'format': '%(relativeCreated)-4d %(fmessage)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + }, + 'filters': { + 'action': { + '()': 'moulinette.utils.log.ActionFilter', + }, + }, + 'handlers': { + 'tty': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'moulinette.interfaces.cli.TTYHandler', + 'formatter': 'tty-debug' if debug else '', + }, + 'file': { + 'class': 'logging.FileHandler', + 'formatter': 'precise', + 'filename': logfile, + 'filters': ['action'], + }, + }, + 'loggers': { + 'yunohost': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], + 'propagate': False, + }, + 'moulinette': { + 'level': 'DEBUG', + 'handlers': [], + 'propagate': True, + }, + 'moulinette.interface': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if not quiet else ['file'], + 'propagate': False, + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['file', 'tty'] if debug else ['file'], + }, + }) + # ####################################################################### # + # Logging configuration for API # + # ####################################################################### # + else: + configure_logging({ + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'console': { + 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + }, + }, + 'filters': { + 'action': { + '()': 'moulinette.utils.log.ActionFilter', + }, + }, + 'handlers': { + 'api': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'moulinette.interfaces.api.APIQueueHandler', + }, + 'file': { + 'class': 'logging.handlers.WatchedFileHandler', + 'formatter': 'precise', + 'filename': logfile, + 'filters': ['action'], + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', + 'stream': 'ext://sys.stdout', + 'filters': ['action'], + }, + }, + 'loggers': { + 'yunohost': { + 'level': 'DEBUG', + 'handlers': ['file', 'api'] + ['console'] if debug else [], + 'propagate': False, + }, + 'moulinette': { + 'level': 'DEBUG', + 'handlers': [], + 'propagate': True, + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['file'] + ['console'] if debug else [], + }, + }) From 69a520657c13aba8964c659963413cd8a1e6a951 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 18:59:44 +0200 Subject: [PATCH 010/138] Add init_i18n for convenience when using yunohost as a lib... --- src/yunohost/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 06e3d773d..8a23f8e00 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -6,8 +6,9 @@ import sys import glob import moulinette +from moulinette import m18n from moulinette.utils.log import configure_logging - +from moulinette.interfaces.cli import colorize, get_locale def is_installed(): return os.path.isfile('/etc/yunohost/installed') @@ -66,20 +67,18 @@ def check_command_is_valid_before_postinstall(args): 'log display'] if (len(args) < 2 or (args[0] + ' ' + args[1] not in allowed_if_not_postinstalled)): - - # This function is called before m18n is initialized, so we only initialized - # the specific bit to be able to call m18n.n/g()... - from moulinette import m18n - from moulinette.interfaces.cli import colorize, get_locale - - # Init i18n - m18n.load_namespace('yunohost') - m18n.set_locale(get_locale()) - + init_i18n() print(colorize(m18n.g('error'), 'red') + " " + m18n.n('yunohost_not_installed')) sys.exit(1) +def init_i18n(): + # This should only be called when not willing to go through moulinette.cli + # or moulinette.api but still willing to call m18n.n/g... + m18n.load_namespace('yunohost') + m18n.set_locale(get_locale()) + + def init_logging(interface="cli", debug=False, quiet=False, From 7df8e8421df39bbed03f1fc7cb53102876a400a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Apr 2020 04:03:02 +0200 Subject: [PATCH 011/138] Forgot to set interface as api for init logging --- src/yunohost/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 8a23f8e00..e2821f558 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -34,7 +34,7 @@ def cli(debug, quiet, output_as, timeout, args, parser): def api(debug, host, port): - init_logging(debug=debug) + init_logging(interface="api", debug=debug) def is_installed_api(): return {'installed': is_installed()} From 84e39a416a69c7974e36cf32af1d4e40e376bf03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 04:43:04 +0200 Subject: [PATCH 012/138] Add an 'init' helper for scripts/tests to initialize everything needed... --- src/yunohost/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index e2821f558..3e7ddb496 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -72,6 +72,19 @@ def check_command_is_valid_before_postinstall(args): sys.exit(1) +def init(interface="cli", debug=False, quiet=False, logdir="/var/log/yunohost"): + """ + This is a small util function ONLY meant to be used to initialize a Yunohost + context when ran from tests or from scripts. + """ + init_logging(interface=interface, debug=debug, quiet=quiet, logdir=logdir) + init_i18n() + from moulinette.core import MoulinetteLock + lock = MoulinetteLock("yunohost", timeout=30) + lock.acquire() + return lock + + def init_i18n(): # This should only be called when not willing to go through moulinette.cli # or moulinette.api but still willing to call m18n.n/g... From 96e115c6091317d4d77e324e4f40f9e659648e03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 04:43:29 +0200 Subject: [PATCH 013/138] Use 'init' helper from tests! --- src/yunohost/tests/conftest.py | 67 ++-------------------------------- 1 file changed, 4 insertions(+), 63 deletions(-) diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 073c880f8..69f8bcfed 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -1,12 +1,11 @@ import os import pytest import sys -import moulinette +import moulinette from moulinette import m18n from yunohost.utils.error import YunohostError from contextlib import contextmanager - sys.path.append("..") @@ -68,65 +67,7 @@ moulinette.core.Moulinette18n.n = new_m18nn def pytest_cmdline_main(config): - """Configure logging and initialize the moulinette""" - # Define loggers handlers - handlers = set(['tty']) - root_handlers = set(handlers) - # Define loggers level - level = 'DEBUG' - if config.option.yunodebug: - tty_level = 'DEBUG' - else: - tty_level = 'INFO' - - # Custom logging configuration - logging = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'tty-debug': { - 'format': '%(relativeCreated)-4d %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' - }, - }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', - }, - }, - 'handlers': { - 'tty': { - 'level': tty_level, - 'class': 'moulinette.interfaces.cli.TTYHandler', - 'formatter': '', - }, - }, - 'loggers': { - 'yunohost': { - 'level': level, - 'handlers': handlers, - 'propagate': False, - }, - 'moulinette': { - 'level': level, - 'handlers': [], - 'propagate': True, - }, - 'moulinette.interface': { - 'level': level, - 'handlers': handlers, - 'propagate': False, - }, - }, - 'root': { - 'level': level, - 'handlers': root_handlers, - }, - } - - # Initialize moulinette - moulinette.init(logging_config=logging, _from_source=False) - moulinette.m18n.load_namespace('yunohost') + sys.path.insert(0, "/usr/lib/moulinette/") + import yunohost + yunohost.init(debug=config.option.yunodebug) From d8be90165c2345ec956e24f789a7215f923679dd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 May 2020 06:11:53 +0200 Subject: [PATCH 014/138] Propagate changes from moulinette/simplify-interface-init --- src/yunohost/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 3e7ddb496..810d6127a 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -3,13 +3,13 @@ import os import sys -import glob import moulinette from moulinette import m18n from moulinette.utils.log import configure_logging from moulinette.interfaces.cli import colorize, get_locale + def is_installed(): return os.path.isfile('/etc/yunohost/installed') @@ -23,11 +23,10 @@ def cli(debug, quiet, output_as, timeout, args, parser): check_command_is_valid_before_postinstall(args) ret = moulinette.cli( - ['yunohost'] + extensions(), args, output_as=output_as, timeout=timeout, - parser_kwargs={'top_parser': parser}, + top_parser=parser ) sys.exit(ret) @@ -43,21 +42,13 @@ def api(debug, host, port): # postinstall already done ... ret = moulinette.api( - ['yunohost'] + extensions(), host=host, port=port, routes={('GET', '/installed'): is_installed_api}, - use_websocket=True ) sys.exit(ret) -def extensions(): - # This is probably not used anywhere, but the actionsmap and code can be - # extended by creating such files that contain bits of actionmap... - return [f.split('/')[-1][:-4] for f in glob.glob("/usr/share/moulinette/actionsmap/ynh_*.yml")] - - def check_command_is_valid_before_postinstall(args): allowed_if_not_postinstalled = ['tools postinstall', From 7ccd6e1348321491ebcb2c6afec7be1de395e926 Mon Sep 17 00:00:00 2001 From: Julien Rabier Date: Mon, 11 May 2020 21:37:17 +0000 Subject: [PATCH 015/138] fix destination concurrency Hi, Postfix has this very peculiar behavior where the target of some config keys changes depending on the value. Here, if `smtp_destination_concurrency_limit` is set to 1, then according to http://www.postfix.org/postconf.5.html#default_destination_concurrency_limit it doesn't mean "1 concurrent mail per domain, but per recipiend address". So, if set to 1, it means we can send any volume of e-mails concurrently (with a 5s delay) if all recipient addresses are different. In order to avoid this, we should increase the value to restore the expected behavior (concurrency per domain, not per recipient). --- data/templates/postfix/main.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index 61cbfa2e6..18e457a76 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -170,7 +170,7 @@ smtpd_milters = inet:localhost:11332 milter_default_action = accept # Avoid to send simultaneously too many emails -smtp_destination_concurrency_limit = 1 +smtp_destination_concurrency_limit = 2 default_destination_rate_delay = 5s # Avoid email adress scanning From b970cba6c4e6e3d4a656a447bc086df333179c1b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jun 2020 17:37:54 +0200 Subject: [PATCH 016/138] Following previous regenconf fixes, the regen-conf was now outputing a whole lot of 'forget-about-it' when e.g. running regen-conf for nginx because of files flagged as 'should-not-exists'... so this is a dirty workaround --- src/yunohost/regenconf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 4aad5f921..6445248bd 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -236,6 +236,8 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run conf_files[f] = None # End discussion about stale file hashes + force_update_hashes_for_this_category = False + for system_path, pending_path in conf_files.items(): logger.debug("processing pending conf '%s' to system conf '%s'", pending_path, system_path) @@ -284,7 +286,8 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run os.remove(pending_path) conf_hashes[system_path] = None conf_status = 'forget-about-it' - regenerated = True + force_update_hashes_for_this_category = True + continue elif not saved_hash or force: if force: logger.debug("> system conf has been manually removed") @@ -377,7 +380,7 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run else: logger.success(m18n.n('regenconf_would_be_updated', category=category)) - if succeed_regen and not dry_run: + if (succeed_regen or force_update_hashes_for_this_category) and not dry_run: _update_conf_hashes(category, conf_hashes) # Append the category results From 98f144e61fc30d0f8d7f67c4518d7f7f4973c5b3 Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Fri, 5 Jun 2020 20:41:49 +0000 Subject: [PATCH 017/138] Translated using Weblate (Catalan) Currently translated at 100.0% (655 of 655 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index f6802f216..ea801fe53 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -651,5 +651,19 @@ "diagnosis_swap_tip": "Vigileu i tingueu en compte que els servidor està allotjant memòria d'intercanvi en una targeta SD o en l'emmagatzematge SSD, això pot reduir dràsticament l'esperança de vida del dispositiu.", "restore_already_installed_apps": "No s'han pogut restaurar les següents aplicacions perquè ja estan instal·lades: {apps}", "app_packaging_format_not_supported": "No es pot instal·lar aquesta aplicació ja que el format del paquet no és compatible amb la versió de YunoHost del sistema. Hauríeu de considerar actualitzar el sistema.", - "diagnosis_dns_try_dyndns_update_force": "La configuració DNS d'aquest domini hauria de ser gestionada automàticament per YunoHost. Si aquest no és el cas, podeu intentar forçar-ne l'actualització utilitzant yunohost dyndns update --force." + "diagnosis_dns_try_dyndns_update_force": "La configuració DNS d'aquest domini hauria de ser gestionada automàticament per YunoHost. Si aquest no és el cas, podeu intentar forçar-ne l'actualització utilitzant yunohost dyndns update --force.", + "migration_0015_cleaning_up": "Netejant la memòria cau i els paquets que ja no són necessaris…", + "migration_0015_specific_upgrade": "Començant l'actualització dels paquets del sistema que s'han d'actualitzar de forma independent…", + "migration_0015_modified_files": "Tingueu en compte que s'han trobat els següents fitxers que es van modificar manualment i podria ser que es sobreescriguin durant l'actualització: {manually_modified_files}", + "migration_0015_problematic_apps_warning": "Tingueu en compte que s'han trobat les següents aplicacions que podrien ser problemàtiques. Sembla que aquestes aplicacions no s'han instal·lat des del catàleg d'aplicacions de YunoHost, o no estan marcades com «funcionant». En conseqüència, no es pot garantir que segueixin funcionant després de l'actualització: {problematic_apps}", + "migration_0015_general_warning": "Tingueu en compte que aquesta migració és una operació delicada. L'equip de YunoHost ha fet tots els possibles per revisar i testejar, però tot i això podria ser que la migració trenqui alguna part del sistema o algunes aplicacions.\n\nPer tant, està recomana:\n - Fer una còpia de seguretat de totes les dades o aplicacions crítiques. Més informació a https://yunohost.org/backup;\n - Ser pacient un cop comenci la migració: en funció de la connexió Internet i del maquinari, podria estar unes hores per actualitzar-ho tot.", + "migration_0015_system_not_fully_up_to_date": "El sistema no està completament al dia. Heu de fer una actualització normal abans de fer la migració a Buster.", + "migration_0015_not_enough_free_space": "Hi ha poc espai lliure a /var/! HI hauria d'haver un mínim de 1GB lliure per poder fer aquesta migració.", + "migration_0015_not_stretch": "La distribució actual de Debian no és Stretch!", + "migration_0015_yunohost_upgrade": "Començant l'actualització del nucli de YunoHost…", + "migration_0015_still_on_stretch_after_main_upgrade": "Alguna cosa ha anat malament durant la actualització principal, sembla que el sistema encara està en Debian Stretch", + "migration_0015_main_upgrade": "Començant l'actualització principal…", + "migration_0015_patching_sources_list": "Apedaçament de source.lists…", + "migration_0015_start": "Començant la migració a Buster", + "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x" } From e8cab1feeb793bb1729ded789b26a643e586a7f4 Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Sat, 6 Jun 2020 21:31:55 +0000 Subject: [PATCH 018/138] Translated using Weblate (Catalan) Currently translated at 100.0% (656 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index ea801fe53..f29a65c43 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -665,5 +665,6 @@ "migration_0015_main_upgrade": "Començant l'actualització principal…", "migration_0015_patching_sources_list": "Apedaçament de source.lists…", "migration_0015_start": "Començant la migració a Buster", - "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x" + "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x", + "regenconf_need_to_explicitly_specify_ssh": "La configuració ssh ha estat modificada manualment, però heu d'especificar explícitament la categoria «ssh» amb --force per fer realment els canvis." } From 19f3f865104d86189e692e3308cb71f6de887c76 Mon Sep 17 00:00:00 2001 From: ppr Date: Wed, 10 Jun 2020 16:51:15 +0000 Subject: [PATCH 019/138] Translated using Weblate (French) Currently translated at 99.4% (652 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index bd32d7f08..b24f13544 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -168,7 +168,7 @@ "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} n’est pas sur le point d’expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)", "certmanager_domain_http_not_working": "Le domaine {domain:s} ne semble pas être accessible via HTTP. Merci de vérifier la catégorie 'Web' dans le diagnostic pour plus d'informations. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)", "certmanager_error_no_A_record": "Aucun enregistrement DNS 'A' n’a été trouvé pour {domain:s}. Vous devez faire pointer votre nom de domaine vers votre machine pour être en mesure d’installer un certificat Let’s Encrypt ! (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", - "certmanager_domain_dns_ip_differs_from_public_ip": "L’enregistrement DNS 'A' du domaine {domain:s} est différent de l’adresse IP de ce serveur. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", + "certmanager_domain_dns_ip_differs_from_public_ip": "L'enregistrement DNS du domaine {domain:s} est différent de l’adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans le diagnostic. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", "certmanager_cannot_read_cert": "Quelque chose s’est mal passé lors de la tentative d’ouverture du certificat actuel pour le domaine {domain:s} (fichier : {file:s}), la cause est : {reason:s}", "certmanager_cert_install_success_selfsigned": "Le certificat auto-signé est maintenant installé pour le domaine « {domain:s} »", "certmanager_cert_install_success": "Le certificat Let’s Encrypt est maintenant installé pour le domaine « {domain:s} »", @@ -535,7 +535,7 @@ "diagnosis_ip_broken_resolvconf": "La résolution du nom de domaine semble être cassée sur votre serveur, ce qui semble lié au fait que /etc/resolv.conf ne pointe pas vers 127.0.0.1.", "diagnosis_dns_good_conf": "Les enregistrements DNS sont correctement configurés pour le domaine {domain} (catégorie {category})", "diagnosis_dns_bad_conf": "Certains enregistrements DNS sont manquants ou incorrects pour le domaine {domain} (catégorie {category})", - "diagnosis_dns_discrepancy": "Cet enregistrement DNS ne semble pas correspondre à la configuration recommandée :
Type : {type}
Nom : {name}
Valeur actuelle : {current}
Valeur attendue : {value}", + "diagnosis_dns_discrepancy": "Cet enregistrement DNS ne semble pas correspondre à la configuration recommandée :
Type : {type}
Nom : {name}
La valeur actuelle est : {current}
La valeur attendue est : {value}", "diagnosis_services_bad_status": "Le service {service} est {status} :-(", "diagnosis_diskusage_verylow": "L'espace de stockage {mountpoint} (sur l’appareil {device}) ne dispose que de {free} ({free_percent}%) espace restant (sur {total}). Vous devriez vraiment envisager de nettoyer de l’espace !", "diagnosis_diskusage_low": "L'espace de stockage {mountpoint} (sur l'appareil {device}) ne dispose que de {free} ({free_percent}%) espace restant (sur {total}). Faites attention.", @@ -649,5 +649,23 @@ "diagnosis_domain_expiration_error": "Certains domaines vont expirer TRÈS PROCHAINEMENT !", "diagnosis_domain_expires_in": "{domain} expire dans {days} jours.", "certmanager_domain_not_diagnosed_yet": "Il n'y a pas encore de résultat de diagnostic pour le domaine %s. Merci de relancer un diagnostic pour les catégories 'Enregistrements DNS' et 'Web' dans la section Diagnostique pour vérifier si le domaine est prêt pour Let's Encrypt. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)", - "diagnosis_swap_tip": "Merci d'être prudent et conscient que si vous hébergez une partition SWAP sur une carte SD ou un disque SSD, cela risque de réduire drastiquement l’espérance de vie du périphérique." + "diagnosis_swap_tip": "Merci d'être prudent et conscient que si vous hébergez une partition SWAP sur une carte SD ou un disque SSD, cela risque de réduire drastiquement l’espérance de vie du périphérique.", + "restore_already_installed_apps": "Les applications suivantes ne peuvent pas être restaurées car elles sont déjà installées : {apps}", + "regenconf_need_to_explicitly_specify_ssh": "La configuration de ssh a été modifiée manuellement. Vous devez explicitement indiquer la mention --force à \"ssh\" pour appliquer les changements.", + "migration_0015_cleaning_up": "Nettoyage du cache et des paquets qui ne sont plus utiles …", + "migration_0015_specific_upgrade": "Commencement de la mise à jour des paquets du système qui doivent être mis à jour séparément …", + "migration_0015_modified_files": "Veuillez noter que les fichiers suivants ont été modifiés manuellement et pourraient être écrasés à la suite de la mise à niveau : {manually_modified_files}", + "migration_0015_problematic_apps_warning": "Veuillez noter que des applications qui peuvent poser problèmes ont été détectées. Il semble qu'elles n'aient pas été installées à partir du catalogue d'applications YunoHost, ou bien qu'elles ne soient pas signalées comme \"fonctionnelles\". Par conséquent, il n'est pas possible de garantir que les applications suivantes fonctionneront encore après la mise à niveau : {problematic_apps}", + "migration_0015_general_warning": "Veuillez noter que cette migration est une opération délicate. L'équipe YunoHost a fait de son mieux pour la revérifier et la tester, mais la migration pourrait quand même casser des éléments du système ou de ses applications.\n\nIl est donc recommandé :\n…- de faire une sauvegarde de toute donnée ou application critique. Plus d'informations ici https://yunohost.org/backup ;\n…- d'être patient après le lancement de la migration. Selon votre connexion internet et votre matériel, la mise à niveau peut prendre jusqu'à quelques heures.", + "migration_0015_system_not_fully_up_to_date": "Votre système n'est pas entièrement à jour. Veuillez effectuer une mise à jour normale avant de lancer la migration vers Buster.", + "migration_0015_not_enough_free_space": "L'espace libre est très faible dans /var/ ! Vous devriez avoir au moins 1 Go de libre pour effectuer cette migration.", + "migration_0015_not_stretch": "La distribution Debian actuelle n'est pas Stretch !", + "migration_0015_yunohost_upgrade": "Lancement de la mise à jour de YunoHost …", + "migration_0015_still_on_stretch_after_main_upgrade": "Quelque chose s'est mal passé lors de la mise à niveau, le système semble toujours être sous Debian Stretch", + "migration_0015_main_upgrade": "Début de la mise à niveau générale …", + "migration_0015_patching_sources_list": "Mise à jour du fichier sources.lists …", + "migration_0015_start": "Démarrage de la migration vers Buster", + "migration_description_0015_migrate_to_buster": "Mise à niveau du système vers Debian Buster et YunoHost 4.x", + "diagnosis_dns_try_dyndns_update_force": "La configuration DNS de ce domaine devrait être automatiquement gérée par Yunohost. Si ce n'est pas le cas, vous pouvez essayer de forcer une mise à jour en utilisant yunohost dyndns update --force.", + "app_packaging_format_not_supported": "Cette application ne peut pas être installée car son format n'est pas pris en charge par votre version de Yunohost. Vous devriez probablement envisager de mettre à jour votre système." } From 868ffad7d1bcadafdf036489630d4de9909d7a4c Mon Sep 17 00:00:00 2001 From: Yasss Gurl Date: Tue, 16 Jun 2020 12:33:20 +0000 Subject: [PATCH 020/138] Translated using Weblate (German) Currently translated at 33.8% (222 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/de/ --- locales/de.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/de.json b/locales/de.json index 0c5b031ce..fab9eaf2c 100644 --- a/locales/de.json +++ b/locales/de.json @@ -340,5 +340,6 @@ "diagnosis_ip_weird_resolvconf": "DNS Auflösung scheint zu funktionieren, aber sei vorsichtig wenn du eine eigene /etc/resolv.conf verwendest.", "diagnosis_display_tip": "Um die gefundenen Probleme zu sehen, kannst Du zum Diagnose-Bereich des webadmin gehen, oder 'yunohost diagnosis show --issues' in der Kommandozeile ausführen.", "backup_archive_corrupted": "Das Backup-Archiv '{archive}' scheint beschädigt: {error}", - "backup_archive_cant_retrieve_info_json": "Die Informationen für das Archiv '{archive}' konnten nicht geladen werden... Die Datei info.json wurde nicht gefunden (oder ist kein gültiges json)." + "backup_archive_cant_retrieve_info_json": "Die Informationen für das Archiv '{archive}' konnten nicht geladen werden... Die Datei info.json wurde nicht gefunden (oder ist kein gültiges json).", + "app_packaging_format_not_supported": "Diese App kann nicht installiert werden da das Paketformat nicht von der Yunohost-Version unterstützt wird. Denken Sie darüber nach das System zu aktualisieren." } From 35eec1eea3959220c36d6b23436ac51703304d9b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 18 Jun 2020 15:08:51 +0200 Subject: [PATCH 021/138] Move the general regen_conf after we regen ssh, otherwise there's a non-relevant warning displayed because the regen-conf tells to rerun ssh specifically which is already done... --- src/yunohost/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 7cdcc8641..c6ea3221a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -368,7 +368,6 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, service_enable("yunohost-firewall") service_start("yunohost-firewall") - regen_conf(force=True) regen_conf(names=["ssh"], force=True) # Restore original ssh conf, as chosen by the @@ -384,6 +383,8 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, if os.path.exists(original_sshd_conf): os.rename(original_sshd_conf, '/etc/ssh/sshd_config') + regen_conf(force=True) + logger.success(m18n.n('yunohost_configured')) logger.warning(m18n.n('yunohost_postinstall_end_tip')) From 804f4b3e21481407fe2b6d1c39469a94fb837605 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 6 Jun 2020 16:07:18 +0200 Subject: [PATCH 022/138] [fix] lower domain Lower domain to avoid some edge cases issues See: https://forum.yunohost.org/t/invalid-domain-causes-diagnosis-web-to-fail-fr-on-demand/11765 --- src/yunohost/domain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index acd4a1131..e1d2b1649 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -89,6 +89,10 @@ def domain_add(operation_logger, domain, dyndns=False): raise YunohostError('domain_exists') operation_logger.start() + + # Lower domain to avoid some edge cases issues + # See: https://forum.yunohost.org/t/invalid-domain-causes-diagnosis-web-to-fail-fr-on-demand/11765 + domain = domain.lower() # DynDNS domain if dyndns: From cd115ed83a283f129369af45ba105a0de65462fe Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 10 Jun 2020 15:19:53 +0200 Subject: [PATCH 023/138] dirmngr is needed for apt-key / gpg ... usually installed but it's only a Recommends so sometimes it's not >.> --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index fdbf4c98b..96b5bbb08 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python:Depends}, ${misc:Depends} , python-psutil, python-requests, python-dnspython, python-openssl , python-miniupnpc, python-dbus, python-jinja2 , python-toml, python-packaging - , apt, apt-transport-https + , apt, apt-transport-https, dirmngr , nginx, nginx-extras (>=1.6.2) , php-fpm, php-ldap, php-intl , mariadb-server, php-mysql | php-mysqlnd From 0f0194beff031b3e181fb3f5a14eba086c8bb4cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 11 Jun 2020 22:48:41 +0200 Subject: [PATCH 024/138] Fix error / debug message when diagnosis ain't happy when renewing certs --- src/yunohost/certificate.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 4b5adb754..882e37863 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -369,12 +369,11 @@ def certificate_renew(domain_list, force=False, no_checks=False, email=False, st if not no_checks: try: _check_domain_is_ready_for_ACME(domain) - except: - msg = "Certificate renewing for %s failed !" % (domain) - logger.error(msg) + except Exception as e: + logger.error(e) if email: logger.error("Sending email with details to root ...") - _email_renewing_failed(domain, msg) + _email_renewing_failed(domain, e) continue logger.info( From d615546b18faec992f1ad737ab4778d06d260e46 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 Jun 2020 19:01:57 +0200 Subject: [PATCH 025/138] Add yunohost version to logs metadata --- src/yunohost/log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 5e3a1f411..2b5f91e1b 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -35,6 +35,7 @@ from logging import FileHandler, getLogger, Formatter from moulinette import m18n, msettings from moulinette.core import MoulinetteError from yunohost.utils.error import YunohostError +from yunohost.utils.packages import get_ynh_package_version from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, read_yaml @@ -456,6 +457,7 @@ class OperationLogger(object): data = { 'started_at': self.started_at, 'operation': self.operation, + 'yunohost_version': get_ynh_package_version("yunohost")["version"], } if self.related_to is not None: data['related_to'] = self.related_to From 38704cbae49d0b4964bfcf19075818cfba3b182d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 Jun 2020 19:22:01 +0200 Subject: [PATCH 026/138] Tweak log filter logic to be able to apply it during --share as well --- src/yunohost/log.py | 42 +++++++++++++++++++++++------------------ src/yunohost/service.py | 7 +------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 2b5f91e1b..875d4e49d 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -136,6 +136,26 @@ def log_display(path, number=None, share=False, filter_irrelevant=False): share """ + if filter_irrelevant: + filters = [ + r"set [+-]x$", + r"set [+-]o xtrace$", + r"local \w+$", + r"local legacy_args=.*$", + r".*Helper used in legacy mode.*", + r"args_array=.*$", + r"local -A args_array$", + r"ynh_handle_getopts_args", + r"ynh_script_progression" + ] + else: + filters = [] + + def _filter_lines(lines, filters=[]): + + filters = [re.compile(f) for f in filters] + return [l for l in lines if not any(f.search(l.strip()) for f in filters)] + # Normalize log/metadata paths and filenames abs_path = path log_path = None @@ -174,7 +194,8 @@ def log_display(path, number=None, share=False, filter_irrelevant=False): content += read_file(md_path) content += "\n============\n\n" if os.path.exists(log_path): - content += read_file(log_path) + actual_log = read_file(log_path) + content += "\n".join(_filter_lines(actual_log.split("\n"), filters)) url = yunopaste(content) @@ -203,27 +224,12 @@ def log_display(path, number=None, share=False, filter_irrelevant=False): # Display logs if exist if os.path.exists(log_path): - - if filter_irrelevant: - filters = [ - r"set [+-]x$", - r"set [+-]o xtrace$", - r"local \w+$", - r"local legacy_args=.*$", - r".*Helper used in legacy mode.*", - r"args_array=.*$", - r"local -A args_array$", - r"ynh_handle_getopts_args", - r"ynh_script_progression" - ] - else: - filters = [] - from yunohost.service import _tail if number: - logs = _tail(log_path, int(number), filters=filters) + logs = _tail(log_path, int(number)) else: logs = read_file(log_path) + logs = _filter_lines(logs, filters) infos['log_path'] = log_path infos['logs'] = logs diff --git a/src/yunohost/service.py b/src/yunohost/service.py index faaf70cdc..278cea0a4 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -639,7 +639,7 @@ def _save_services(services): raise -def _tail(file, n, filters=[]): +def _tail(file, n): """ Reads a n lines from f with an offset of offset lines. The return value is a tuple in the form ``(lines, has_more)`` where `has_more` is @@ -650,8 +650,6 @@ def _tail(file, n, filters=[]): avg_line_length = 74 to_read = n - if filters: - filters = [re.compile(f) for f in filters] try: if file.endswith(".gz"): @@ -673,9 +671,6 @@ def _tail(file, n, filters=[]): pos = f.tell() lines = f.read().splitlines() - for filter_ in filters: - lines = [l for l in lines if not filter_.search(l)] - if len(lines) >= to_read: return lines[-to_read:] From 51d53be54cd30ead5e0c28e772dda8b27269a965 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 Jun 2020 19:22:39 +0200 Subject: [PATCH 027/138] Alway filter irrelevant log lines when sharing it --- src/yunohost/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 875d4e49d..c71de2ab3 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -136,6 +136,9 @@ def log_display(path, number=None, share=False, filter_irrelevant=False): share """ + if share: + filter_irrelevant = True + if filter_irrelevant: filters = [ r"set [+-]x$", From f45254884b28943fa8b415c84e4361578b0f8280 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jun 2020 17:37:54 +0200 Subject: [PATCH 028/138] Following previous regenconf fixes, the regen-conf was now outputing a whole lot of 'forget-about-it' when e.g. running regen-conf for nginx because of files flagged as 'should-not-exists'... so this is a dirty workaround --- src/yunohost/regenconf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 4aad5f921..6445248bd 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -236,6 +236,8 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run conf_files[f] = None # End discussion about stale file hashes + force_update_hashes_for_this_category = False + for system_path, pending_path in conf_files.items(): logger.debug("processing pending conf '%s' to system conf '%s'", pending_path, system_path) @@ -284,7 +286,8 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run os.remove(pending_path) conf_hashes[system_path] = None conf_status = 'forget-about-it' - regenerated = True + force_update_hashes_for_this_category = True + continue elif not saved_hash or force: if force: logger.debug("> system conf has been manually removed") @@ -377,7 +380,7 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run else: logger.success(m18n.n('regenconf_would_be_updated', category=category)) - if succeed_regen and not dry_run: + if (succeed_regen or force_update_hashes_for_this_category) and not dry_run: _update_conf_hashes(category, conf_hashes) # Append the category results From 3b6083df4abd12fa20cf5e9dc5feed782dd220e7 Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Fri, 5 Jun 2020 20:41:49 +0000 Subject: [PATCH 029/138] Translated using Weblate (Catalan) Currently translated at 100.0% (655 of 655 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index f6802f216..ea801fe53 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -651,5 +651,19 @@ "diagnosis_swap_tip": "Vigileu i tingueu en compte que els servidor està allotjant memòria d'intercanvi en una targeta SD o en l'emmagatzematge SSD, això pot reduir dràsticament l'esperança de vida del dispositiu.", "restore_already_installed_apps": "No s'han pogut restaurar les següents aplicacions perquè ja estan instal·lades: {apps}", "app_packaging_format_not_supported": "No es pot instal·lar aquesta aplicació ja que el format del paquet no és compatible amb la versió de YunoHost del sistema. Hauríeu de considerar actualitzar el sistema.", - "diagnosis_dns_try_dyndns_update_force": "La configuració DNS d'aquest domini hauria de ser gestionada automàticament per YunoHost. Si aquest no és el cas, podeu intentar forçar-ne l'actualització utilitzant yunohost dyndns update --force." + "diagnosis_dns_try_dyndns_update_force": "La configuració DNS d'aquest domini hauria de ser gestionada automàticament per YunoHost. Si aquest no és el cas, podeu intentar forçar-ne l'actualització utilitzant yunohost dyndns update --force.", + "migration_0015_cleaning_up": "Netejant la memòria cau i els paquets que ja no són necessaris…", + "migration_0015_specific_upgrade": "Començant l'actualització dels paquets del sistema que s'han d'actualitzar de forma independent…", + "migration_0015_modified_files": "Tingueu en compte que s'han trobat els següents fitxers que es van modificar manualment i podria ser que es sobreescriguin durant l'actualització: {manually_modified_files}", + "migration_0015_problematic_apps_warning": "Tingueu en compte que s'han trobat les següents aplicacions que podrien ser problemàtiques. Sembla que aquestes aplicacions no s'han instal·lat des del catàleg d'aplicacions de YunoHost, o no estan marcades com «funcionant». En conseqüència, no es pot garantir que segueixin funcionant després de l'actualització: {problematic_apps}", + "migration_0015_general_warning": "Tingueu en compte que aquesta migració és una operació delicada. L'equip de YunoHost ha fet tots els possibles per revisar i testejar, però tot i això podria ser que la migració trenqui alguna part del sistema o algunes aplicacions.\n\nPer tant, està recomana:\n - Fer una còpia de seguretat de totes les dades o aplicacions crítiques. Més informació a https://yunohost.org/backup;\n - Ser pacient un cop comenci la migració: en funció de la connexió Internet i del maquinari, podria estar unes hores per actualitzar-ho tot.", + "migration_0015_system_not_fully_up_to_date": "El sistema no està completament al dia. Heu de fer una actualització normal abans de fer la migració a Buster.", + "migration_0015_not_enough_free_space": "Hi ha poc espai lliure a /var/! HI hauria d'haver un mínim de 1GB lliure per poder fer aquesta migració.", + "migration_0015_not_stretch": "La distribució actual de Debian no és Stretch!", + "migration_0015_yunohost_upgrade": "Començant l'actualització del nucli de YunoHost…", + "migration_0015_still_on_stretch_after_main_upgrade": "Alguna cosa ha anat malament durant la actualització principal, sembla que el sistema encara està en Debian Stretch", + "migration_0015_main_upgrade": "Començant l'actualització principal…", + "migration_0015_patching_sources_list": "Apedaçament de source.lists…", + "migration_0015_start": "Començant la migració a Buster", + "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x" } From 47c2bce67a3714474af17867478baab28742a3c7 Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Sat, 6 Jun 2020 21:31:55 +0000 Subject: [PATCH 030/138] Translated using Weblate (Catalan) Currently translated at 100.0% (656 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index ea801fe53..f29a65c43 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -665,5 +665,6 @@ "migration_0015_main_upgrade": "Començant l'actualització principal…", "migration_0015_patching_sources_list": "Apedaçament de source.lists…", "migration_0015_start": "Començant la migració a Buster", - "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x" + "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x", + "regenconf_need_to_explicitly_specify_ssh": "La configuració ssh ha estat modificada manualment, però heu d'especificar explícitament la categoria «ssh» amb --force per fer realment els canvis." } From e93cbb1151d29e78a4aaaa42a7ed0b0564dd11ab Mon Sep 17 00:00:00 2001 From: ppr Date: Wed, 10 Jun 2020 16:51:15 +0000 Subject: [PATCH 031/138] Translated using Weblate (French) Currently translated at 99.4% (652 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index bd32d7f08..b24f13544 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -168,7 +168,7 @@ "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} n’est pas sur le point d’expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)", "certmanager_domain_http_not_working": "Le domaine {domain:s} ne semble pas être accessible via HTTP. Merci de vérifier la catégorie 'Web' dans le diagnostic pour plus d'informations. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)", "certmanager_error_no_A_record": "Aucun enregistrement DNS 'A' n’a été trouvé pour {domain:s}. Vous devez faire pointer votre nom de domaine vers votre machine pour être en mesure d’installer un certificat Let’s Encrypt ! (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", - "certmanager_domain_dns_ip_differs_from_public_ip": "L’enregistrement DNS 'A' du domaine {domain:s} est différent de l’adresse IP de ce serveur. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", + "certmanager_domain_dns_ip_differs_from_public_ip": "L'enregistrement DNS du domaine {domain:s} est différent de l’adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans le diagnostic. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", "certmanager_cannot_read_cert": "Quelque chose s’est mal passé lors de la tentative d’ouverture du certificat actuel pour le domaine {domain:s} (fichier : {file:s}), la cause est : {reason:s}", "certmanager_cert_install_success_selfsigned": "Le certificat auto-signé est maintenant installé pour le domaine « {domain:s} »", "certmanager_cert_install_success": "Le certificat Let’s Encrypt est maintenant installé pour le domaine « {domain:s} »", @@ -535,7 +535,7 @@ "diagnosis_ip_broken_resolvconf": "La résolution du nom de domaine semble être cassée sur votre serveur, ce qui semble lié au fait que /etc/resolv.conf ne pointe pas vers 127.0.0.1.", "diagnosis_dns_good_conf": "Les enregistrements DNS sont correctement configurés pour le domaine {domain} (catégorie {category})", "diagnosis_dns_bad_conf": "Certains enregistrements DNS sont manquants ou incorrects pour le domaine {domain} (catégorie {category})", - "diagnosis_dns_discrepancy": "Cet enregistrement DNS ne semble pas correspondre à la configuration recommandée :
Type : {type}
Nom : {name}
Valeur actuelle : {current}
Valeur attendue : {value}", + "diagnosis_dns_discrepancy": "Cet enregistrement DNS ne semble pas correspondre à la configuration recommandée :
Type : {type}
Nom : {name}
La valeur actuelle est : {current}
La valeur attendue est : {value}", "diagnosis_services_bad_status": "Le service {service} est {status} :-(", "diagnosis_diskusage_verylow": "L'espace de stockage {mountpoint} (sur l’appareil {device}) ne dispose que de {free} ({free_percent}%) espace restant (sur {total}). Vous devriez vraiment envisager de nettoyer de l’espace !", "diagnosis_diskusage_low": "L'espace de stockage {mountpoint} (sur l'appareil {device}) ne dispose que de {free} ({free_percent}%) espace restant (sur {total}). Faites attention.", @@ -649,5 +649,23 @@ "diagnosis_domain_expiration_error": "Certains domaines vont expirer TRÈS PROCHAINEMENT !", "diagnosis_domain_expires_in": "{domain} expire dans {days} jours.", "certmanager_domain_not_diagnosed_yet": "Il n'y a pas encore de résultat de diagnostic pour le domaine %s. Merci de relancer un diagnostic pour les catégories 'Enregistrements DNS' et 'Web' dans la section Diagnostique pour vérifier si le domaine est prêt pour Let's Encrypt. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)", - "diagnosis_swap_tip": "Merci d'être prudent et conscient que si vous hébergez une partition SWAP sur une carte SD ou un disque SSD, cela risque de réduire drastiquement l’espérance de vie du périphérique." + "diagnosis_swap_tip": "Merci d'être prudent et conscient que si vous hébergez une partition SWAP sur une carte SD ou un disque SSD, cela risque de réduire drastiquement l’espérance de vie du périphérique.", + "restore_already_installed_apps": "Les applications suivantes ne peuvent pas être restaurées car elles sont déjà installées : {apps}", + "regenconf_need_to_explicitly_specify_ssh": "La configuration de ssh a été modifiée manuellement. Vous devez explicitement indiquer la mention --force à \"ssh\" pour appliquer les changements.", + "migration_0015_cleaning_up": "Nettoyage du cache et des paquets qui ne sont plus utiles …", + "migration_0015_specific_upgrade": "Commencement de la mise à jour des paquets du système qui doivent être mis à jour séparément …", + "migration_0015_modified_files": "Veuillez noter que les fichiers suivants ont été modifiés manuellement et pourraient être écrasés à la suite de la mise à niveau : {manually_modified_files}", + "migration_0015_problematic_apps_warning": "Veuillez noter que des applications qui peuvent poser problèmes ont été détectées. Il semble qu'elles n'aient pas été installées à partir du catalogue d'applications YunoHost, ou bien qu'elles ne soient pas signalées comme \"fonctionnelles\". Par conséquent, il n'est pas possible de garantir que les applications suivantes fonctionneront encore après la mise à niveau : {problematic_apps}", + "migration_0015_general_warning": "Veuillez noter que cette migration est une opération délicate. L'équipe YunoHost a fait de son mieux pour la revérifier et la tester, mais la migration pourrait quand même casser des éléments du système ou de ses applications.\n\nIl est donc recommandé :\n…- de faire une sauvegarde de toute donnée ou application critique. Plus d'informations ici https://yunohost.org/backup ;\n…- d'être patient après le lancement de la migration. Selon votre connexion internet et votre matériel, la mise à niveau peut prendre jusqu'à quelques heures.", + "migration_0015_system_not_fully_up_to_date": "Votre système n'est pas entièrement à jour. Veuillez effectuer une mise à jour normale avant de lancer la migration vers Buster.", + "migration_0015_not_enough_free_space": "L'espace libre est très faible dans /var/ ! Vous devriez avoir au moins 1 Go de libre pour effectuer cette migration.", + "migration_0015_not_stretch": "La distribution Debian actuelle n'est pas Stretch !", + "migration_0015_yunohost_upgrade": "Lancement de la mise à jour de YunoHost …", + "migration_0015_still_on_stretch_after_main_upgrade": "Quelque chose s'est mal passé lors de la mise à niveau, le système semble toujours être sous Debian Stretch", + "migration_0015_main_upgrade": "Début de la mise à niveau générale …", + "migration_0015_patching_sources_list": "Mise à jour du fichier sources.lists …", + "migration_0015_start": "Démarrage de la migration vers Buster", + "migration_description_0015_migrate_to_buster": "Mise à niveau du système vers Debian Buster et YunoHost 4.x", + "diagnosis_dns_try_dyndns_update_force": "La configuration DNS de ce domaine devrait être automatiquement gérée par Yunohost. Si ce n'est pas le cas, vous pouvez essayer de forcer une mise à jour en utilisant yunohost dyndns update --force.", + "app_packaging_format_not_supported": "Cette application ne peut pas être installée car son format n'est pas pris en charge par votre version de Yunohost. Vous devriez probablement envisager de mettre à jour votre système." } From bf88d3e41e4a850bcbc7a7fdc72a5c65d3210c4e Mon Sep 17 00:00:00 2001 From: Yasss Gurl Date: Tue, 16 Jun 2020 12:33:20 +0000 Subject: [PATCH 032/138] Translated using Weblate (German) Currently translated at 33.8% (222 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/de/ --- locales/de.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/de.json b/locales/de.json index 0c5b031ce..fab9eaf2c 100644 --- a/locales/de.json +++ b/locales/de.json @@ -340,5 +340,6 @@ "diagnosis_ip_weird_resolvconf": "DNS Auflösung scheint zu funktionieren, aber sei vorsichtig wenn du eine eigene /etc/resolv.conf verwendest.", "diagnosis_display_tip": "Um die gefundenen Probleme zu sehen, kannst Du zum Diagnose-Bereich des webadmin gehen, oder 'yunohost diagnosis show --issues' in der Kommandozeile ausführen.", "backup_archive_corrupted": "Das Backup-Archiv '{archive}' scheint beschädigt: {error}", - "backup_archive_cant_retrieve_info_json": "Die Informationen für das Archiv '{archive}' konnten nicht geladen werden... Die Datei info.json wurde nicht gefunden (oder ist kein gültiges json)." + "backup_archive_cant_retrieve_info_json": "Die Informationen für das Archiv '{archive}' konnten nicht geladen werden... Die Datei info.json wurde nicht gefunden (oder ist kein gültiges json).", + "app_packaging_format_not_supported": "Diese App kann nicht installiert werden da das Paketformat nicht von der Yunohost-Version unterstützt wird. Denken Sie darüber nach das System zu aktualisieren." } From 9c1eca8d4ae714679ec8478f1b8ff1d1d467dd0d Mon Sep 17 00:00:00 2001 From: Julien Rabier Date: Mon, 11 May 2020 21:37:17 +0000 Subject: [PATCH 033/138] fix destination concurrency Hi, Postfix has this very peculiar behavior where the target of some config keys changes depending on the value. Here, if `smtp_destination_concurrency_limit` is set to 1, then according to http://www.postfix.org/postconf.5.html#default_destination_concurrency_limit it doesn't mean "1 concurrent mail per domain, but per recipiend address". So, if set to 1, it means we can send any volume of e-mails concurrently (with a 5s delay) if all recipient addresses are different. In order to avoid this, we should increase the value to restore the expected behavior (concurrency per domain, not per recipient). --- data/templates/postfix/main.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index 61cbfa2e6..18e457a76 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -170,7 +170,7 @@ smtpd_milters = inet:localhost:11332 milter_default_action = accept # Avoid to send simultaneously too many emails -smtp_destination_concurrency_limit = 1 +smtp_destination_concurrency_limit = 2 default_destination_rate_delay = 5s # Avoid email adress scanning From 7805837b1dcfec03c7309f8f44ff0589e503acd0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 18 Jun 2020 15:08:51 +0200 Subject: [PATCH 034/138] Move the general regen_conf after we regen ssh, otherwise there's a non-relevant warning displayed because the regen-conf tells to rerun ssh specifically which is already done... --- src/yunohost/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 9221e6c7d..40d66586c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -368,7 +368,6 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, service_enable("yunohost-firewall") service_start("yunohost-firewall") - regen_conf(force=True) regen_conf(names=["ssh"], force=True) # Restore original ssh conf, as chosen by the @@ -384,6 +383,8 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, if os.path.exists(original_sshd_conf): os.rename(original_sshd_conf, '/etc/ssh/sshd_config') + regen_conf(force=True) + logger.success(m18n.n('yunohost_configured')) logger.warning(m18n.n('yunohost_postinstall_end_tip')) From 9c33f6426922836b2949ca293b57858e0d817942 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 18 Jun 2020 15:19:44 +0200 Subject: [PATCH 035/138] Update changelog for 3.8.4.9 --- debian/changelog | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/debian/changelog b/debian/changelog index 9f07e62d1..52ceddcb6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +yunohost (3.8.4.9) stable; urgency=low + + - [fix] Force lowercase on domain names (804f4b3e) + - [fix] Add dirmngr to Depends:, needed for apt-key / gpg (cd115ed8) + - [fix] Improve debugging when diagnosis ain't happy when renewing certs (0f0194be) + - [enh] Add yunohost version to logs metadata (d615546b) + - [enh] Alway filter irrelevant log lines when sharing it (38704cba, 51d53be5) + - [fix] Regen-conf outputing many 'forget-about-it' because of files flagged as to be removed (f4525488) + - [fix] postfix per-domain destination concurrency (#988) + - [fix] Call regenconf for ssh before the general regenconf during the postinstall to avoid an irrelevant warning (7805837b) + - [i18n] Translations updated for Catalan, French, German + + Thanks to all contributors <3 ! (taziden, ljf, ppr, xaloc33, Yasss Gurl) + + -- Alexandre Aubin Thu, 18 Jun 2020 15:13:01 +0200 + yunohost (3.8.4.8) stable; urgency=low - [fix] Don't add unprotected_urls if it's already in skipped_urls (#1005) From 9624c4dc1cfcfa8fe4e69b2bc6dacbba0697cc41 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 10 Jun 2020 16:09:05 +0200 Subject: [PATCH 036/138] Fix small issue with unscd upgrade/downgrade ... new version ain't always 0.53.1, so find it using dirty scrapping --- src/yunohost/data_migrations/0015_migrate_to_buster.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 08e85e704..78f6c24c9 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -50,7 +50,11 @@ class MyMigration(Migration): # which for apt appears as a lower version (hence the --allow-downgrades and the hardcoded version number) unscd_version = check_output('dpkg -s unscd | grep "^Version: " | cut -d " " -f 2') if "yunohost" in unscd_version: - self.apt_install('unscd=0.53-1+b1 --allow-downgrades') + new_version = check_output("apt policy unscd 2>/dev/null | grep -v '\\*\\*\\*' | grep -A100 'Version table' | grep debian -B1 | head -n 1 | awk '{print $1}'") + if new_version: + self.apt_install('unscd=%s --allow-downgrades' % new_version) + else: + logger.warning("Could not identify which version of unscd to install") # Upgrade libpam-modules independently, small issue related to willing to overwrite a file previously provided by Yunohost libpammodules_version = check_output('dpkg -s libpam-modules | grep "^Version: " | cut -d " " -f 2') From e1d5450da97ca5b93875a981b6d110f519492fbe Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 18 Jun 2020 16:20:10 +0200 Subject: [PATCH 037/138] Update changelog for 3.8.5.2 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 1fe498b31..faae23924 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.8.5.2) testing; urgency=low + + - [fix] Small issue with unscd upgrade/downgrade ... new version ain't always 0.53.1, so find it using dirty scrapping + + -- Alexandre Aubin Thu, 18 Jun 2020 16:19:35 +0200 + yunohost (3.8.5.1) testing; urgency=low - [fix] Update Stretch->Buster migration disclaimer to make it clear that this is alpha-stage From 993329083641b79d25a09793d5376b001956410d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Jun 2020 15:08:22 +0200 Subject: [PATCH 038/138] Create backup archive dir during postinstall --- src/yunohost/tools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index c6ea3221a..83b4eb629 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -359,6 +359,12 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, except Exception as e: logger.warning(str(e)) + # Create the archive directory (makes it easier for people to upload backup + # archives, otherwise it's only created after running `yunohost backup + # create` once. + from yunohost.backup import _create_archive_dir + _create_archive_dir() + # Init migrations (skip them, no need to run them on a fresh system) _skip_all_migrations() From 7a1e779684073e781231f4223560de0f0620a8ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Jun 2020 18:46:53 +0200 Subject: [PATCH 039/138] Fix unscd downgrade + add debug infos to apt_install() --- src/yunohost/data_migrations/0015_migrate_to_buster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 78f6c24c9..836660701 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -50,7 +50,7 @@ class MyMigration(Migration): # which for apt appears as a lower version (hence the --allow-downgrades and the hardcoded version number) unscd_version = check_output('dpkg -s unscd | grep "^Version: " | cut -d " " -f 2') if "yunohost" in unscd_version: - new_version = check_output("apt policy unscd 2>/dev/null | grep -v '\\*\\*\\*' | grep -A100 'Version table' | grep debian -B1 | head -n 1 | awk '{print $1}'") + new_version = check_output("apt policy unscd 2>/dev/null | grep -v '\\*\\*\\*' | grep -A100 'Version table' | grep http -B1 | head -n 1 | awk '{print $1}'").strip() if new_version: self.apt_install('unscd=%s --allow-downgrades' % new_version) else: @@ -200,4 +200,6 @@ class MyMigration(Migration): cmd = "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + cmd + logger.debug("Running: %s" % cmd) + call_async_output(cmd, callbacks, shell=True) From 5a87fea5963b5eaf7a617d1961bc10c4affbab1b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Jun 2020 18:51:19 +0200 Subject: [PATCH 040/138] Update changelog for 3.8.5.3 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index faae23924..89f78b61a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.8.5.3) testing; urgency=low + + - [fix] Fix the fix about unscd downgrade :/ + + -- Alexandre Aubin Fri, 19 Jun 2020 18:50:58 +0200 + yunohost (3.8.5.2) testing; urgency=low - [fix] Small issue with unscd upgrade/downgrade ... new version ain't always 0.53.1, so find it using dirty scrapping From c47390277642b8a7040e1237f67f5fdabf9cdea6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Jun 2020 19:02:02 +0200 Subject: [PATCH 041/138] Yes, we don't give a flying fuck that sudo-ldap is already provided --- src/yunohost/tools.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 83b4eb629..d5fab2fb1 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -593,12 +593,17 @@ def tools_upgrade(operation_logger, apps=None, system=False, allow_yunohost_upgr logger.debug("Running apt command :\n{}".format(dist_upgrade)) + def is_relevant(l): - return "Reading database ..." not in l.rstrip() + irrelevants = [ + "service sudo-ldap already provided", + "Reading database ..." + ] + return all(i not in l.rstrip() for i in irrelevants) callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") if is_relevant(l) else logger.debug(l.rstrip() + "\r"), - lambda l: logger.warning(l.rstrip()), + lambda l: logger.warning(l.rstrip()) if is_relevant(l) else logger.debug(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) if returncode != 0: From 8f8be838b6ea0cd596d6e619ffc2d7b9f00f9811 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Jun 2020 19:57:51 +0200 Subject: [PATCH 042/138] Enforce permissions on rspamd log directory --- data/hooks/conf_regen/31-rspamd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/conf_regen/31-rspamd b/data/hooks/conf_regen/31-rspamd index 861549e27..6da4af643 100755 --- a/data/hooks/conf_regen/31-rspamd +++ b/data/hooks/conf_regen/31-rspamd @@ -42,6 +42,8 @@ do_post_regen() { chown _rspamd /etc/dkim/*.mail.key chmod 400 /etc/dkim/*.mail.key + [ ! -e /var/log/rspamd ] || chown _rspamd:_rspamd /var/log/rspamd + regen_conf_files=$1 [ -z "$regen_conf_files" ] && exit 0 From b373eb6076e25d995b73094a4e946f6562a0a317 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Jun 2020 23:35:50 +0200 Subject: [PATCH 043/138] Fix parsing of unscd version... --- src/yunohost/data_migrations/0015_migrate_to_buster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 836660701..bbbf8bf16 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -50,7 +50,7 @@ class MyMigration(Migration): # which for apt appears as a lower version (hence the --allow-downgrades and the hardcoded version number) unscd_version = check_output('dpkg -s unscd | grep "^Version: " | cut -d " " -f 2') if "yunohost" in unscd_version: - new_version = check_output("apt policy unscd 2>/dev/null | grep -v '\\*\\*\\*' | grep -A100 'Version table' | grep http -B1 | head -n 1 | awk '{print $1}'").strip() + new_version = check_output("LC_ALL=C apt policy unscd 2>/dev/null | grep -v '\\*\\*\\*' | grep http -B1 | head -n 1 | awk '{print $1}'").strip() if new_version: self.apt_install('unscd=%s --allow-downgrades' % new_version) else: From 649179338a4bc3ada9c8cf300ed6f128e8b4fbdb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Jun 2020 23:38:42 +0200 Subject: [PATCH 044/138] Update changelog for 3.8.5.4 --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 89f78b61a..8e3eea7d8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +yunohost (3.8.5.4) testing; urgency=low + + - [fix] Fix unscd version parsing *again* + - [fix] Enforce permissions on rspamd log directory + - [enh] Ignore stupid warnings about sudo-ldap that is already provided + + -- Alexandre Aubin Sun, 21 Jun 2020 23:37:09 +0200 + yunohost (3.8.5.3) testing; urgency=low - [fix] Fix the fix about unscd downgrade :/ From 82d59d59be7eaf3b87660e129aa0424aeaafccc7 Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Tue, 23 Jun 2020 10:26:09 +0200 Subject: [PATCH 045/138] silence info: wasn't deleted because it doesn't exist. --- data/helpers.d/apt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index 5589830c3..377464786 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -467,8 +467,8 @@ ynh_remove_extra_repo () { ynh_secure_remove "/etc/apt/sources.list.d/$name.list" ynh_secure_remove "/etc/apt/preferences.d/$name" - ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.gpg" - ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.asc" + ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.gpg" > /dev/null + ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.asc" > /dev/null # Update the list of package to exclude the old repo ynh_package_update From 74ef4d1cf783f7b8cf1d73807495d2837c119e8a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Jun 2020 19:46:22 +0200 Subject: [PATCH 046/138] Remove useless robot protection code --- data/templates/nginx/yunohost_admin.conf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/data/templates/nginx/yunohost_admin.conf b/data/templates/nginx/yunohost_admin.conf index d13dbfe90..ff823ba30 100644 --- a/data/templates/nginx/yunohost_admin.conf +++ b/data/templates/nginx/yunohost_admin.conf @@ -28,12 +28,6 @@ server { } location /yunohost { - # Block crawlers bot - if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { - return 403; - } - # X-Robots-Tag to precise the rules applied. - add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; } From f1bfc521aec0b6687ec3af7e08652be3e4982734 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Jun 2020 20:04:11 +0200 Subject: [PATCH 047/138] Allow to extend the nginx default_server configuration --- data/templates/nginx/yunohost_admin.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/templates/nginx/yunohost_admin.conf b/data/templates/nginx/yunohost_admin.conf index d13dbfe90..d405192fc 100644 --- a/data/templates/nginx/yunohost_admin.conf +++ b/data/templates/nginx/yunohost_admin.conf @@ -9,6 +9,8 @@ server { location /yunohost/admin { return 301 https://$http_host$request_uri; } + + include /etc/nginx/conf.d/default.d/*.conf; } server { @@ -40,4 +42,5 @@ server { include /etc/nginx/conf.d/yunohost_admin.conf.inc; include /etc/nginx/conf.d/yunohost_api.conf.inc; + include /etc/nginx/conf.d/default.d/*.conf; } From ac9182d69fdabbdaffd7cde84b7c0485d0482e89 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Jun 2020 21:42:35 +0200 Subject: [PATCH 048/138] Move redirect to /yunohost/admin to a separate nginx conf file to allow customizing it more easily --- data/hooks/conf_regen/15-nginx | 5 +++++ data/templates/nginx/redirect_to_admin.conf | 3 +++ data/templates/nginx/yunohost_admin.conf | 12 ------------ 3 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 data/templates/nginx/redirect_to_admin.conf diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 4924b6f91..a7574b070 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -26,6 +26,9 @@ do_init_regen() { ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf" + mkdir -p $nginx_conf_dir/default.d/ + cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/ + # Restart nginx if conf looks good, otherwise display error and exit unhappy nginx -t 2>/dev/null || { nginx -t; exit 1; } systemctl restart nginx || { journalctl --no-pager --lines=10 -u nginx >&2; exit 1; } @@ -77,6 +80,8 @@ do_pre_regen() { done ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf" + mkdir -p $nginx_conf_dir/default.d/ + cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/ # remove old domain conf files conf_files=$(ls -1 /etc/nginx/conf.d \ diff --git a/data/templates/nginx/redirect_to_admin.conf b/data/templates/nginx/redirect_to_admin.conf new file mode 100644 index 000000000..22748daa3 --- /dev/null +++ b/data/templates/nginx/redirect_to_admin.conf @@ -0,0 +1,3 @@ +location / { + return 302 https://$http_host/yunohost/admin; +} diff --git a/data/templates/nginx/yunohost_admin.conf b/data/templates/nginx/yunohost_admin.conf index d405192fc..b6d0b6d97 100644 --- a/data/templates/nginx/yunohost_admin.conf +++ b/data/templates/nginx/yunohost_admin.conf @@ -2,14 +2,6 @@ server { listen 80 default_server; listen [::]:80 default_server; - location / { - return 302 https://$http_host/yunohost/admin; - } - - location /yunohost/admin { - return 301 https://$http_host$request_uri; - } - include /etc/nginx/conf.d/default.d/*.conf; } @@ -25,10 +17,6 @@ server { more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; more_set_headers "Referrer-Policy : 'same-origin'"; - location / { - return 302 https://$http_host/yunohost/admin; - } - location /yunohost { # Block crawlers bot if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { From d435889776680b64261eebdac20a097459db9d4e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 29 Jun 2020 22:04:04 +0200 Subject: [PATCH 049/138] Make sure to validate/upgrade that we don't have any active weak certificate used by nginx at the beginning of the buster migration, otherwise nginx will later miserably fail to start --- data/hooks/conf_regen/02-ssl | 5 +-- locales/en.json | 1 + .../data_migrations/0015_migrate_to_buster.py | 39 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/data/hooks/conf_regen/02-ssl b/data/hooks/conf_regen/02-ssl index a893b21e1..310a5d526 100755 --- a/data/hooks/conf_regen/02-ssl +++ b/data/hooks/conf_regen/02-ssl @@ -69,12 +69,11 @@ do_init_regen() { -out "${ssl_dir}/certs/yunohost_crt.pem" \ -batch >>$LOGFILE 2>&1 - last_cert=$(ls $ssl_dir/newcerts/*.pem | sort -V | tail -n 1) chmod 640 "${ssl_dir}/certs/yunohost_key.pem" - chmod 640 "$last_cert" + chmod 640 "${ssl_dir}/certs/yunohost_crt.pem" cp "${ssl_dir}/certs/yunohost_key.pem" "$ynh_key" - cp "$last_cert" "$ynh_crt" + cp "${ssl_dir}/certs/yunohost_crt.pem" "$ynh_crt" ln -sf "$ynh_crt" /etc/ssl/certs/yunohost_crt.pem ln -sf "$ynh_key" /etc/ssl/private/yunohost_key.pem fi diff --git a/locales/en.json b/locales/en.json index abc1c1092..7e8e51260 100644 --- a/locales/en.json +++ b/locales/en.json @@ -479,6 +479,7 @@ "migration_0015_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently…", "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore…", + "migration_0015_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.", diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index bbbf8bf16..4f7f8afeb 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -16,6 +16,7 @@ from yunohost.utils.packages import get_ynh_package_version, _list_upgradable_ap logger = getActionLogger('yunohost.migration') + class MyMigration(Migration): "Upgrade the system to Debian Buster and Yunohost 4.x" @@ -28,6 +29,13 @@ class MyMigration(Migration): logger.info(m18n.n("migration_0015_start")) + # + # Make sure certificates do not use weak signature hash algorithms (md5, sha1) + # otherwise nginx will later refuse to start which result in + # catastrophic situation + # + self.validate_and_upgrade_cert_if_necessary() + # # Patch sources.list # @@ -203,3 +211,34 @@ class MyMigration(Migration): logger.debug("Running: %s" % cmd) call_async_output(cmd, callbacks, shell=True) + + def validate_and_upgrade_cert_if_necessary(self): + + active_certs = set(check_output("grep -roh '/.*crt.pem' /etc/nginx/").strip().split("\n")) + + cmd = "LC_ALL=C openssl x509 -in %s -text -noout | grep -i 'Signature Algorithm:' | awk '{print $3}' | uniq" + + default_crt = '/etc/yunohost/certs/yunohost.org/crt.pem' + default_key = '/etc/yunohost/certs/yunohost.org/key.pem' + default_signature = check_output(cmd % default_crt).strip() if default_crt in active_certs else None + if default_signature is not None and (default_signature.startswith("md5") or default_signature.startswith("sha1")): + logger.warning("%s is using a pretty old certificate incompatible with newer versions of nginx ... attempting to regenerate a fresh one" % default_crt) + + os.system("mv %s %s.old" % (default_crt, default_crt)) + os.system("mv %s %s.old" % (default_key, default_key)) + ret = os.system("/usr/share/yunohost/hooks/conf_regen/02-ssl init") + + if ret != 0 or not os.path.exists(default_crt): + logger.error("Upgrading the certificate failed ... reverting") + os.system("mv %s.old %s" % (default_crt, default_crt)) + os.system("mv %s.old %s" % (default_key, default_key)) + + signatures = {cert: check_output(cmd % cert).strip() for cert in active_certs} + + def cert_is_weak(cert): + sig = signatures[cert] + return sig.startswith("md5") or sig.startswith("sha1") + + weak_certs = [cert for cert in signatures.keys() if cert_is_weak(cert)] + if weak_certs: + raise YunohostError("migration_0015_weak_certs", certs=", ".join(weak_certs)) From 7bfe564aab2300172b6aad68656dddbdd8933a27 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 25 Jun 2020 10:53:08 +0200 Subject: [PATCH 050/138] fix get_files_diff if {orig,new}_file is None --- src/yunohost/regenconf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 6445248bd..a94f5023d 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -450,13 +450,13 @@ def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True): """ - if os.path.exists(orig_file): + if orig_file and os.path.exists(orig_file): with open(orig_file, 'r') as orig_file: orig_file = orig_file.readlines() else: orig_file = [] - if os.path.exists(new_file): + if new_file and os.path.exists(new_file): with open(new_file, 'r') as new_file: new_file = new_file.readlines() else: From 95cbc23733c34aacd1ba3cfb32469ed7ae73547b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Sun, 19 Jul 2020 19:21:53 +0200 Subject: [PATCH 051/138] Update en.json --- locales/en.json | 114 ++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/locales/en.json b/locales/en.json index 7e8e51260..a76176dc1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -29,26 +29,26 @@ "app_manifest_invalid": "Something is wrong with the app manifest: {error}", "app_not_upgraded": "The app '{failed_app}' failed to upgrade, and as a consequence the following apps' upgrades have been cancelled: {apps}", "app_not_correctly_installed": "{app:s} seems to be incorrectly installed", - "app_not_installed": "Could not find the app '{app:s}' in the list of installed apps: {all_apps}", + "app_not_installed": "Could not find {app:s} in the list of installed apps: {all_apps}", "app_not_properly_removed": "{app:s} has not been properly removed", "app_removed": "{app:s} removed", - "app_requirements_checking": "Checking required packages for {app}…", + "app_requirements_checking": "Checking required packages for {app}...", "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}", - "app_remove_after_failed_install": "Removing the app following the installation failure…", + "app_remove_after_failed_install": "Removing the app following the installation failure...", "app_sources_fetch_failed": "Could not fetch sources files, is the URL correct?", - "app_start_install": "Installing the app '{app}'…", - "app_start_remove": "Removing the app '{app}'…", - "app_start_backup": "Collecting files to be backed up for the app '{app}'…", - "app_start_restore": "Restoring the app '{app}'…", + "app_start_install": "Installing {app}...", + "app_start_remove": "Removing {app}...", + "app_start_backup": "Collecting files to be backed up for {app}...", + "app_start_restore": "Restoring {app}...", "app_unknown": "Unknown app", "app_unsupported_remote_type": "Unsupported remote type used for the app", "app_upgrade_several_apps": "The following apps will be upgraded: {apps}", - "app_upgrade_app_name": "Now upgrading {app}…", + "app_upgrade_app_name": "Now upgrading {app}...", "app_upgrade_failed": "Could not upgrade {app:s}: {error}", "app_upgrade_script_failed": "An error occurred inside the app upgrade script", "app_upgrade_some_app_failed": "Some apps could not be upgraded", "app_upgraded": "{app:s} upgraded", - "app_packaging_format_not_supported": "This app cannot be installed because its packaging format is not supported by your Yunohost version. You should probably consider upgrading your system.", + "app_packaging_format_not_supported": "This app cannot be installed because its packaging format is not supported by your YunoHost version. You should probably consider upgrading your system.", "apps_already_up_to_date": "All apps are already up-to-date", "apps_catalog_init_success": "App catalog system initialized!", "apps_catalog_updating": "Updating application catalog…", @@ -64,22 +64,22 @@ "ask_new_path": "New path", "ask_password": "Password", "backup_abstract_method": "This backup method has yet to be implemented", - "backup_actually_backuping": "Creating a backup archive from the collected files…", - "backup_app_failed": "Could not back up the app '{app:s}'", - "backup_applying_method_borg": "Sending all files to backup into borg-backup repository…", - "backup_applying_method_copy": "Copying all files to backup…", - "backup_applying_method_custom": "Calling the custom backup method '{method:s}'…", - "backup_applying_method_tar": "Creating the backup TAR archive…", - "backup_archive_app_not_found": "Could not find the app '{app:s}' in the backup archive", + "backup_actually_backuping": "Creating a backup archive from the collected files...", + "backup_app_failed": "Could not back up {app:s}", + "backup_applying_method_borg": "Sending all files to backup into borg-backup repository...", + "backup_applying_method_copy": "Copying all files to backup...", + "backup_applying_method_custom": "Calling the custom backup method '{method:s}'...", + "backup_applying_method_tar": "Creating the backup TAR archive...", + "backup_archive_app_not_found": "Could not find {app:s} in the backup archive", "backup_archive_broken_link": "Could not access the backup archive (broken link to {path:s})", "backup_archive_name_exists": "A backup archive with this name already exists.", "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Could not open the backup archive", - "backup_archive_cant_retrieve_info_json": "Could not load infos for archive '{archive}' ... The info.json cannot be retrieved (or is not a valid json).", + "backup_archive_cant_retrieve_info_json": "Could not load infos for archive '{archive}'... The info.json cannot be retrieved (or is not a valid json).", "backup_archive_corrupted": "It looks like the backup archive '{archive}' is corrupted : {error}", "backup_archive_system_part_not_available": "System part '{part:s}' unavailable in this backup", "backup_archive_writing_error": "Could not add the files '{source:s}' (named in the archive '{dest:s}') to be backed up into the compressed archive '{archive:s}'", - "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size:s} MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)", + "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size:s}MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)", "backup_borg_not_implemented": "The Borg backup method is not yet implemented", "backup_cant_mount_uncompress_archive": "Could not mount the uncompressed archive as write protected", "backup_cleaning_failed": "Could not clean up the temporary backup folder", @@ -98,20 +98,20 @@ "backup_method_copy_finished": "Backup copy finalized", "backup_method_custom_finished": "Custom backup method '{method:s}' finished", "backup_method_tar_finished": "TAR backup archive created", - "backup_mount_archive_for_restore": "Preparing archive for restoration…", + "backup_mount_archive_for_restore": "Preparing archive for restoration...", "backup_no_uncompress_archive_dir": "There is no such uncompressed archive directory", "backup_nothings_done": "Nothing to save", "backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", "backup_output_directory_not_empty": "You should pick an empty output directory", "backup_output_directory_required": "You must provide an output directory for the backup", "backup_output_symlink_dir_broken": "Your archive directory '{path:s}' is a broken symlink. Maybe you forgot to re/mount or plug in the storage medium it points to.", - "backup_permission": "Backup permission for app {app:s}", + "backup_permission": "Backup permission for {app:s}", "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support PHP 7, you may be unable to restore your PHP apps (reason: {error:s})", - "backup_running_hooks": "Running backup hooks…", + "backup_running_hooks": "Running backup hooks...", "backup_system_part_failed": "Could not backup the '{part:s}' system part", "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive", "backup_with_no_backup_script_for_app": "The app '{app:s}' has no backup script. Ignoring.", - "backup_with_no_restore_script_for_app": "The '{app:s}' has no restoration script, you will not be able to automatically restore the backup of this app.", + "backup_with_no_restore_script_for_app": "{app:s} has no restoration script, you will not be able to automatically restore the backup of this app.", "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for {domain} right now because its nginx conf lacks the corresponding code snippet... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for the domain '{domain:s}' is not issued by Let's Encrypt. Cannot renew it automatically!", "certmanager_attempt_to_renew_valid_cert": "The certificate for the domain '{domain:s}' is not about to expire! (You may use --force if you know what you're doing)", @@ -121,7 +121,7 @@ "certmanager_cert_install_success_selfsigned": "Self-signed certificate now installed for the domain '{domain:s}'", "certmanager_cert_renew_success": "Let's Encrypt certificate renewed for the domain '{domain:s}'", "certmanager_cert_signing_failed": "Could not sign the new certificate", - "certmanager_certificate_fetching_or_enabling_failed": "Trying to use the new certificate for {domain:s} did not work…", + "certmanager_certificate_fetching_or_enabling_failed": "Trying to use the new certificate for {domain:s} did not work...", "certmanager_couldnt_fetch_intermediate_cert": "Timed out when trying to fetch intermediate certificate from Let's Encrypt. Certificate installation/renewal aborted—please try again later.", "certmanager_domain_not_diagnosed_yet": "There is no diagnosis result for domain %s yet. Please re-run a diagnosis for categories 'DNS records' and 'Web' in the diagnosis section to check if the domain is ready for Let's Encrypt. (Or if you know what you are doing, use '--no-checks' to turn off those checks.)", "certmanager_domain_cert_not_selfsigned": "The certificate for domain {domain:s} is not self-signed. Are you sure you want to replace it? (Use '--force' to do so.)", @@ -155,9 +155,9 @@ "diagnosis_everything_ok": "Everything looks good for {category}!", "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}': {error}", "diagnosis_no_cache": "No diagnosis cache yet for category '{category}'", - "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !", + "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4!", "diagnosis_ip_no_ipv4": "The server does not have working IPv4.", - "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", + "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6!", "diagnosis_ip_no_ipv6": "The server does not have working IPv6.", "diagnosis_ip_no_ipv6_tip": "Having a working IPv6 is not mandatory for your server to work, but it is better for the health of the Internet as a whole. IPv6 should usually be automatically configured by the system or your provider if it's available. Otherwise, you might need to configure a few things manually as explained in the documentation here: https://yunohost.org/#/ipv6. If you cannot enable IPv6 or if it seems too technical for you, you can also safely ignore this warning.", "diagnosis_ip_global": "Global IP: {global}", @@ -285,7 +285,7 @@ "dyndns_cron_removed": "DynDNS cron job removed", "dyndns_ip_update_failed": "Could not update IP address to DynDNS", "dyndns_ip_updated": "Updated your IP on DynDNS", - "dyndns_key_generating": "Generating DNS key… It may take a while.", + "dyndns_key_generating": "Generating DNS key... It may take a while.", "dyndns_key_not_found": "DNS key not found for the domain", "dyndns_no_domain_registered": "No domain registered with DynDNS", "dyndns_provider_unreachable": "Unable to reach DynDNS provider {provider}: either your YunoHost is not correctly connected to the internet or the dynette server is down.", @@ -293,9 +293,9 @@ "dyndns_registration_failed": "Could not register DynDNS domain: {error:s}", "dyndns_domain_not_provided": "DynDNS provider {provider:s} cannot provide domain {domain:s}.", "dyndns_unavailable": "The domain '{domain:s}' is unavailable.", - "executing_command": "Executing command '{command:s}'…", - "executing_script": "Executing script '{script:s}'…", - "extracting": "Extracting…", + "executing_command": "Executing command '{command:s}'...", + "executing_script": "Executing script '{script:s}'...", + "extracting": "Extracting...", "experimental_feature": "Warning: This feature is experimental and not considered stable, you should not use it unless you know what you are doing.", "field_invalid": "Invalid field '{:s}'", "file_does_not_exist": "The file {path:s} does not exist.", @@ -327,7 +327,7 @@ "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters long—though it is good practice to use a longer password (i.e. a passphrase) and/or to a variation of characters (uppercase, lowercase, digits and special characters).", "group_already_exist": "Group {group} already exists", "group_already_exist_on_system": "Group {group} already exists in the system groups", - "group_already_exist_on_system_but_removing_it": "Group {group} already exists in the system groups, but YunoHost will remove it…", + "group_already_exist_on_system_but_removing_it": "Group {group} already exists in the system groups, but YunoHost will remove it...", "group_created": "Group '{group}' created", "group_creation_failed": "Could not create the group '{group}': {error}", "group_cannot_edit_all_users": "The group 'all_users' cannot be edited manually. It is a special group meant to contain all users registered in YunoHost", @@ -409,10 +409,10 @@ "migrate_tsig_end": "Migration to HMAC-SHA-512 finished", "migrate_tsig_failed": "Could not migrate the DynDNS domain '{domain}' to HMAC-SHA-512, rolling back. Error: {error_code}, {error}", "migrate_tsig_start": "Insufficiently secure key algorithm detected for TSIG signature of the domain '{domain}', initiating migration to the more secure HMAC-SHA-512", - "migrate_tsig_wait": "Waiting three minutes for the DynDNS server to take the new key into account…", - "migrate_tsig_wait_2": "2min…", - "migrate_tsig_wait_3": "1min…", - "migrate_tsig_wait_4": "30 seconds…", + "migrate_tsig_wait": "Waiting three minutes for the DynDNS server to take the new key into account...", + "migrate_tsig_wait_2": "2 min...", + "migrate_tsig_wait_3": "1 min...", + "migrate_tsig_wait_4": "30 seconds...", "migrate_tsig_not_needed": "You do not appear to use a DynDNS domain, so no migration is needed.", "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", "migration_description_0002_migrate_to_tsig_sha256": "Improve security of DynDNS TSIG updates by using SHA-512 instead of MD5", @@ -430,9 +430,9 @@ "migration_description_0014_remove_app_status_json": "Remove legacy status.json app files", "migration_description_0015_migrate_to_buster": "Upgrade the system to Debian Buster and YunoHost 4.x", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", - "migration_0003_patching_sources_list": "Patching the sources.lists…", - "migration_0003_main_upgrade": "Starting main upgrade…", - "migration_0003_fail2ban_upgrade": "Starting the Fail2Ban upgrade…", + "migration_0003_patching_sources_list": "Patching the sources.lists...", + "migration_0003_main_upgrade": "Starting main upgrade...", + "migration_0003_fail2ban_upgrade": "Starting the Fail2Ban upgrade...", "migration_0003_restoring_origin_nginx_conf": "Your file /etc/nginx/nginx.conf was edited somehow. The migration is going to reset it to its original state first… The previous file will be available as {backup_dest}.", "migration_0003_yunohost_upgrade": "Starting the YunoHost package upgrade… The migration will end, but the actual upgrade will happen immediately afterwards. After the operation is complete, you might have to log in to the webadmin page again.", "migration_0003_not_jessie": "The current Debian distribution is not Jessie!", @@ -453,32 +453,32 @@ "migration_0008_dsa": "• The DSA key will be turned off. Hence, you might need to invalidate a spooky warning from your SSH client, and recheck the fingerprint of your server;", "migration_0008_warning": "If you understand those warnings and want YunoHost to override your current configuration, run the migration. Otherwise, you can also skip the migration, though it is not recommended.", "migration_0008_no_warning": "Overriding your SSH configuration should be safe, though this cannot be promised! Run the migration to override it. Otherwise, you can also skip the migration, though it is not recommended.", - "migration_0009_not_needed": "This migration already happened somehow… (?) Skipping.", + "migration_0009_not_needed": "This migration already happened somehow... (?) Skipping.", "migration_0011_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_0011_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", "migration_0011_create_group": "Creating a group for each user…", "migration_0011_done": "Migration completed. You are now able to manage usergroups.", "migration_0011_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.", "migration_0011_LDAP_update_failed": "Could not update LDAP. Error: {error:s}", - "migration_0011_migrate_permission": "Migrating permissions from apps settings to LDAP…", + "migration_0011_migrate_permission": "Migrating permissions from apps settings to LDAP...", "migration_0011_migration_failed_trying_to_rollback": "Could not migrate… trying to roll back the system.", "migration_0011_rollback_success": "System rolled back.", - "migration_0011_update_LDAP_database": "Updating LDAP database…", - "migration_0011_update_LDAP_schema": "Updating LDAP schema…", + "migration_0011_update_LDAP_database": "Updating LDAP database...", + "migration_0011_update_LDAP_schema": "Updating LDAP schema...", "migration_0011_failed_to_remove_stale_object": "Could not remove stale object {dn}: {error}", "migration_0015_start" : "Starting migration to Buster", - "migration_0015_patching_sources_list": "Patching the sources.lists…", - "migration_0015_main_upgrade": "Starting main upgrade…", + "migration_0015_patching_sources_list": "Patching the sources.lists...", + "migration_0015_main_upgrade": "Starting main upgrade...", "migration_0015_still_on_stretch_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Stretch", - "migration_0015_yunohost_upgrade" : "Starting YunoHost core upgrade…", + "migration_0015_yunohost_upgrade" : "Starting YunoHost core upgrade...", "migration_0015_not_stretch" : "The current Debian distribution is not Stretch!", "migration_0015_not_enough_free_space" : "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.", "migration_0015_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Buster.", "migration_0015_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", "migration_0015_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", "migration_0015_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", - "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently…", - "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore…", + "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently...", + "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore...", "migration_0015_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", @@ -486,7 +486,7 @@ "migrations_failed_to_load_migration": "Could not load migration {id}: {error}", "migrations_exclusive_options": "'--auto', '--skip', and '--force-rerun' are mutually exclusive options.", "migrations_list_conflict_pending_done": "You cannot use both '--previous' and '--done' at the same time.", - "migrations_loading_migration": "Loading migration {id}…", + "migrations_loading_migration": "Loading migration {id}...", "migrations_migration_has_failed": "Migration {id} did not complete, aborting. Error: {exception}", "migrations_must_provide_explicit_targets": "You must provide explicit targets when using '--skip' or '--force-rerun'", "migrations_need_to_accept_disclaimer": "To run the migration {id}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option '--accept-disclaimer'.", @@ -494,8 +494,8 @@ "migrations_no_such_migration": "There is no migration called '{id}'", "migrations_not_pending_cant_skip": "Those migrations are not pending, so cannot be skipped: {ids}", "migrations_pending_cant_rerun": "Those migrations are still pending, so cannot be run again: {ids}", - "migrations_running_forward": "Running migration {id}…", - "migrations_skip_migration": "Skipping migration {id}…", + "migrations_running_forward": "Running migration {id}...", + "migrations_skip_migration": "Skipping migration {id}...", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", "no_internet_connection": "The server is not connected to the Internet", @@ -549,11 +549,11 @@ "regenconf_would_be_updated": "The configuration would have been updated for category '{category}'", "regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'…", "regenconf_failed": "Could not regenerate the configuration for category(s): {categories}", - "regenconf_pending_applying": "Applying pending configuration for category '{category}'…", + "regenconf_pending_applying": "Applying pending configuration for category '{category}'...", "regenconf_need_to_explicitly_specify_ssh": "The ssh configuration has been manually modified, but you need to explicitly specify category 'ssh' with --force to actually apply the changes.", "restore_already_installed_app": "An app with the ID '{app:s}' is already installed", "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}", - "restore_app_failed": "Could not restore the app '{app:s}'", + "restore_app_failed": "Could not restore {app:s}", "restore_cleaning_failed": "Could not clean up the temporary restoration directory", "restore_complete": "Restored", "restore_confirm_yunohost_installed": "Do you really want to restore an already installed system? [{answers:s}]", @@ -616,7 +616,7 @@ "ssowat_conf_updated": "SSOwat configuration updated", "system_upgraded": "System upgraded", "system_username_exists": "Username already exists in the list of system users", - "this_action_broke_dpkg": "This action broke dpkg/APT (the system package managers)… You can try to solve this issue by connecting through SSH and running `sudo apt install --fix-broken` and/or `sudo dpkg --configure -a`.", + "this_action_broke_dpkg": "This action broke dpkg/APT (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo apt install --fix-broken` and/or `sudo dpkg --configure -a`.", "tools_upgrade_at_least_one": "Please specify '--apps', or '--system'", "tools_upgrade_cant_both": "Cannot upgrade both system and apps at the same time", "tools_upgrade_cant_hold_critical_packages": "Could not hold critical packages…", @@ -626,15 +626,15 @@ "tools_upgrade_special_packages": "Now upgrading 'special' (yunohost-related) packages…", "tools_upgrade_special_packages_explanation": "The special upgrade will continue in the background. Please don't start any other actions on your server for the next ~10 minutes (depending on hardware speed). After this, you may have to re-log in to the webadmin. The upgrade log will be available in Tools → Log (in the webadmin) or using 'yunohost log list' (from the command-line).", "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed.\nPress [Enter] to get the command line back", - "unbackup_app": "App '{app:s}' will not be saved", + "unbackup_app": "{app:s} will not be saved", "unexpected_error": "Something unexpected went wrong: {error}", "unlimit": "No quota", - "unrestore_app": "App '{app:s}' will not be restored", + "unrestore_app": "{app:s} will not be restored", "update_apt_cache_failed": "Could not to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines, which might help identify problematic lines: \n{sourceslist}", "update_apt_cache_warning": "Something went wrong while updating the cache of APT (Debian's package manager). Here is a dump of the sources.list lines, which might help identify problematic lines: \n{sourceslist}", - "updating_apt_cache": "Fetching available upgrades for system packages…", + "updating_apt_cache": "Fetching available upgrades for system packages...", "upgrade_complete": "Upgrade complete", - "upgrading_packages": "Upgrading packages…", + "upgrading_packages": "Upgrading packages...", "upnp_dev_not_found": "No UPnP device found", "upnp_disabled": "UPnP turned off", "upnp_enabled": "UPnP turned on", @@ -653,7 +653,7 @@ "yunohost_ca_creation_failed": "Could not create certificate authority", "yunohost_ca_creation_success": "Local certification authority created.", "yunohost_configured": "YunoHost is now configured", - "yunohost_installing": "Installing YunoHost…", + "yunohost_installing": "Installing YunoHost...", "yunohost_not_installed": "YunoHost is not correctly installed. Please run 'yunohost tools postinstall'", "yunohost_postinstall_end_tip": "The post-install completed! To finalize your setup, please consider:\n - adding a first user through the 'Users' section of the webadmin (or 'yunohost user create ' in command-line);\n - diagnose potential issues through the 'Diagnosis' section of the webadmin (or 'yunohost diagnosis run' in command-line);\n - reading the 'Finalizing your setup' and 'Getting to know Yunohost' parts in the admin documentation: https://yunohost.org/admindoc." } From 1a2f26dc34600e97c65eb0359d79b91c85a07eab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 24 Jul 2020 00:52:46 +0200 Subject: [PATCH 052/138] [fix] Let's not redefine the value for the 'service' var ... --- src/yunohost/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 278cea0a4..f12c0743c 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -97,8 +97,8 @@ def service_add(name, description=None, log=None, log_type=None, test_status=Non service["test_status"] = test_status else: # Try to get the description from systemd service - _, service = _get_service_information_from_systemd(name) - type_ = service.get("Type") if service is not None else "" + _, systemd_info = _get_service_information_from_systemd(name) + type_ = systemd_info.get("Type") if systemd_info is not None else "" if type_ == "oneshot": logger.warning("/!\\ Packagers! Please provide a --test_status when adding oneshot-type services in Yunohost, such that it has a reliable way to check if the service is running or not.") From bbbafaddd263e2000542d95a4075b3b61d857638 Mon Sep 17 00:00:00 2001 From: ppr Date: Fri, 26 Jun 2020 17:14:39 +0000 Subject: [PATCH 053/138] Translated using Weblate (French) Currently translated at 99.8% (655 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index b24f13544..4ef18e1e0 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -168,7 +168,7 @@ "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} n’est pas sur le point d’expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)", "certmanager_domain_http_not_working": "Le domaine {domain:s} ne semble pas être accessible via HTTP. Merci de vérifier la catégorie 'Web' dans le diagnostic pour plus d'informations. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)", "certmanager_error_no_A_record": "Aucun enregistrement DNS 'A' n’a été trouvé pour {domain:s}. Vous devez faire pointer votre nom de domaine vers votre machine pour être en mesure d’installer un certificat Let’s Encrypt ! (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", - "certmanager_domain_dns_ip_differs_from_public_ip": "L'enregistrement DNS du domaine {domain:s} est différent de l’adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans le diagnostic. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", + "certmanager_domain_dns_ip_differs_from_public_ip": "L'enregistrement DNS du domaine {domain:s} est différent de l’adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans la section diagnostic. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", "certmanager_cannot_read_cert": "Quelque chose s’est mal passé lors de la tentative d’ouverture du certificat actuel pour le domaine {domain:s} (fichier : {file:s}), la cause est : {reason:s}", "certmanager_cert_install_success_selfsigned": "Le certificat auto-signé est maintenant installé pour le domaine « {domain:s} »", "certmanager_cert_install_success": "Le certificat Let’s Encrypt est maintenant installé pour le domaine « {domain:s} »", @@ -392,7 +392,7 @@ "service_reloaded_or_restarted": "Le service « {service:s} » a été rechargé ou redémarré", "this_action_broke_dpkg": "Cette action a laissé des paquets non configurés par dpkg/apt (les gestionnaires de paquets système). Vous pouvez essayer de résoudre ce problème en vous connectant via SSH et en exécutant `sudo apt install --fix-broken` et/ou `sudo dpkg --configure -a`.", "app_action_cannot_be_ran_because_required_services_down": "Ces services requis doivent être en cours d’exécution pour exécuter cette action : {services}. Essayez de les redémarrer pour continuer (et éventuellement rechercher pourquoi ils sont en panne).", - "admin_password_too_long": "Veuillez choisir un mot de passe de moins de 127 caractères", + "admin_password_too_long": "Veuillez choisir un mot de passe comportant moins de 127 caractères", "log_regen_conf": "Régénérer les configurations du système '{}'", "migration_0009_not_needed": "Cette migration semble avoir déjà été jouée ? On l’ignore.", "regenconf_file_backed_up": "Le fichier de configuration '{conf}' a été sauvegardé sous '{backup}'", @@ -660,7 +660,7 @@ "migration_0015_system_not_fully_up_to_date": "Votre système n'est pas entièrement à jour. Veuillez effectuer une mise à jour normale avant de lancer la migration vers Buster.", "migration_0015_not_enough_free_space": "L'espace libre est très faible dans /var/ ! Vous devriez avoir au moins 1 Go de libre pour effectuer cette migration.", "migration_0015_not_stretch": "La distribution Debian actuelle n'est pas Stretch !", - "migration_0015_yunohost_upgrade": "Lancement de la mise à jour de YunoHost …", + "migration_0015_yunohost_upgrade": "Démarrage de la mise à jour de YunoHost …", "migration_0015_still_on_stretch_after_main_upgrade": "Quelque chose s'est mal passé lors de la mise à niveau, le système semble toujours être sous Debian Stretch", "migration_0015_main_upgrade": "Début de la mise à niveau générale …", "migration_0015_patching_sources_list": "Mise à jour du fichier sources.lists …", From fcdc1bf37bcaaf487d7de6e7154c74253bd90787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quent=C3=AD?= Date: Sun, 21 Jun 2020 12:53:03 +0000 Subject: [PATCH 054/138] Translated using Weblate (Occitan) Currently translated at 57.2% (375 of 656 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/oc/ --- locales/oc.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/locales/oc.json b/locales/oc.json index 90be0b561..7ba8525ce 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -580,5 +580,7 @@ "diagnosis_basesystem_hardware": "L’arquitectura del servidor es {virt} {arch}", "diagnosis_basesystem_hardware_board": "Lo modèl de carta del servidor es {model}", "backup_archive_corrupted": "Sembla que l’archiu de la salvagarda « {archive} » es corromput : {error}", - "diagnosis_domain_expires_in": "{domain} expiraà d’aquí {days} jorns." + "diagnosis_domain_expires_in": "{domain} expiraà d’aquí {days} jorns.", + "migration_0015_cleaning_up": "Netejatge de la memòria cache e dels paquets pas mai necessaris…", + "restore_already_installed_apps": "Restauracion impossibla de las aplicacions seguentas que son ja installadas : {apps}" } From df239f946e5a35d601676025c75cf811712dbb7f Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Tue, 30 Jun 2020 13:15:41 +0000 Subject: [PATCH 055/138] Translated using Weblate (Catalan) Currently translated at 100.0% (657 of 657 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index f29a65c43..254bae8d1 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -666,5 +666,6 @@ "migration_0015_patching_sources_list": "Apedaçament de source.lists…", "migration_0015_start": "Començant la migració a Buster", "migration_description_0015_migrate_to_buster": "Actualitza els sistema a Debian Buster i YunoHost 4.x", - "regenconf_need_to_explicitly_specify_ssh": "La configuració ssh ha estat modificada manualment, però heu d'especificar explícitament la categoria «ssh» amb --force per fer realment els canvis." + "regenconf_need_to_explicitly_specify_ssh": "La configuració ssh ha estat modificada manualment, però heu d'especificar explícitament la categoria «ssh» amb --force per fer realment els canvis.", + "migration_0015_weak_certs": "S'han trobat els següents certificats que encara utilitzen algoritmes de signatura febles i s'han d'actualitzar per a ser compatibles amb la propera versió de nginx: {certs}" } From ae38781714ecb3d991539026e2fb211c11908f03 Mon Sep 17 00:00:00 2001 From: ppr Date: Tue, 30 Jun 2020 20:03:07 +0000 Subject: [PATCH 056/138] Translated using Weblate (French) Currently translated at 99.8% (656 of 657 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index 4ef18e1e0..4b229d470 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -662,10 +662,11 @@ "migration_0015_not_stretch": "La distribution Debian actuelle n'est pas Stretch !", "migration_0015_yunohost_upgrade": "Démarrage de la mise à jour de YunoHost …", "migration_0015_still_on_stretch_after_main_upgrade": "Quelque chose s'est mal passé lors de la mise à niveau, le système semble toujours être sous Debian Stretch", - "migration_0015_main_upgrade": "Début de la mise à niveau générale …", + "migration_0015_main_upgrade": "Démarrage de la mise à niveau générale …", "migration_0015_patching_sources_list": "Mise à jour du fichier sources.lists …", "migration_0015_start": "Démarrage de la migration vers Buster", "migration_description_0015_migrate_to_buster": "Mise à niveau du système vers Debian Buster et YunoHost 4.x", "diagnosis_dns_try_dyndns_update_force": "La configuration DNS de ce domaine devrait être automatiquement gérée par Yunohost. Si ce n'est pas le cas, vous pouvez essayer de forcer une mise à jour en utilisant yunohost dyndns update --force.", - "app_packaging_format_not_supported": "Cette application ne peut pas être installée car son format n'est pas pris en charge par votre version de Yunohost. Vous devriez probablement envisager de mettre à jour votre système." + "app_packaging_format_not_supported": "Cette application ne peut pas être installée car son format n'est pas pris en charge par votre version de Yunohost. Vous devriez probablement envisager de mettre à jour votre système.", + "migration_0015_weak_certs": "Il a été constaté que les certificats suivants utilisent encore des algorithmes de signature peu robustes et doivent être mis à jour pour être compatibles avec la prochaine version de nginx : {certs}" } From 6412959af48bcb93fffb18b0b0de4f5ebbd40894 Mon Sep 17 00:00:00 2001 From: Leandro Noferini Date: Sat, 11 Jul 2020 09:54:26 +0000 Subject: [PATCH 057/138] Translated using Weblate (Italian) Currently translated at 21.2% (139 of 657 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/it/ --- locales/it.json | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/locales/it.json b/locales/it.json index 19884b5bb..88968863c 100644 --- a/locales/it.json +++ b/locales/it.json @@ -27,16 +27,16 @@ "user_deleted": "L'utente è stato cancellato", "admin_password": "Password dell'amministrazione", "admin_password_change_failed": "Impossibile cambiare la password", - "admin_password_changed": "La password dell'amministrazione è stata cambiata", + "admin_password_changed": "La password d'amministrazione è stata cambiata", "app_install_files_invalid": "Questi file non possono essere installati", - "app_manifest_invalid": "Manifesto dell'applicazione non valido: {error}", + "app_manifest_invalid": "C'è qualcosa di scorretto nel manifesto dell'applicazione: {error}", "app_not_correctly_installed": "{app:s} sembra di non essere installata correttamente", "app_not_properly_removed": "{app:s} non è stata correttamente rimossa", "action_invalid": "L'azione '{action:s}' non è valida", - "app_removed": "{app:s} è stata rimossa", + "app_removed": "{app:s} rimossa", "app_sources_fetch_failed": "Impossibile riportare i file sorgenti, l'URL è corretto?", - "app_upgrade_failed": "Impossibile aggiornare {app:s}", - "app_upgraded": "{app:s} è stata aggiornata", + "app_upgrade_failed": "Impossibile aggiornare {app:s}: {error}", + "app_upgraded": "{app:s} aggiornata", "app_requirements_checking": "Controllo i pacchetti richiesti per {app}…", "app_requirements_unmeet": "Requisiti non soddisfatti per {app}, il pacchetto {pkgname} ({version}) deve essere {spec}", "ask_firstname": "Nome", @@ -175,11 +175,11 @@ "app_already_up_to_date": "{app:s} è già aggiornata", "app_change_url_failed_nginx_reload": "Non riesco a riavviare NGINX. Questo è il risultato di 'nginx -t':\n{nginx_errors:s}", "app_change_url_identical_domains": "Il vecchio ed il nuovo dominio/percorso_url sono identici ('{domain:s}{path:s}'), nessuna operazione necessaria.", - "app_change_url_no_script": "L'applicazione '{app_name:s}' non supporta ancora la modifica dell'URL. Forse dovresti aggiornare l'applicazione.", + "app_change_url_no_script": "L'applicazione '{app_name:s}' non supporta ancora la modifica dell'URL. Forse dovresti aggiornarla.", "app_change_url_success": "L'URL dell'applicazione {app:s} è stato cambiato in {domain:s}{path:s}", "app_make_default_location_already_used": "Impostazione dell'applicazione '{app}' come predefinita del dominio non riuscita perché il dominio {domain} è già in uso per l'altra applicazione '{other_app}'", "app_location_unavailable": "Questo URL non è più disponibile o va in conflitto con la/le applicazione/i già installata/e:\n{apps:s}", - "app_upgrade_app_name": "Aggiornando l'applicazione {app}…", + "app_upgrade_app_name": "Aggiornamento dell'applicazione {app}…", "app_upgrade_some_app_failed": "Impossibile aggiornare alcune applicazioni", "backup_abstract_method": "Questo metodo di backup non è ancora stato implementato", "backup_applying_method_borg": "Inviando tutti i file da salvare nel backup nel deposito borg-backup…", @@ -214,9 +214,9 @@ "admin_password_too_long": "Per favore scegli una password più corta di 127 caratteri", "app_not_upgraded": "Impossibile aggiornare le applicazioni '{failed_app}' e di conseguenza l'aggiornamento delle seguenti applicazione è stato cancellato: {apps}", "app_start_install": "Installando l'applicazione '{app}'…", - "app_start_remove": "Rimuovendo l'applicazione {app}…", - "app_start_backup": "Raccogliendo file da salvare nel backup per {app}…", - "app_start_restore": "Ripristinando l'applicazione {app}…", + "app_start_remove": "Rimozione dell'applicazione {app}…", + "app_start_backup": "Raccogliendo file da salvare nel backup per '{app}'…", + "app_start_restore": "Ripristino dell'applicazione '{app}'…", "app_upgrade_several_apps": "Le seguenti app saranno aggiornate : {apps}", "ask_new_domain": "Nuovo dominio", "ask_new_path": "Nuovo percorso", @@ -264,7 +264,7 @@ "global_settings_reset_success": "Successo. Le tue impostazioni precedenti sono state salvate in {path:s}", "global_settings_setting_example_bool": "Esempio di opzione booleana", "global_settings_setting_example_enum": "Esempio di opzione enum", - "already_up_to_date": "Niente da fare! Tutto è già aggiornato!", + "already_up_to_date": "Niente da fare. Tutto è già aggiornato.", "global_settings_setting_example_int": "Esempio di opzione int", "global_settings_setting_example_string": "Esempio di opzione string", "global_settings_setting_security_nginx_compatibility": "Bilanciamento tra compatibilità e sicurezza per il server web nginx. Riguarda gli algoritmi di cifratura (e altri aspetti legati alla sicurezza)", @@ -339,5 +339,6 @@ "app_remove_after_failed_install": "Rimozione dell'applicazione a causa del fallimento dell'installazione…", "app_install_script_failed": "Si è verificato un errore nello script di installazione dell'applicazione", "app_install_failed": "Impossibile installare {app}:{error}", - "app_full_domain_unavailable": "Spiacente, questa app deve essere installata su un proprio dominio, ma altre applicazioni sono state installate sul dominio '{domain}'. Dovresti invece usare un sotto-dominio dedicato per questa app." + "app_full_domain_unavailable": "Spiacente, questa app deve essere installata su un proprio dominio, ma altre applicazioni sono state installate sul dominio '{domain}'. Dovresti invece usare un sotto-dominio dedicato per questa app.", + "app_upgrade_script_failed": "È stato trovato un errore nello script di aggiornamento dell'applicazione" } From fca1df68057e2beeb7117cbb050a163eedaf3fa8 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 27 Jul 2020 15:30:15 +0200 Subject: [PATCH 058/138] [FIX] patch sources list for armbian (#1028) * [FIX] patch sources list for armbian * Update src/yunohost/data_migrations/0015_migrate_to_buster.py Co-authored-by: Alexandre Aubin --- src/yunohost/data_migrations/0015_migrate_to_buster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 4f7f8afeb..6af161c89 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -173,7 +173,7 @@ class MyMigration(Migration): command = "sed -i -e 's@ stretch @ buster @g' " \ "-e '/backports/ s@^#*@#@' " \ "-e 's@ stretch/updates @ buster/updates @g' " \ - "-e 's@ stretch-updates @ buster-updates @g' " \ + "-e 's@ stretch-@ buster-@g' " \ "{}".format(f) os.system(command) From 852dea07f3dc491b72cf8bd6754274ee58e151e4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Jul 2020 17:12:55 +0200 Subject: [PATCH 059/138] Tweak custom disclaimer about the migration still being a bit touchy in preparation for stable release --- src/yunohost/data_migrations/0015_migrate_to_buster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 6af161c89..2de5c7c08 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -150,7 +150,7 @@ class MyMigration(Migration): message = m18n.n("migration_0015_general_warning") - message = "THIS MIGRATION IS CURRENTLY IN ALPHA-STAGE TESTING. YOU SHOULD *NOT* RUN IT FOR NOW ON A PRODUCTION SERVER UNLESS YOU KNOW EXACTLY WHAT YOU ARE DOING OR HAVE A WAY TO ROLLBACK. MORE INFO ON THE FORUM!\n\n" + message + message = "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n" + message if problematic_apps: message += "\n\n" + m18n.n("migration_0015_problematic_apps_warning", problematic_apps=problematic_apps) From 02385c58b0b35ec218d6adc6877efa968f50d5d0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Jul 2020 19:13:28 +0200 Subject: [PATCH 060/138] Update changelog for 3.8.5.5 --- debian/changelog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/debian/changelog b/debian/changelog index 8e3eea7d8..392ead86a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,21 @@ +yunohost (3.8.5.5) stable; urgency=low + + - [enh] Allow to extend the nginx default_server configuration (f1bfc521) + - [mod] Move redirect to /yunohost/admin to a separate nginx conf file to allow customizing it more easily (ac9182d6) + - [enh] Make sure to validate/upgrade that we don't have any active weak certificate used by nginx at the beginning of the buster migration, otherwise nginx will later miserably fail to start (d4358897) + - [fix] get_files_diff crashing if {orig,new}_file is None (7bfe564a) + - [enh] Remove some useless message about file that "wasn't deleted because it doesn't exist." (#1024) + - [mod] Remove useless robot protection code (#1026) + - [fix] Let's not redefine the value for the 'service' var ... (1a2f26dc) + - [fix] More general stretch->buster patching for sources.list (#1028) + - [mod] Tweak custom disclaimer about the migration still being a bit touchy in preparation for stable release (852dea07) + - [mod] Typo/wording in en.json (#1030) + - [i18n] Translations updated for Catalan, French, Italian, Occitan + + Thanks to all contributors <3 ! (É. Gaspar, Kay0u, L. Noferini, ppr, Quentí, xaloc33) + + -- Alexandre Aubin Mon, 27 Jul 2020 19:03:33 +0200 + yunohost (3.8.5.4) testing; urgency=low - [fix] Fix unscd version parsing *again* From 6b72fc5fc5fa42277bdb23e3d166e8c177b04033 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 3 Apr 2020 00:23:36 +0200 Subject: [PATCH 061/138] It just make no sense to backup/restore the mysql password... --- data/hooks/backup/11-conf_ynh_mysql | 13 --------- data/hooks/restore/11-conf_ynh_mysql | 42 ---------------------------- 2 files changed, 55 deletions(-) delete mode 100755 data/hooks/backup/11-conf_ynh_mysql delete mode 100644 data/hooks/restore/11-conf_ynh_mysql diff --git a/data/hooks/backup/11-conf_ynh_mysql b/data/hooks/backup/11-conf_ynh_mysql deleted file mode 100755 index 031707337..000000000 --- a/data/hooks/backup/11-conf_ynh_mysql +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Exit hook on subcommand error or unset variable -set -eu - -# Source YNH helpers -source /usr/share/yunohost/helpers - -# Backup destination -backup_dir="${1}/conf/ynh/mysql" - -# Save MySQL root password -ynh_backup "/etc/yunohost/mysql" "${backup_dir}/root_pwd" diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql deleted file mode 100644 index f54641d6f..000000000 --- a/data/hooks/restore/11-conf_ynh_mysql +++ /dev/null @@ -1,42 +0,0 @@ -backup_dir="$1/conf/ynh/mysql" -MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')" - -. /usr/share/yunohost/helpers - -# ensure that mysql is running -service mysql status >/dev/null 2>&1 \ - || service mysql start - -# retrieve current and new password -[ -f /etc/yunohost/mysql ] \ - && curr_pwd=$(cat /etc/yunohost/mysql) -new_pwd=$(cat "${backup_dir}/root_pwd" || cat "${backup_dir}/mysql") -[ -z "$curr_pwd" ] && curr_pwd="yunohost" -[ -z "$new_pwd" ] && { - new_pwd=$(ynh_string_random 10) -} - -# attempt to change it -mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || { - - echo "It seems that you have already configured MySQL." \ - "YunoHost needs to have a root access to MySQL to runs its" \ - "applications, and is going to reset the MySQL root password." \ - "You can find this new password in /etc/yunohost/mysql." >&2 - - # set new password with debconf - debconf-set-selections << EOF -$MYSQL_PKG mysql-server/root_password password $new_pwd -$MYSQL_PKG mysql-server/root_password_again password $new_pwd -EOF - - # reconfigure Debian package - dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1 -} - -# store new root password -echo "$new_pwd" | tee /etc/yunohost/mysql -chmod 400 /etc/yunohost/mysql - -# reload the grant tables -mysqladmin -s -u root -p"$new_pwd" reload From bc19bef59e41d0227eb95b2a2c67d81c7841c77c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Apr 2020 18:57:35 +0200 Subject: [PATCH 062/138] Workaround for custom backup/restore hook issue --- data/hooks/restore/11-conf_ynh_mysql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 data/hooks/restore/11-conf_ynh_mysql diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql new file mode 100644 index 000000000..11353425a --- /dev/null +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -0,0 +1,5 @@ +# We don't backup/restore mysql password anymore +# c.f. https://github.com/YunoHost/yunohost/pull/912 + +# This is a dummy empty file as a workaround for +# https://github.com/YunoHost/issues/issues/1553 until it is fixed From 8d5422d13f33c65a4ae321dc6592079dbb83e0f1 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Sun, 8 Mar 2020 00:18:48 +0100 Subject: [PATCH 063/138] [fix] Fix getopts and helpers --- data/helpers.d/getopts | 8 ++++---- data/helpers.d/mysql | 15 +++++++++++++-- data/helpers.d/postgresql | 14 ++++++++++++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index 10c06930c..a4bbe20e6 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -80,10 +80,10 @@ ynh_handle_getopts_args () { arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}" # And replace long option (value of the option_flag) by the short option, the option_flag itself # (e.g. for [u]=user, --user will be -u) - # Replace long option with = - arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}/-${option_flag} }" - # And long option without = - arguments[arg]="${arguments[arg]//--${args_array[$option_flag]%=}/-${option_flag}}" + # Replace long option with = (match the beginning of the argument) + arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")" + # And long option without = (match the whole line) + arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")" done done diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 84acc1029..05f75e0a2 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -44,8 +44,13 @@ ynh_mysql_execute_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" + if [ -n "$database" ] + then + database="--database=$database" + fi + ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \ - --database="$database" <<< "$sql" + $database <<< "$sql" } # Execute a command from a file as root user @@ -65,8 +70,14 @@ ynh_mysql_execute_file_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" + if [ -n "$database" ] + then + database="--database=$database" + fi + + ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \ - --database="$database" < "$file" + $database < "$file" } # Create a database and grant optionnaly privilegies to a user diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 284c02d3e..7eb4e7289 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -46,8 +46,13 @@ ynh_psql_execute_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" + if [ -n "$database" ] + then + database="--database=$database" + fi + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" <<<"$sql" + $database <<<"$sql" } # Execute a command from a file as root user @@ -67,8 +72,13 @@ ynh_psql_execute_file_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" + if [ -n "$database" ] + then + database="--database=$database" + fi + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" <"$file" + $database <"$file" } # Create a database and grant optionnaly privilegies to a user From 834b76700062784de8f2cec5bbbf6318fba39367 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 12 Aug 2019 11:43:49 +0200 Subject: [PATCH 064/138] [fix] Explicitly create home using mkhomedir_helper instead of obscure pam rule that doesn't work anymore --- debian/conf/pam/mkhomedir | 6 ------ debian/install | 1 - debian/postinst | 5 ----- src/yunohost/firewall.py | 6 +----- src/yunohost/user.py | 3 +-- 5 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 debian/conf/pam/mkhomedir diff --git a/debian/conf/pam/mkhomedir b/debian/conf/pam/mkhomedir deleted file mode 100644 index eedc8b745..000000000 --- a/debian/conf/pam/mkhomedir +++ /dev/null @@ -1,6 +0,0 @@ -Name: Create home directory during login -Default: yes -Priority: 900 -Session-Type: Additional -Session: - required pam_mkhomedir.so umask=0022 skel=/etc/skel diff --git a/debian/install b/debian/install index 777d7973e..1691a4849 100644 --- a/debian/install +++ b/debian/install @@ -13,7 +13,6 @@ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ data/helpers /usr/share/yunohost/ data/helpers.d/* /usr/share/yunohost/helpers.d/ -debian/conf/pam/* /usr/share/pam-configs/ lib/metronome/modules/* /usr/lib/metronome/modules/ locales/* /usr/lib/moulinette/yunohost/locales/ src/yunohost /usr/lib/moulinette diff --git a/debian/postinst b/debian/postinst index 9c78c8432..4b43b2506 100644 --- a/debian/postinst +++ b/debian/postinst @@ -29,11 +29,6 @@ do_configure() { # Yunoprompt systemctl enable yunoprompt.service - - # remove old PAM config and update it - [[ ! -f /usr/share/pam-configs/my_mkhomedir ]] \ - || rm /usr/share/pam-configs/my_mkhomedir - pam-auth-update --package } # summary of how this script can be called: diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index 9d209dbb8..4c3cfb0f7 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -26,11 +26,7 @@ import os import sys import yaml -try: - import miniupnpc -except ImportError: - sys.stderr.write('Error: Yunohost CLI Require miniupnpc lib\n') - sys.exit(1) +import miniupnpc from moulinette import m18n from yunohost.utils.error import YunohostError diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 7f8f2dc35..ed2a44670 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -208,8 +208,7 @@ def user_create(operation_logger, username, firstname, lastname, mail, password, try: # Attempt to create user home folder - subprocess.check_call( - ['su', '-', username, '-c', "''"]) + subprocess.check_call(["mkhomedir_helper", username]) except subprocess.CalledProcessError: if not os.path.isdir('/home/{0}'.format(username)): logger.warning(m18n.n('user_home_creation_failed'), From 0a9f4d59cb7cd6438376035ebc4e4326981d0213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Mollier?= Date: Wed, 14 Aug 2019 16:28:40 +0200 Subject: [PATCH 065/138] [fix] Ldap interface seems to expect lists everywhere now? --- src/yunohost/app.py | 1 + src/yunohost/ssh.py | 4 ++-- src/yunohost/tools.py | 18 +++++++++--------- src/yunohost/user.py | 40 ++++++++++++++++++++-------------------- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 6af892b03..afa81127d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -865,6 +865,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu os.path.join(extracted_app_folder, 'scripts/remove'), args=[app_instance_name], env=env_dict_remove )[0] + # Here again, calling hook_exec could fail miserably, or get # manually interrupted (by mistake or because script was stuck) # In that case we still want to proceed with the rest of the diff --git a/src/yunohost/ssh.py b/src/yunohost/ssh.py index f0110b34e..be876ce16 100644 --- a/src/yunohost/ssh.py +++ b/src/yunohost/ssh.py @@ -25,7 +25,7 @@ def user_ssh_allow(username): from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - ldap.update('uid=%s,ou=users' % username, {'loginShell': '/bin/bash'}) + ldap.update('uid=%s,ou=users' % username, {'loginShell': ['/bin/bash']}) # Somehow this is needed otherwise the PAM thing doesn't forget about the # old loginShell value ? @@ -46,7 +46,7 @@ def user_ssh_disallow(username): from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - ldap.update('uid=%s,ou=users' % username, {'loginShell': '/bin/false'}) + ldap.update('uid=%s,ou=users' % username, {'loginShell': ['/bin/false']}) # Somehow this is needed otherwise the PAM thing doesn't forget about the # old loginShell value ? diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index d5fab2fb1..75a27aa66 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -88,15 +88,15 @@ def tools_ldapinit(): logger.warn("Error when trying to inject '%s' -> '%s' into ldap: %s" % (rdn, attr_dict, e)) admin_dict = { - 'cn': 'admin', - 'uid': 'admin', - 'description': 'LDAP Administrator', - 'gidNumber': '1007', - 'uidNumber': '1007', - 'homeDirectory': '/home/admin', - 'loginShell': '/bin/bash', + 'cn': ['admin'], + 'uid': ['admin'], + 'description': ['LDAP Administrator'], + 'gidNumber': ['1007'], + 'uidNumber': ['1007'], + 'homeDirectory': ['/home/admin'], + 'loginShell': ['/bin/bash'], 'objectClass': ['organizationalRole', 'posixAccount', 'simpleSecurityObject'], - 'userPassword': 'yunohost' + 'userPassword': ['yunohost'] } ldap.update('cn=admin', admin_dict) @@ -140,7 +140,7 @@ def tools_adminpw(new_password, check_strength=True): ldap = _get_ldap_interface() try: - ldap.update("cn=admin", {"userPassword": new_hash, }) + ldap.update("cn=admin", {"userPassword": [ new_hash ], }) except: logger.exception('unable to change admin password') raise YunohostError('admin_password_change_failed') diff --git a/src/yunohost/user.py b/src/yunohost/user.py index ed2a44670..d19da177c 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -178,19 +178,19 @@ def user_create(operation_logger, username, firstname, lastname, mail, password, fullname = '%s %s' % (firstname, lastname) attr_dict = { 'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh'], - 'givenName': firstname, - 'sn': lastname, - 'displayName': fullname, - 'cn': fullname, - 'uid': username, - 'mail': mail, - 'maildrop': username, - 'mailuserquota': mailbox_quota, - 'userPassword': _hash_user_password(password), - 'gidNumber': uid, - 'uidNumber': uid, - 'homeDirectory': '/home/' + username, - 'loginShell': '/bin/false' + 'givenName': [firstname], + 'sn': [lastname], + 'displayName': [fullname], + 'cn': [fullname], + 'uid': [username], + 'mail': mail, # NOTE: this one seems to be already a list + 'maildrop': [username], + 'mailuserquota': [mailbox_quota], + 'userPassword': [_hash_user_password(password)], + 'gidNumber': [uid], + 'uidNumber': [uid], + 'homeDirectory': ['/home/' + username], + 'loginShell': ['/bin/false'] } # If it is the first user, add some aliases @@ -316,21 +316,21 @@ def user_update(operation_logger, username, firstname=None, lastname=None, mail= # Get modifications from arguments new_attr_dict = {} if firstname: - new_attr_dict['givenName'] = firstname # TODO: Validate - new_attr_dict['cn'] = new_attr_dict['displayName'] = firstname + ' ' + user['sn'][0] + new_attr_dict['givenName'] = [firstname] # TODO: Validate + new_attr_dict['cn'] = new_attr_dict['displayName'] = [firstname + ' ' + user['sn'][0]] if lastname: - new_attr_dict['sn'] = lastname # TODO: Validate - new_attr_dict['cn'] = new_attr_dict['displayName'] = user['givenName'][0] + ' ' + lastname + new_attr_dict['sn'] = [lastname] # TODO: Validate + new_attr_dict['cn'] = new_attr_dict['displayName'] = [user['givenName'][0] + ' ' + lastname] if lastname and firstname: - new_attr_dict['cn'] = new_attr_dict['displayName'] = firstname + ' ' + lastname + new_attr_dict['cn'] = new_attr_dict['displayName'] = [firstname + ' ' + lastname] if change_password: # Ensure sufficiently complex password assert_password_is_strong_enough("user", change_password) - new_attr_dict['userPassword'] = _hash_user_password(change_password) + new_attr_dict['userPassword'] = [_hash_user_password(change_password)] if mail: main_domain = _get_maindomain() @@ -395,7 +395,7 @@ def user_update(operation_logger, username, firstname=None, lastname=None, mail= new_attr_dict['maildrop'] = user['maildrop'] if mailbox_quota is not None: - new_attr_dict['mailuserquota'] = mailbox_quota + new_attr_dict['mailuserquota'] = [mailbox_quota] operation_logger.start() From 4f9216b07c939e57074ff2d3a9fd00a1b9820d70 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Feb 2020 21:17:14 +0100 Subject: [PATCH 066/138] [deb] Tmp version for debian builds during experimental/alpha dev for Buster --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 392ead86a..7f1e50190 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (4.0.0~alpha) unstable; urgency=low + + - Tmp version for debian builds during experimental/alpha dev for Buster + + -- Alexandre Aubin Fri, 7 Feb 2020 21:15:00 +0000 + yunohost (3.8.5.5) stable; urgency=low - [enh] Allow to extend the nginx default_server configuration (f1bfc521) From e9d5abf5e6efe0430a2992fbe133cae79f489718 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Feb 2020 22:21:08 +0100 Subject: [PATCH 067/138] [deb] Clean control file, remove some legacy Conflicts and Replaces -- it is safe to do so because 3.x instances already have these Conflicts / Replaces, so it's okay to remove them for 4.x --- debian/control | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/debian/control b/debian/control index 96b5bbb08..7cb1761b8 100644 --- a/debian/control +++ b/debian/control @@ -14,9 +14,8 @@ Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 3.7), ssowat (>= 3.7) , python-psutil, python-requests, python-dnspython, python-openssl , python-miniupnpc, python-dbus, python-jinja2 - , python-toml, python-packaging + , python-toml, python-packaging, python-publicsuffix , apt, apt-transport-https, dirmngr - , nginx, nginx-extras (>=1.6.2) , php-fpm, php-ldap, php-intl , mariadb-server, php-mysql | php-mysqlnd , openssh-server, iptables, fail2ban, dnsutils, bind9utils @@ -25,10 +24,10 @@ Depends: ${python:Depends}, ${misc:Depends} , dnsmasq, avahi-daemon, libnss-mdns, resolvconf, libnss-myhostname , postfix, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre , dovecot-core, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved, dovecot-antispam - , rspamd (>= 1.6.0), opendkim-tools, postsrsd, procmail, mailutils + , rspamd, opendkim-tools, postsrsd, procmail, mailutils , redis-server , metronome (>=3.14.0) - , git, curl, wget, cron, unzip, jq + , git, curl, wget, cron, unzip, jq, bc , lsb-release, haveged, fake-hwclock, equivs, lsof, whois, python-publicsuffix Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping @@ -39,17 +38,8 @@ Recommends: yunohost-admin , libdbd-ldap-perl, libnet-dns-perl Suggests: htop, vim, rsync, acpi-support-base, udisks2 Conflicts: iptables-persistent - , moulinette-yunohost, yunohost-config - , yunohost-config-others, yunohost-config-postfix - , yunohost-config-dovecot, yunohost-config-slapd - , yunohost-config-nginx, yunohost-config-amavis - , yunohost-config-mysql, yunohost-predepends - , apache2, bind9 -Replaces: moulinette-yunohost, yunohost-config - , yunohost-config-others, yunohost-config-postfix - , yunohost-config-dovecot, yunohost-config-slapd - , yunohost-config-nginx, yunohost-config-amavis - , yunohost-config-mysql, yunohost-predepends + , apache2 + , bind9 Description: manageable and configured self-hosting server YunoHost aims to make self-hosting accessible to everyone. It configures an email, Web and IM server alongside a LDAP base. It also provides From d935ab64f6141f36a386b709c13635bf11520c01 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 2 May 2020 04:37:09 +0200 Subject: [PATCH 068/138] [deb] Add conflicts with versions from backports for critical dependencies --- debian/control | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/control b/debian/control index 7cb1761b8..6abb4c0c4 100644 --- a/debian/control +++ b/debian/control @@ -40,6 +40,13 @@ Suggests: htop, vim, rsync, acpi-support-base, udisks2 Conflicts: iptables-persistent , apache2 , bind9 + , nginx-extras (>= 1.16) + , openssl (>= 1.1.1g) + , slapd (>= 2.4.49) + , dovecot-core (>= 1:2.3.7) + , redis-server (>= 5:5.0.7) + , fail2ban (>= 0.11) + , iptables (>= 1.8.3) Description: manageable and configured self-hosting server YunoHost aims to make self-hosting accessible to everyone. It configures an email, Web and IM server alongside a LDAP base. It also provides From 04894b1b87884516a9f2da1217cb0cf5b76103c1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Feb 2020 22:32:55 +0100 Subject: [PATCH 069/138] [deb] Update moulinette and ssowat required version to >= 4.0.0~alpha --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 6abb4c0c4..cd6b6e495 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python:Depends}, ${misc:Depends} - , moulinette (>= 3.7), ssowat (>= 3.7) + , moulinette (>= 4.0.0~alpha), ssowat (>= 4.0.0~alpha) , python-psutil, python-requests, python-dnspython, python-openssl , python-miniupnpc, python-dbus, python-jinja2 , python-toml, python-packaging, python-publicsuffix From 60dd1fc8481c424e3d639c051e2949e9dbd98d8b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 5 Mar 2020 22:52:19 +0100 Subject: [PATCH 070/138] [cleanup] Remove legacy migrations and code snippets which won't be needed anymore on buster --- locales/en.json | 48 +-- src/yunohost/app.py | 6 - .../0001_change_cert_group_to_sslcert.py | 15 - .../0002_migrate_to_tsig_sha256.py | 86 ---- .../0003_migrate_to_stretch.py | 379 ------------------ .../0004_php5_to_php7_pools.py | 99 ----- .../0005_postgresql_9p4_to_9p6.py | 41 -- .../0006_sync_admin_and_root_passwords.py | 78 ---- ...0007_ssh_conf_managed_by_yunohost_step1.py | 70 ---- ...0008_ssh_conf_managed_by_yunohost_step2.py | 105 ----- .../0009_decouple_regenconf_from_services.py | 39 -- .../0010_migrate_to_apps_json.py | 13 - ...stgresql_password_to_md5_authentication.py | 16 - .../0013_futureproof_apps_catalog_system.py | 51 --- .../0014_remove_app_status_json.py | 31 -- src/yunohost/dyndns.py | 16 - src/yunohost/regenconf.py | 10 - src/yunohost/tests/test_appscatalog.py | 55 --- 18 files changed, 1 insertion(+), 1157 deletions(-) delete mode 100644 src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py delete mode 100644 src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py delete mode 100644 src/yunohost/data_migrations/0003_migrate_to_stretch.py delete mode 100644 src/yunohost/data_migrations/0004_php5_to_php7_pools.py delete mode 100644 src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py delete mode 100644 src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py delete mode 100644 src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py delete mode 100644 src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py delete mode 100644 src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py delete mode 100644 src/yunohost/data_migrations/0010_migrate_to_apps_json.py delete mode 100644 src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py delete mode 100644 src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py delete mode 100644 src/yunohost/data_migrations/0014_remove_app_status_json.py diff --git a/locales/en.json b/locales/en.json index a76176dc1..c0f79436f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -406,54 +406,8 @@ "mail_unavailable": "This e-mail address is reserved and shall be automatically allocated to the very first user", "main_domain_change_failed": "Unable to change the main domain", "main_domain_changed": "The main domain has been changed", - "migrate_tsig_end": "Migration to HMAC-SHA-512 finished", - "migrate_tsig_failed": "Could not migrate the DynDNS domain '{domain}' to HMAC-SHA-512, rolling back. Error: {error_code}, {error}", - "migrate_tsig_start": "Insufficiently secure key algorithm detected for TSIG signature of the domain '{domain}', initiating migration to the more secure HMAC-SHA-512", - "migrate_tsig_wait": "Waiting three minutes for the DynDNS server to take the new key into account...", - "migrate_tsig_wait_2": "2 min...", - "migrate_tsig_wait_3": "1 min...", - "migrate_tsig_wait_4": "30 seconds...", - "migrate_tsig_not_needed": "You do not appear to use a DynDNS domain, so no migration is needed.", - "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", - "migration_description_0002_migrate_to_tsig_sha256": "Improve security of DynDNS TSIG updates by using SHA-512 instead of MD5", - "migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0", - "migration_description_0004_php5_to_php7_pools": "Reconfigure the PHP pools to use PHP 7 instead of 5", - "migration_description_0005_postgresql_9p4_to_9p6": "Migrate databases from PostgreSQL 9.4 to 9.6", - "migration_description_0006_sync_admin_and_root_passwords": "Synchronize admin and root passwords", - "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)", - "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)", - "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services", - "migration_description_0010_migrate_to_apps_json": "Remove deprecated apps catalogs and use the new unified 'apps.json' list instead (outdated, replaced by migration 13)", - "migration_description_0011_setup_group_permission": "Set up user groups and permissions for apps and services", - "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections", - "migration_description_0013_futureproof_apps_catalog_system": "Migrate to the new future-proof apps catalog system", - "migration_description_0014_remove_app_status_json": "Remove legacy status.json app files", + "migration_description_0011_setup_group_permission": "Set up user group and set up permission for apps and services", "migration_description_0015_migrate_to_buster": "Upgrade the system to Debian Buster and YunoHost 4.x", - "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", - "migration_0003_patching_sources_list": "Patching the sources.lists...", - "migration_0003_main_upgrade": "Starting main upgrade...", - "migration_0003_fail2ban_upgrade": "Starting the Fail2Ban upgrade...", - "migration_0003_restoring_origin_nginx_conf": "Your file /etc/nginx/nginx.conf was edited somehow. The migration is going to reset it to its original state first… The previous file will be available as {backup_dest}.", - "migration_0003_yunohost_upgrade": "Starting the YunoHost package upgrade… The migration will end, but the actual upgrade will happen immediately afterwards. After the operation is complete, you might have to log in to the webadmin page again.", - "migration_0003_not_jessie": "The current Debian distribution is not Jessie!", - "migration_0003_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Stretch.", - "migration_0003_still_on_jessie_after_main_upgrade": "Something went wrong during the main upgrade: Is the system still on Jessie‽ To investigate the issue, please look at {log}:s…", - "migration_0003_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.\n\nAdditionally, the port for SMTP, used by external e-mail clients (like Thunderbird or K9-Mail) was changed from 465 (SSL/TLS) to 587 (STARTTLS). The old port (465) will automatically be closed, and the new port (587) will be opened in the firewall. You and your users *will* have to adapt the configuration of your e-mail clients accordingly.", - "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", - "migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", - "migration_0005_postgresql_94_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", - "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 is installed, but not postgresql 9.6‽ Something weird might have happened on your system :(…", - "migration_0005_not_enough_space": "Make sufficient space available in {path} to run the migration.", - "migration_0006_disclaimer": "YunoHost now expects the admin and root passwords to be synchronized. This migration replaces your root password with the admin password.", - "migration_0007_cancelled": "Could not improve the way your SSH configuration is managed.", - "migration_0007_cannot_restart": "SSH can't be restarted after trying to cancel migration number 6.", - "migration_0008_general_disclaimer": "To improve the security of your server, it is recommended to let YunoHost manage the SSH configuration. Your current SSH setup differs from the recommendation. If you let YunoHost reconfigure it, the way you connect to your server through SSH will change thusly:", - "migration_0008_port": "• You will have to connect using port 22 instead of your current custom SSH port. Feel free to reconfigure it;", - "migration_0008_root": "• You will not be able to connect as root through SSH. Instead you should use the admin user;", - "migration_0008_dsa": "• The DSA key will be turned off. Hence, you might need to invalidate a spooky warning from your SSH client, and recheck the fingerprint of your server;", - "migration_0008_warning": "If you understand those warnings and want YunoHost to override your current configuration, run the migration. Otherwise, you can also skip the migration, though it is not recommended.", - "migration_0008_no_warning": "Overriding your SSH configuration should be safe, though this cannot be promised! Run the migration to override it. Otherwise, you can also skip the migration, though it is not recommended.", - "migration_0009_not_needed": "This migration already happened somehow... (?) Skipping.", "migration_0011_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_0011_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", "migration_0011_create_group": "Creating a group for each user…", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index afa81127d..032536247 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2683,12 +2683,6 @@ def _read_apps_catalog_list(): Read the json corresponding to the list of apps catalogs """ - # Legacy code - can be removed after moving to buster (if the migration got merged before buster) - if os.path.exists('/etc/yunohost/appslists.json'): - from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("futureproof_apps_catalog_system") - migration.run() - try: list_ = read_yaml(APPS_CATALOG_CONF) # Support the case where file exists but is empty diff --git a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py deleted file mode 100644 index 5670f3329..000000000 --- a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py +++ /dev/null @@ -1,15 +0,0 @@ -import subprocess -import glob -from yunohost.tools import Migration -from moulinette.utils.filesystem import chown - - -class MyMigration(Migration): - - "Change certificates group permissions from 'metronome' to 'ssl-cert'" - - all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem") - - def run(self): - for filename in self.all_certificate_files: - chown(filename, uid="root", gid="ssl-cert") diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py deleted file mode 100644 index 65158ba2c..000000000 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ /dev/null @@ -1,86 +0,0 @@ -import glob -import os -import requests -import base64 -import time -import json - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger - -from yunohost.tools import Migration -from yunohost.dyndns import _guess_current_dyndns_domain - -logger = getActionLogger('yunohost.migration') - - -class MyMigration(Migration): - - "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG" - - def run(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): - - if domain is None or private_key_path is None: - try: - (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) - assert "+157" in private_key_path - except (YunohostError, AssertionError): - logger.info(m18n.n("migrate_tsig_not_needed")) - return - - logger.info(m18n.n('migrate_tsig_start', domain=domain)) - public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" - public_key_md5 = open(public_key_path).read().strip().split(' ')[-1] - - os.system('cd /etc/yunohost/dyndns && ' - 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) - os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') - - # +165 means that this file store a hmac-sha512 key - new_key_path = glob.glob('/etc/yunohost/dyndns/*+165*.key')[0] - public_key_sha512 = open(new_key_path).read().strip().split(' ', 6)[-1] - - try: - r = requests.put('https://%s/migrate_key_to_sha512/' % (dyn_host), - data={ - 'public_key_md5': base64.b64encode(public_key_md5), - 'public_key_sha512': base64.b64encode(public_key_sha512), - }, timeout=30) - except requests.ConnectionError: - raise YunohostError('no_internet_connection') - - if r.status_code != 201: - try: - error = json.loads(r.text)['error'] - except Exception: - # failed to decode json - error = r.text - - import traceback - from StringIO import StringIO - stack = StringIO() - traceback.print_stack(file=stack) - logger.error(stack.getvalue()) - - # Migration didn't succeed, so we rollback and raise an exception - os.system("mv /etc/yunohost/dyndns/*+165* /tmp") - - raise YunohostError('migrate_tsig_failed', domain=domain, - error_code=str(r.status_code), error=error) - - # remove old certificates - os.system("mv /etc/yunohost/dyndns/*+157* /tmp") - - # sleep to wait for dyndns cache invalidation - logger.info(m18n.n('migrate_tsig_wait')) - time.sleep(60) - logger.info(m18n.n('migrate_tsig_wait_2')) - time.sleep(60) - logger.info(m18n.n('migrate_tsig_wait_3')) - time.sleep(30) - logger.info(m18n.n('migrate_tsig_wait_4')) - time.sleep(30) - - logger.info(m18n.n('migrate_tsig_end')) - return diff --git a/src/yunohost/data_migrations/0003_migrate_to_stretch.py b/src/yunohost/data_migrations/0003_migrate_to_stretch.py deleted file mode 100644 index e916b1ae8..000000000 --- a/src/yunohost/data_migrations/0003_migrate_to_stretch.py +++ /dev/null @@ -1,379 +0,0 @@ -import glob -import os -from shutil import copy2 - -from moulinette import m18n, msettings -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file - -from yunohost.tools import Migration -from yunohost.app import unstable_apps -from yunohost.service import _run_service_command -from yunohost.regenconf import (manually_modified_files, - manually_modified_files_compared_to_debian_default) -from yunohost.utils.filesystem import free_space_in_directory -from yunohost.utils.packages import get_ynh_package_version -from yunohost.utils.network import get_network_interfaces -from yunohost.firewall import firewall_allow, firewall_disallow - -logger = getActionLogger('yunohost.migration') - -YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"] - - -class MyMigration(Migration): - - "Upgrade the system to Debian Stretch and Yunohost 3.0" - - mode = "manual" - - def run(self): - - self.logfile = "/var/log/yunohost/{}.log".format(self.name) - - self.check_assertions() - - logger.info(m18n.n("migration_0003_start", logfile=self.logfile)) - - # Preparing the upgrade - self.restore_original_nginx_conf_if_needed() - - logger.info(m18n.n("migration_0003_patching_sources_list")) - self.patch_apt_sources_list() - self.backup_files_to_keep() - self.apt_update() - apps_packages = self.get_apps_equivs_packages() - self.unhold(["metronome"]) - self.hold(YUNOHOST_PACKAGES + apps_packages + ["fail2ban"]) - - # Main dist-upgrade - logger.info(m18n.n("migration_0003_main_upgrade")) - _run_service_command("stop", "mysql") - self.apt_dist_upgrade(conf_flags=["old", "miss", "def"]) - _run_service_command("start", "mysql") - if self.debian_major_version() == 8: - raise YunohostError("migration_0003_still_on_jessie_after_main_upgrade", log=self.logfile) - - # Specific upgrade for fail2ban... - logger.info(m18n.n("migration_0003_fail2ban_upgrade")) - self.unhold(["fail2ban"]) - # Don't move this if folder already exists. If it does, we probably are - # running this script a 2nd, 3rd, ... time but /etc/fail2ban will - # be re-created only for the first dist-upgrade of fail2ban - if not os.path.exists("/etc/fail2ban.old"): - os.system("mv /etc/fail2ban /etc/fail2ban.old") - self.apt_dist_upgrade(conf_flags=["new", "miss", "def"]) - _run_service_command("restart", "fail2ban") - - self.disable_predicable_interface_names() - - # Clean the mess - os.system("apt autoremove --assume-yes") - os.system("apt clean --assume-yes") - - # We moved to port 587 for SMTP - # https://busylog.net/smtp-tls-ssl-25-465-587/ - firewall_allow("Both", 587) - firewall_disallow("Both", 465) - - # Upgrade yunohost packages - logger.info(m18n.n("migration_0003_yunohost_upgrade")) - self.restore_files_to_keep() - self.unhold(YUNOHOST_PACKAGES + apps_packages) - self.upgrade_yunohost_packages() - - def debian_major_version(self): - # The python module "platform" and lsb_release are not reliable because - # on some setup, they still return Release=8 even after upgrading to - # stretch ... (Apparently this is related to OVH overriding some stuff - # with /etc/lsb-release for instance -_-) - # Instead, we rely on /etc/os-release which should be the raw info from - # the distribution... - return int(check_output("grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2")) - - def yunohost_major_version(self): - return int(get_ynh_package_version("yunohost")["version"].split('.')[0]) - - def check_assertions(self): - - # Be on jessie (8.x) and yunohost 2.x - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 9.x but yunohost package - # would still be in 2.x... - if not self.debian_major_version() == 8 \ - and not self.yunohost_major_version() == 2: - raise YunohostError("migration_0003_not_jessie") - - # Have > 1 Go free space on /var/ ? - if free_space_in_directory("/var/") / (1024**3) < 1.0: - raise YunohostError("There is not enough free space in /var/ to run the migration. You need at least 1GB free space") - - # Check system is up to date - # (but we don't if 'stretch' is already in the sources.list ... - # which means maybe a previous upgrade crashed and we're re-running it) - if " stretch " not in read_file("/etc/apt/sources.list"): - self.apt_update() - apt_list_upgradable = check_output("apt list --upgradable -a") - if "upgradable" in apt_list_upgradable: - raise YunohostError("migration_0003_system_not_fully_up_to_date") - - @property - def disclaimer(self): - - # Avoid having a super long disclaimer + uncessary check if we ain't - # on jessie / yunohost 2.x anymore - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 9.x but yunohost package - # would still be in 2.x... - if not self.debian_major_version() == 8 \ - and not self.yunohost_major_version() == 2: - return None - - # Get list of problematic apps ? I.e. not official or community+working - problematic_apps = unstable_apps() - problematic_apps = "".join(["\n - " + app for app in problematic_apps]) - - # Manually modified files ? (c.f. yunohost service regen-conf) - modified_files = manually_modified_files() - # We also have a specific check for nginx.conf which some people - # modified and needs to be upgraded... - if "/etc/nginx/nginx.conf" in manually_modified_files_compared_to_debian_default(): - modified_files.append("/etc/nginx/nginx.conf") - modified_files = "".join(["\n - " + f for f in modified_files]) - - message = m18n.n("migration_0003_general_warning") - - if problematic_apps: - message += "\n\n" + m18n.n("migration_0003_problematic_apps_warning", problematic_apps=problematic_apps) - - if modified_files: - message += "\n\n" + m18n.n("migration_0003_modified_files", manually_modified_files=modified_files) - - return message - - def patch_apt_sources_list(self): - - sources_list = glob.glob("/etc/apt/sources.list.d/*.list") - sources_list.append("/etc/apt/sources.list") - - # This : - # - replace single 'jessie' occurence by 'stretch' - # - comments lines containing "backports" - # - replace 'jessie/updates' by 'strech/updates' (or same with a -) - # - switch yunohost's repo to forge - for f in sources_list: - command = "sed -i -e 's@ jessie @ stretch @g' " \ - "-e '/backports/ s@^#*@#@' " \ - "-e 's@ jessie/updates @ stretch/updates @g' " \ - "-e 's@ jessie-updates @ stretch-updates @g' " \ - "-e 's@repo.yunohost@forge.yunohost@g' " \ - "{}".format(f) - os.system(command) - - def get_apps_equivs_packages(self): - - command = "dpkg --get-selections" \ - " | grep -v deinstall" \ - " | awk '{print $1}'" \ - " | { grep 'ynh-deps$' || true; }" - - output = check_output(command).strip() - - return output.split('\n') if output else [] - - def hold(self, packages): - for package in packages: - os.system("apt-mark hold {}".format(package)) - - def unhold(self, packages): - for package in packages: - os.system("apt-mark unhold {}".format(package)) - - def apt_update(self): - - command = "apt-get update" - logger.debug("Running apt command :\n{}".format(command)) - command += " 2>&1 | tee -a {}".format(self.logfile) - - os.system(command) - - def upgrade_yunohost_packages(self): - - # - # Here we use a dirty hack to run a command after the current - # "yunohost tools migrations migrate", because the upgrade of - # yunohost will also trigger another "yunohost tools migrations migrate" - # (also the upgrade of the package, if executed from the webadmin, is - # likely to kill/restart the api which is in turn likely to kill this - # command before it ends...) - # - - MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" - - upgrade_command = "" - upgrade_command += " DEBIAN_FRONTEND=noninteractive" - upgrade_command += " APT_LISTCHANGES_FRONTEND=none" - upgrade_command += " apt-get install" - upgrade_command += " --assume-yes " - upgrade_command += " ".join(YUNOHOST_PACKAGES) - # We also install php-zip and php7.0-acpu to fix an issue with - # nextcloud and kanboard that need it when on stretch. - upgrade_command += " php-zip php7.0-apcu" - upgrade_command += " 2>&1 | tee -a {}".format(self.logfile) - - wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) - - command = "({} && {}; echo 'Migration complete!') &".format(wait_until_end_of_yunohost_command, - upgrade_command) - - logger.debug("Running command :\n{}".format(command)) - - os.system(command) - - def apt_dist_upgrade(self, conf_flags): - - # Make apt-get happy - os.system("echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections") - # Don't send an email to root about the postgresql migration. It should be handled automatically after. - os.system("echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections") - - command = "" - command += " DEBIAN_FRONTEND=noninteractive" - command += " APT_LISTCHANGES_FRONTEND=none" - command += " apt-get" - command += " --fix-broken --show-upgraded --assume-yes" - for conf_flag in conf_flags: - command += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag) - command += " dist-upgrade" - - logger.debug("Running apt command :\n{}".format(command)) - - command += " 2>&1 | tee -a {}".format(self.logfile) - - is_api = msettings.get('interface') == 'api' - if is_api: - callbacks = ( - lambda l: logger.info(l.rstrip()), - lambda l: logger.warning(l.rstrip()), - ) - call_async_output(command, callbacks, shell=True) - else: - # We do this when running from the cli to have the output of the - # command showing in the terminal, since 'info' channel is only - # enabled if the user explicitly add --verbose ... - os.system(command) - - # Those are files that should be kept and restored before the final switch - # to yunohost 3.x... They end up being modified by the various dist-upgrades - # (or need to be taken out momentarily), which then blocks the regen-conf - # as they are flagged as "manually modified"... - files_to_keep = [ - "/etc/mysql/my.cnf", - "/etc/nslcd.conf", - "/etc/postfix/master.cf", - "/etc/fail2ban/filter.d/yunohost.conf" - ] - - def backup_files_to_keep(self): - - logger.debug("Backuping specific files to keep ...") - - # Create tmp directory if it does not exists - tmp_dir = os.path.join("/tmp/", self.name) - if not os.path.exists(tmp_dir): - os.mkdir(tmp_dir, 0o700) - - for f in self.files_to_keep: - dest_file = f.strip('/').replace("/", "_") - - # If the file is already there, we might be re-running the migration - # because it previously crashed. Hence we keep the existing file. - if os.path.exists(os.path.join(tmp_dir, dest_file)): - continue - - copy2(f, os.path.join(tmp_dir, dest_file)) - - def restore_files_to_keep(self): - - logger.debug("Restoring specific files to keep ...") - - tmp_dir = os.path.join("/tmp/", self.name) - - for f in self.files_to_keep: - dest_file = f.strip('/').replace("/", "_") - copy2(os.path.join(tmp_dir, dest_file), f) - - # On some setups, /etc/nginx/nginx.conf got edited. But this file needs - # to be upgraded because of the way the new module system works for nginx. - # (in particular, having the line that include the modules at the top) - # - # So here, if it got edited, we force the restore of the original conf - # *before* starting the actual upgrade... - # - # An alternative strategy that was attempted was to hold the nginx-common - # package and have a specific upgrade for it like for fail2ban, but that - # leads to apt complaining about not being able to upgrade for shitty - # reasons >.> - def restore_original_nginx_conf_if_needed(self): - if "/etc/nginx/nginx.conf" not in manually_modified_files_compared_to_debian_default(): - return - - if not os.path.exists("/etc/nginx/nginx.conf"): - return - - # If stretch is in the sources.list, we already started migrating on - # stretch so we don't re-do this - if " stretch " in read_file("/etc/apt/sources.list"): - return - - backup_dest = "/home/yunohost.conf/backup/nginx.conf.bkp_before_stretch" - - logger.warning(m18n.n("migration_0003_restoring_origin_nginx_conf", - backup_dest=backup_dest)) - - os.system("mv /etc/nginx/nginx.conf %s" % backup_dest) - - command = "" - command += " DEBIAN_FRONTEND=noninteractive" - command += " APT_LISTCHANGES_FRONTEND=none" - command += " apt-get" - command += " --fix-broken --show-upgraded --assume-yes" - command += ' -o Dpkg::Options::="--force-confmiss"' - command += " install --reinstall" - command += " nginx-common" - - logger.debug("Running apt command :\n{}".format(command)) - - command += " 2>&1 | tee -a {}".format(self.logfile) - - is_api = msettings.get('interface') == 'api' - if is_api: - callbacks = ( - lambda l: logger.info(l.rstrip()), - lambda l: logger.warning(l.rstrip()), - ) - call_async_output(command, callbacks, shell=True) - else: - # We do this when running from the cli to have the output of the - # command showing in the terminal, since 'info' channel is only - # enabled if the user explicitly add --verbose ... - os.system(command) - - def disable_predicable_interface_names(self): - - # Try to see if currently used interface names are predictable ones or not... - # If we ain't using "eth0" or "wlan0", assume we are using predictable interface - # names and therefore they shouldnt be disabled - network_interfaces = get_network_interfaces().keys() - if "eth0" not in network_interfaces and "wlan0" not in network_interfaces: - return - - interfaces_config = read_file("/etc/network/interfaces") - if "eth0" not in interfaces_config and "wlan0" not in interfaces_config: - return - - # Disable predictive interface names - # c.f. https://unix.stackexchange.com/a/338730 - os.system("ln -s /dev/null /etc/systemd/network/99-default.link") diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py deleted file mode 100644 index 1b90c4ff0..000000000 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ /dev/null @@ -1,99 +0,0 @@ -import os -import glob -from shutil import copy2 - -from moulinette.utils.log import getActionLogger - -from yunohost.tools import Migration -from yunohost.service import _run_service_command - -logger = getActionLogger('yunohost.migration') - -PHP5_POOLS = "/etc/php5/fpm/pool.d" -PHP7_POOLS = "/etc/php/7.0/fpm/pool.d" - -PHP5_SOCKETS_PREFIX = "/var/run/php5-fpm" -PHP7_SOCKETS_PREFIX = "/run/php/php7.0-fpm" - -MIGRATION_COMMENT = "; YunoHost note : this file was automatically moved from {}".format(PHP5_POOLS) - - -class MyMigration(Migration): - - "Migrate php5-fpm 'pool' conf files to php7 stuff" - - dependencies = ["migrate_to_stretch"] - - def run(self): - # Get list of php5 pool files - php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) - - # Keep only basenames - php5_pool_files = [os.path.basename(f) for f in php5_pool_files] - - # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) - php5_pool_files = [f for f in php5_pool_files if f != "www.conf"] - - for f in php5_pool_files: - - # Copy the files to the php7 pool - src = "{}/{}".format(PHP5_POOLS, f) - dest = "{}/{}".format(PHP7_POOLS, f) - copy2(src, dest) - - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest) - os.system(c) - - # Also add a comment that it was automatically moved from php5 - # (for human traceability and backward migration) - c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) - os.system(c) - - # Some old comments starting with '#' instead of ';' are not - # compatible in php7 - c = "sed -i 's/^#/;#/g' {}".format(dest) - os.system(c) - - # Reload/restart the php pools - _run_service_command("restart", "php7.0-fpm") - _run_service_command("enable", "php7.0-fpm") - os.system("systemctl stop php5-fpm") - os.system("systemctl disable php5-fpm") - os.system("rm /etc/logrotate.d/php5-fpm") # We remove this otherwise the logrotate cron will be unhappy - - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f) - os.system(c) - - # Reload nginx - _run_service_command("reload", "nginx") - - def backward(self): - - # Get list of php7 pool files - php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) - - # Keep only files which have the migration comment - php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT] - - # Delete those files - for f in php7_pool_files: - os.remove(f) - - # Reload/restart the php pools - _run_service_command("stop", "php7.0-fpm") - os.system("systemctl start php5-fpm") - - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f) - os.system(c) - - # Reload nginx - _run_service_command("reload", "nginx") diff --git a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py deleted file mode 100644 index 3127f2c65..000000000 --- a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py +++ /dev/null @@ -1,41 +0,0 @@ -import subprocess - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger - -from yunohost.tools import Migration -from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory - -logger = getActionLogger('yunohost.migration') - - -class MyMigration(Migration): - - "Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch" - - dependencies = ["migrate_to_stretch"] - - def run(self): - - if not self.package_is_installed("postgresql-9.4"): - logger.warning(m18n.n("migration_0005_postgresql_94_not_installed")) - return - - if not self.package_is_installed("postgresql-9.6"): - raise YunohostError("migration_0005_postgresql_96_not_installed") - - if not space_used_by_directory("/var/lib/postgresql/9.4") > free_space_in_directory("/var/lib/postgresql"): - raise YunohostError("migration_0005_not_enough_space", path="/var/lib/postgresql/") - - subprocess.check_call("service postgresql stop", shell=True) - subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True) - subprocess.check_call("pg_upgradecluster -m upgrade 9.4 main", shell=True) - subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True) - subprocess.check_call("service postgresql start", shell=True) - - def package_is_installed(self, package_name): - - p = subprocess.Popen("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), shell=True) - p.communicate() - return p.returncode == 0 diff --git a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py b/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py deleted file mode 100644 index 844e3028c..000000000 --- a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py +++ /dev/null @@ -1,78 +0,0 @@ -import spwd -import crypt -import random -import string -import subprocess - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import run_commands, check_output -from moulinette.utils.filesystem import append_to_file -from moulinette.authenticators.ldap import Authenticator -from yunohost.tools import Migration - -logger = getActionLogger('yunohost.migration') -SMALL_PWD_LIST = ["yunohost", "olinuxino", "olinux", "raspberry", "admin", "root", "test", "rpi"] - - -class MyMigration(Migration): - - "Synchronize admin and root passwords" - - def run(self): - - new_hash = self._get_admin_hash() - self._replace_root_hash(new_hash) - - logger.info(m18n.n("root_password_replaced_by_admin_password")) - - @property - def mode(self): - - # If the root password is still a "default" value, - # then this is an emergency and migration shall - # be applied automatically - # - # Otherwise, as playing with root password is touchy, - # we set this as a manual migration. - return "auto" if self._is_root_pwd_listed(SMALL_PWD_LIST) else "manual" - - @property - def disclaimer(self): - if self._is_root_pwd_listed(SMALL_PWD_LIST): - return None - - return m18n.n("migration_0006_disclaimer") - - def _get_admin_hash(self): - """ - Fetch the admin hash from the LDAP db using slapcat - """ - admin_hash = check_output("slapcat \ - | grep 'dn: cn=admin,dc=yunohost,dc=org' -A20 \ - | grep userPassword -A2 \ - | tr -d '\n ' \ - | tr ':' ' ' \ - | awk '{print $2}' \ - | base64 -d \ - | sed 's/{CRYPT}//g'") - return admin_hash - - def _replace_root_hash(self, new_hash): - hash_root = spwd.getspnam("root").sp_pwd - - with open('/etc/shadow', 'r') as before_file: - before = before_file.read() - - with open('/etc/shadow', 'w') as after_file: - after_file.write(before.replace("root:" + hash_root, - "root:" + new_hash)) - - def _is_root_pwd_listed(self, pwd_list): - hash_root = spwd.getspnam("root").sp_pwd - - for password in pwd_list: - if hash_root == crypt.crypt(password, hash_root): - return True - return False diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py deleted file mode 100644 index 624288210..000000000 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import re - -from shutil import copyfile - -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import mkdir, rm - -from yunohost.tools import Migration -from yunohost.service import _run_service_command -from yunohost.regenconf import regen_conf -from yunohost.settings import settings_set -from yunohost.utils.error import YunohostError - -logger = getActionLogger('yunohost.migration') - -SSHD_CONF = '/etc/ssh/sshd_config' - - -class MyMigration(Migration): - - """ - This is the first step of a couple of migrations that ensure SSH conf is - managed by YunoHost (even if the "from_script" flag is present, which was - previously preventing it from being managed by YunoHost) - - The goal of this first (automatic) migration is to make sure that the - sshd_config is managed by the regen-conf mechanism. - - If the from_script flag exists, then we keep the current SSH conf such that it - will appear as "manually modified" to the regenconf. - - In step 2 (manual), the admin will be able to choose wether or not to actually - use the recommended configuration, with an appropriate disclaimer. - """ - - def run(self): - - # Check if deprecated DSA Host Key is in config - dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$' - dsa = False - for line in open(SSHD_CONF): - if re.match(dsa_rgx, line) is not None: - dsa = True - break - if dsa: - settings_set("service.ssh.allow_deprecated_dsa_hostkey", True) - - # Here, we make it so that /etc/ssh/sshd_config is managed - # by the regen conf (in particular in the case where the - # from_script flag is present - in which case it was *not* - # managed by the regenconf) - # But because we can't be sure the user wants to use the - # recommended conf, we backup then restore the /etc/ssh/sshd_config - # right after the regenconf, such that it will appear as - # "manually modified". - if os.path.exists('/etc/yunohost/from_script'): - rm('/etc/yunohost/from_script') - copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') - regen_conf(names=['ssh'], force=True) - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - - # Restart ssh and rollback if it failed - if not _run_service_command('restart', 'ssh'): - # We don't rollback completely but it should be enough - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - if not _run_service_command('restart', 'ssh'): - raise YunohostError("migration_0007_cannot_restart") - else: - raise YunohostError("migration_0007_cancelled") diff --git a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py deleted file mode 100644 index ecc8cfdcb..000000000 --- a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py +++ /dev/null @@ -1,105 +0,0 @@ -import os -import re - -from moulinette import m18n -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import chown - -from yunohost.tools import Migration -from yunohost.regenconf import _get_conf_hashes, _calculate_hash -from yunohost.regenconf import regen_conf -from yunohost.settings import settings_set, settings_get -from yunohost.utils.error import YunohostError -from yunohost.backup import ARCHIVES_PATH - - -logger = getActionLogger('yunohost.migration') - -SSHD_CONF = '/etc/ssh/sshd_config' - - -class MyMigration(Migration): - - """ - In this second step, the admin is asked if it's okay to use - the recommended SSH configuration - which also implies - disabling deprecated DSA key. - - This has important implications in the way the user may connect - to its server (key change, and a spooky warning might be given - by SSH later) - - A disclaimer explaining the various things to be aware of is - shown - and the user may also choose to skip this migration. - """ - - dependencies = ["ssh_conf_managed_by_yunohost_step1"] - - def run(self): - settings_set("service.ssh.allow_deprecated_dsa_hostkey", False) - regen_conf(names=['ssh'], force=True) - - # Update local archives folder permissions, so that - # admin can scp archives out of the server - if os.path.isdir(ARCHIVES_PATH): - chown(ARCHIVES_PATH, uid="admin", gid="root") - - @property - def mode(self): - - # If the conf is already up to date - # and no DSA key is used, then we're good to go - # and the migration can be done automatically - # (basically nothing shall change) - ynh_hash = _get_conf_hashes('ssh').get(SSHD_CONF, None) - current_hash = _calculate_hash(SSHD_CONF) - dsa = settings_get("service.ssh.allow_deprecated_dsa_hostkey") - if ynh_hash == current_hash and not dsa: - return "auto" - - return "manual" - - @property - def disclaimer(self): - - if self.mode == "auto": - return None - - # Detect key things to be aware of before enabling the - # recommended configuration - dsa_key_enabled = False - ports = [] - root_login = [] - port_rgx = r'^[ \t]*Port[ \t]+(\d+)[ \t]*(?:#.*)?$' - root_rgx = r'^[ \t]*PermitRootLogin[ \t]([^# \t]*)[ \t]*(?:#.*)?$' - dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$' - for line in open(SSHD_CONF): - - ports = ports + re.findall(port_rgx, line) - - root_login = root_login + re.findall(root_rgx, line) - - if not dsa_key_enabled and re.match(dsa_rgx, line) is not None: - dsa_key_enabled = True - - custom_port = ports != ['22'] and ports != [] - root_login_enabled = root_login and root_login[-1] != 'no' - - # Build message - message = m18n.n("migration_0008_general_disclaimer") - - if custom_port: - message += "\n\n" + m18n.n("migration_0008_port") - - if root_login_enabled: - message += "\n\n" + m18n.n("migration_0008_root") - - if dsa_key_enabled: - message += "\n\n" + m18n.n("migration_0008_dsa") - - if custom_port or root_login_enabled or dsa_key_enabled: - message += "\n\n" + m18n.n("migration_0008_warning") - else: - message += "\n\n" + m18n.n("migration_0008_no_warning") - - return message diff --git a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py b/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py deleted file mode 100644 index c190e2aaa..000000000 --- a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - -from moulinette import m18n -from moulinette.utils.log import getActionLogger - -from moulinette.utils.filesystem import read_file -from yunohost.service import _get_services, _save_services -from yunohost.regenconf import _update_conf_hashes, REGEN_CONF_FILE - -from yunohost.tools import Migration - -logger = getActionLogger('yunohost.migration') - - -class MyMigration(Migration): - """ - Decouple the regen conf mechanism from the concept of services - """ - - def run(self): - - if "conffiles" not in read_file("/etc/yunohost/services.yml") \ - or os.path.exists(REGEN_CONF_FILE): - logger.warning(m18n.n("migration_0009_not_needed")) - return - - # For all services - services = _get_services() - for service, infos in services.items(): - # If there are some conffiles (file hashes) - if "conffiles" in infos.keys(): - # Save them using the new regen conf thingy - _update_conf_hashes(service, infos["conffiles"]) - # And delete the old conffile key from the service infos - del services[service]["conffiles"] - - # (Actually save the modification of services) - _save_services(services) - diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py deleted file mode 100644 index e5ce65608..000000000 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ /dev/null @@ -1,13 +0,0 @@ -from moulinette.utils.log import getActionLogger -from yunohost.tools import Migration - -logger = getActionLogger('yunohost.migration') - - -class MyMigration(Migration): - - "Migrate from official.json to apps.json (outdated, replaced by migration 13)" - - def run(self): - logger.info("This migration is oudated and doesn't do anything anymore. The migration 13 will handle this instead.") - pass diff --git a/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py b/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py deleted file mode 100644 index 636b4f12b..000000000 --- a/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py +++ /dev/null @@ -1,16 +0,0 @@ -import glob -import re -from yunohost.tools import Migration -from moulinette.utils.filesystem import read_file, write_to_file - - -class MyMigration(Migration): - - "Force authentication in md5 for local connexions" - - all_hba_files = glob.glob("/etc/postgresql/*/*/pg_hba.conf") - - def run(self): - for filename in self.all_hba_files: - pg_hba_in = read_file(filename) - write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)password", "local\\1all\\2all\\3md5", pg_hba_in)) diff --git a/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py b/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py deleted file mode 100644 index ff4925183..000000000 --- a/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py +++ /dev/null @@ -1,51 +0,0 @@ - -import os -import shutil - -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json - -from yunohost.tools import Migration -from yunohost.app import (_initialize_apps_catalog_system, - _update_apps_catalog, - APPS_CATALOG_CACHE, - APPS_CATALOG_CONF) - -logger = getActionLogger('yunohost.migration') - -LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json' -LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old" - - -class MyMigration(Migration): - - "Migrate to the new future-proof apps catalog system" - - def run(self): - - if not os.path.exists(LEGACY_APPS_CATALOG_CONF): - logger.info("No need to do anything") - - # Destroy old lecacy cache - if os.path.exists(APPS_CATALOG_CACHE): - shutil.rmtree(APPS_CATALOG_CACHE) - - # and legacy cron - if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"): - os.remove("/etc/cron.daily/yunohost-fetch-appslists") - - # Backup the legacy file - try: - legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF) - # If there's only one catalog, we assume it's just the old official catalog - # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually - if len(legacy_catalogs) > 1: - logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)) - except Exception as e: - logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e))) - - if os.path.exists(LEGACY_APPS_CATALOG_CONF): - os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP) - - _initialize_apps_catalog_system() - _update_apps_catalog() diff --git a/src/yunohost/data_migrations/0014_remove_app_status_json.py b/src/yunohost/data_migrations/0014_remove_app_status_json.py deleted file mode 100644 index 1cb5bc002..000000000 --- a/src/yunohost/data_migrations/0014_remove_app_status_json.py +++ /dev/null @@ -1,31 +0,0 @@ -import os - -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json - -from yunohost.tools import Migration -from yunohost.app import app_setting, APPS_SETTING_PATH - -logger = getActionLogger('yunohost.migration') - -class MyMigration(Migration): - - """Remove legacy app status.json files""" - - def run(self): - - apps = os.listdir(APPS_SETTING_PATH) - - for app in apps: - status_file = os.path.join(APPS_SETTING_PATH, app, "status.json") - if not os.path.exists(status_file): - continue - - try: - status = read_json(status_file) - current_revision = status.get("remote", {}).get("revision", "?") - app_setting(app, 'current_revision', current_revision) - except Exception as e: - logger.warning("Could not migrate status.json from app %s: %s", (app, str(e))) - else: - os.system("rm %s" % status_file) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index efa25f23f..7dcc33cbf 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -205,22 +205,6 @@ def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, key = keys[0] - # This mean that hmac-md5 is used - # (Re?)Trigger the migration to sha256 and return immediately. - # The actual update will be done in next run. - if "+157" in key: - from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("migrate_to_tsig_sha256") - try: - migration.run(dyn_host, domain, key) - except Exception as e: - logger.error(m18n.n('migrations_migration_has_failed', - exception=e, - number=migration.number, - name=migration.name), - exc_info=1) - return - # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me' host = domain.split('.')[1:] host = '.'.join(host) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index a94f5023d..6c9b10aac 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -62,16 +62,6 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run """ - # Legacy code to automatically run the migration - # This is required because regen_conf is called before the migration call - # in debian's postinst script - if os.path.exists("/etc/yunohost/installed") \ - and ("conffiles" in read_file("/etc/yunohost/services.yml") \ - or not os.path.exists(REGEN_CONF_FILE)): - from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("decouple_regenconf_from_services") - migration.run() - result = {} # Return the list of pending conf diff --git a/src/yunohost/tests/test_appscatalog.py b/src/yunohost/tests/test_appscatalog.py index c3ece7907..40cf1489f 100644 --- a/src/yunohost/tests/test_appscatalog.py +++ b/src/yunohost/tests/test_appscatalog.py @@ -315,58 +315,3 @@ def test_apps_catalog_load_with_oudated_api_version(mocker): for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"): cache_json = read_json(cache_file) assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION - - - -def test_apps_catalog_migrate_legacy_explicitly(): - - open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') - mkdir(APPS_CATALOG_CACHE, 0o750, parents=True) - open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}') - open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron") - - from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("futureproof_apps_catalog_system") - - with requests_mock.Mocker() as m: - - # Mock the server response with a dummy apps catalog - m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - migration.run() - - # Old conf shouldnt be there anymore (got renamed to .old) - assert not os.path.exists("/etc/yunohost/appslists.json") - # Old cache should have been removed - assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json") - # Cron should have been changed - assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read() - assert cron_job_is_there() - - # Reading the apps_catalog should work - app_dict = _load_apps_catalog()["apps"] - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - -def test_apps_catalog_migrate_legacy_implicitly(): - - open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') - mkdir(APPS_CATALOG_CACHE, 0o750, parents=True) - open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}') - open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron") - - with requests_mock.Mocker() as m: - m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - app_dict = _load_apps_catalog()["apps"] - - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - # Old conf shouldnt be there anymore (got renamed to .old) - assert not os.path.exists("/etc/yunohost/appslists.json") - # Old cache should have been removed - assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json") - # Cron should have been changed - assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read() - assert cron_job_is_there() - From 06c5f23c2b9d2d044ad6ca1f052b405890c19486 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 5 Mar 2020 22:56:15 +0100 Subject: [PATCH 071/138] [cleanup] These arent used anywhere --- data/templates/unattended/02periodic | 5 --- .../unattended/50unattended-upgrades | 36 ------------------- 2 files changed, 41 deletions(-) delete mode 100644 data/templates/unattended/02periodic delete mode 100644 data/templates/unattended/50unattended-upgrades diff --git a/data/templates/unattended/02periodic b/data/templates/unattended/02periodic deleted file mode 100644 index f16105466..000000000 --- a/data/templates/unattended/02periodic +++ /dev/null @@ -1,5 +0,0 @@ -# https://wiki.debian.org/UnattendedUpgrades#automatic_call_via_.2Fetc.2Fapt.2Fapt.conf.d.2F02periodic -APT::Periodic::Enable "1"; -APT::Periodic::Update-Package-Lists "1"; -APT::Periodic::Unattended-Upgrade "1"; -APT::Periodic::Verbose "1"; diff --git a/data/templates/unattended/50unattended-upgrades b/data/templates/unattended/50unattended-upgrades deleted file mode 100644 index 49b600a3b..000000000 --- a/data/templates/unattended/50unattended-upgrades +++ /dev/null @@ -1,36 +0,0 @@ -// Automatically upgrade packages from these (origin, archive) pairs -Unattended-Upgrade::Allowed-Origins { - "${distro_id} stable"; - "${distro_id} testing"; - "Depot-Debian testing"; - "${distro_id} ${distro_codename}-security"; - "${distro_id} ${distro_codename}-updates"; -// "${distro_id} ${distro_codename}-proposed-updates"; -}; - -// List of packages to not update -Unattended-Upgrade::Package-Blacklist { -// "vim"; -// "libc6"; -// "libc6-dev"; -// "libc6-i686"; -}; - -// Send email to this address for problems or packages upgrades -// If empty or unset then no email is sent, make sure that you -// have a working mail setup on your system. The package 'mailx' -// must be installed or anything that provides /usr/bin/mail. -//Unattended-Upgrade::Mail "root@localhost"; - -// Do automatic removal of new unused dependencies after the upgrade -// (equivalent to apt-get autoremove) -Unattended-Upgrade::Remove-Unused-Dependencies "true"; - -// Automatically reboot *WITHOUT CONFIRMATION* if a -// the file /var/run/reboot-required is found after the upgrade -Unattended-Upgrade::Automatic-Reboot "false"; - - -// Use apt bandwidth limit feature, this example limits the download -// speed to 70kb/sec -//Acquire::http::Dl-Limit "70"; From bde5dab71d218d8f85c3ce0ca1a205272447eb91 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 1 Apr 2020 02:34:20 +0200 Subject: [PATCH 072/138] [cleanup] Fix typo in string name, remove stale string --- locales/en.json | 1 - 1 file changed, 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index c0f79436f..a851543dc 100644 --- a/locales/en.json +++ b/locales/en.json @@ -452,7 +452,6 @@ "migrations_skip_migration": "Skipping migration {id}...", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", - "no_internet_connection": "The server is not connected to the Internet", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", "packages_upgrade_failed": "Could not upgrade all the packages", From fa5130a7fc3a37a7faef58ca2350c19257c53539 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 May 2020 19:50:48 +0200 Subject: [PATCH 073/138] [cleanup] Move migration 0011 code to legacy.py --- locales/en.json | 10 +- src/yunohost/backup.py | 10 +- .../legacy.py} | 91 +++---------------- 3 files changed, 19 insertions(+), 92 deletions(-) rename src/yunohost/{data_migrations/0011_setup_group_permission.py => utils/legacy.py} (52%) diff --git a/locales/en.json b/locales/en.json index a851543dc..3f17b576a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -406,17 +406,13 @@ "mail_unavailable": "This e-mail address is reserved and shall be automatically allocated to the very first user", "main_domain_change_failed": "Unable to change the main domain", "main_domain_changed": "The main domain has been changed", - "migration_description_0011_setup_group_permission": "Set up user group and set up permission for apps and services", "migration_description_0015_migrate_to_buster": "Upgrade the system to Debian Buster and YunoHost 4.x", - "migration_0011_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", - "migration_0011_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", + "migration_description_0016_php70_to_php73_pools": "Migrate php7.0-fpm 'pool' conf files to php7.3", + "migration_description_0017_postgresql_9p6_to_11": "Migrate databases from PostgreSQL 9.6 to 11", + "migration_description_0018_xtable_to_nftable": "Migrate old network traffic rules to the new nftable system", "migration_0011_create_group": "Creating a group for each user…", - "migration_0011_done": "Migration completed. You are now able to manage usergroups.", - "migration_0011_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.", "migration_0011_LDAP_update_failed": "Could not update LDAP. Error: {error:s}", "migration_0011_migrate_permission": "Migrating permissions from apps settings to LDAP...", - "migration_0011_migration_failed_trying_to_rollback": "Could not migrate… trying to roll back the system.", - "migration_0011_rollback_success": "System rolled back.", "migration_0011_update_LDAP_database": "Updating LDAP database...", "migration_0011_update_LDAP_schema": "Updating LDAP schema...", "migration_0011_failed_to_remove_stale_object": "Could not remove stale object {dn}: {error}", diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 449b52bd8..8a6ce4e7f 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1244,12 +1244,11 @@ class RestoreManager(): # # Legacy code if not "all_users" in user_group_list()["groups"].keys(): - from yunohost.tools import _get_migration_by_name - setup_group_permission = _get_migration_by_name("setup_group_permission") + from yunohost.utils.legacy import SetupGroupPermissions # Update LDAP schema restart slapd logger.info(m18n.n("migration_0011_update_LDAP_schema")) regen_conf(names=['slapd'], force=True) - setup_group_permission.migrate_LDAP_db() + SetupGroupPermissions.migrate_LDAP_db() # Remove all permission for all app which is still in the LDAP for permission_name in user_permission_list(ignore_system_perms=True)["permissions"].keys(): @@ -1389,9 +1388,8 @@ class RestoreManager(): else: # Otherwise, we need to migrate the legacy permissions of this # app (included in its settings.yml) - from yunohost.tools import _get_migration_by_name - setup_group_permission = _get_migration_by_name("setup_group_permission") - setup_group_permission.migrate_app_permission(app=app_instance_name) + from yunohost.utils.legacy import SetupGroupPermissions + SetupGroupPermissions.migrate_app_permission(app=app_instance_name) # Prepare env. var. to pass to script env_dict = self._get_env_var(app_instance_name) diff --git a/src/yunohost/data_migrations/0011_setup_group_permission.py b/src/yunohost/utils/legacy.py similarity index 52% rename from src/yunohost/data_migrations/0011_setup_group_permission.py rename to src/yunohost/utils/legacy.py index c55e33cab..b7052b438 100644 --- a/src/yunohost/data_migrations/0011_setup_group_permission.py +++ b/src/yunohost/utils/legacy.py @@ -1,34 +1,19 @@ -import time -import os - from moulinette import m18n from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_yaml -from yunohost.tools import Migration from yunohost.user import user_list, user_group_create, user_group_update from yunohost.app import app_setting, _installed_apps -from yunohost.regenconf import regen_conf, BACKUP_CONF_DIR from yunohost.permission import permission_create, user_permission_update, permission_sync_to_user -logger = getActionLogger('yunohost.migration') - -################################################### -# Tools used also for restoration -################################################### +logger = getActionLogger('yunohost.legacy') -class MyMigration(Migration): - """ - Update the LDAP DB to be able to store the permission - Create a group for each yunohost user - Migrate app permission from apps setting to LDAP - """ +class SetupGroupPermissions(): - required = True - - def remove_if_exists(self, target): + @staticmethod + def remove_if_exists(target): from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() @@ -50,7 +35,8 @@ class MyMigration(Migration): except Exception as e: raise YunohostError("migration_0011_failed_to_remove_stale_object", dn=dn, error=e) - def migrate_LDAP_db(self): + @staticmethod + def migrate_LDAP_db(): logger.info(m18n.n("migration_0011_update_LDAP_database")) @@ -60,8 +46,8 @@ class MyMigration(Migration): ldap_map = read_yaml('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') try: - self.remove_if_exists("ou=permission") - self.remove_if_exists('ou=groups') + SetupGroupPermissions.remove_if_exists("ou=permission") + SetupGroupPermissions.remove_if_exists('ou=groups') attr_dict = ldap_map['parents']['ou=permission'] ldap.add('ou=permission', attr_dict) @@ -93,7 +79,8 @@ class MyMigration(Migration): user_group_create(username, gid=user_info['uidNumber'][0], primary_group=True, sync_perm=False) user_group_update(groupname='all_users', add=username, force=True, sync_perm=False) - def migrate_app_permission(self, app=None): + @staticmethod + def migrate_app_permission(app=None): logger.info(m18n.n("migration_0011_migrate_permission")) apps = _installed_apps() @@ -116,66 +103,12 @@ class MyMigration(Migration): allowed = [user for user in permission.split(',') if user in known_users] else: allowed = ["all_users"] - permission_create(app+".main", url=url, allowed=allowed, sync_perm=False) + permission_create(app + ".main", url=url, allowed=allowed, sync_perm=False) app_setting(app, 'allowed_users', delete=True) # Migrate classic public app still using the legacy unprotected_uris if app_setting(app, "unprotected_uris") == "/" or app_setting(app, "skipped_uris") == "/": - user_permission_update(app+".main", add="visitors", sync_perm=False) + user_permission_update(app + ".main", add="visitors", sync_perm=False) permission_sync_to_user() - - def run(self): - - # FIXME : what do we really want to do here ... - # Imho we should just force-regen the conf in all case, and maybe - # just display a warning if we detect that the conf was manually modified - - # Check if the migration can be processed - ldap_regen_conf_status = regen_conf(names=['slapd'], dry_run=True) - # By this we check if the have been customized - if ldap_regen_conf_status and ldap_regen_conf_status['slapd']['pending']: - logger.warning(m18n.n("migration_0011_slapd_config_will_be_overwritten", conf_backup_folder=BACKUP_CONF_DIR)) - - # Backup LDAP and the apps settings before to do the migration - logger.info(m18n.n("migration_0011_backup_before_migration")) - try: - backup_folder = "/home/yunohost.backup/premigration/" + time.strftime('%Y%m%d-%H%M%S', time.gmtime()) - os.makedirs(backup_folder, 0o750) - os.system("systemctl stop slapd") - os.system("cp -r --preserve /etc/ldap %s/ldap_config" % backup_folder) - os.system("cp -r --preserve /var/lib/ldap %s/ldap_db" % backup_folder) - os.system("cp -r --preserve /etc/yunohost/apps %s/apps_settings" % backup_folder) - except Exception as e: - raise YunohostError("migration_0011_can_not_backup_before_migration", error=e) - finally: - os.system("systemctl start slapd") - - try: - # Update LDAP schema restart slapd - logger.info(m18n.n("migration_0011_update_LDAP_schema")) - regen_conf(names=['slapd'], force=True) - - # Update LDAP database - self.migrate_LDAP_db() - - # Migrate permission - self.migrate_app_permission() - - permission_sync_to_user() - except Exception as e: - logger.warn(m18n.n("migration_0011_migration_failed_trying_to_rollback")) - os.system("systemctl stop slapd") - os.system("rm -r /etc/ldap/slapd.d") # To be sure that we don't keep some part of the old config - os.system("cp -r --preserve %s/ldap_config/. /etc/ldap/" % backup_folder) - os.system("cp -r --preserve %s/ldap_db/. /var/lib/ldap/" % backup_folder) - os.system("cp -r --preserve %s/apps_settings/. /etc/yunohost/apps/" % backup_folder) - os.system("systemctl start slapd") - os.system("rm -r " + backup_folder) - logger.info(m18n.n("migration_0011_rollback_success")) - raise - else: - os.system("rm -r " + backup_folder) - - logger.info(m18n.n("migration_0011_done")) From fa59ad35a9c2b983f7b7a1e7dc89d350ef142fe6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 8 Feb 2020 18:27:44 +0100 Subject: [PATCH 074/138] [conf] Automatically disable/stop systemd-resolved that conflicts with dnsmasq on fresh setups ... --- data/hooks/conf_regen/43-dnsmasq | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/data/hooks/conf_regen/43-dnsmasq b/data/hooks/conf_regen/43-dnsmasq index 8cddec1be..75d74b09c 100755 --- a/data/hooks/conf_regen/43-dnsmasq +++ b/data/hooks/conf_regen/43-dnsmasq @@ -69,8 +69,16 @@ do_post_regen() { short_hostname=$(hostname -s) grep -q "127.0.0.1.*$short_hostname" /etc/hosts || echo -e "\n127.0.0.1\t$short_hostname" >>/etc/hosts - [[ -z "$regen_conf_files" ]] \ - || service dnsmasq restart + [[ -n "$regen_conf_files" ]] || return + + # Remove / disable services likely to conflict with dnsmasq + for SERVICE in systemd-resolved bind9 + do + systemctl is-enabled $SERVICE &>/dev/null && systemctl disable $SERVICE 2>/dev/null + systemctl is-active $SERVICE &>/dev/null && systemctl stop $SERVICE + done + + systemctl restart dnsmasq } FORCE=${2:-0} From 16ce8bf693c9aa4ecd011e597e23f5a4a44b5b3d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Feb 2020 00:48:19 +0100 Subject: [PATCH 075/138] [conf] Remove deprecated option in sshd conf, c.f. https://patchwork.openembedded.org/patch/139981/ --- data/templates/ssh/sshd_config | 3 --- 1 file changed, 3 deletions(-) diff --git a/data/templates/ssh/sshd_config b/data/templates/ssh/sshd_config index 8dc0e8dfc..bd3efdef3 100644 --- a/data/templates/ssh/sshd_config +++ b/data/templates/ssh/sshd_config @@ -27,9 +27,6 @@ HostKey {{ key }}{% endfor %} MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com {% endif %} -# Use kernel sandbox mechanisms where possible in unprivileged processes -UsePrivilegeSeparation sandbox - # LogLevel VERBOSE logs user's key fingerprint on login. # Needed to have a clear audit track of which key was using to log in. SyslogFacility AUTH From 01ccd5dd8cab28794bc1ee41fc5ea1e4f18ea816 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 2 May 2020 03:27:07 +0200 Subject: [PATCH 076/138] [conf] Small tweak in dovecot conf (deprecated settings) --- data/templates/dovecot/dovecot.conf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/data/templates/dovecot/dovecot.conf b/data/templates/dovecot/dovecot.conf index 8fc0e75ae..d64b15356 100644 --- a/data/templates/dovecot/dovecot.conf +++ b/data/templates/dovecot/dovecot.conf @@ -14,18 +14,16 @@ mail_plugins = $mail_plugins quota ############################################################################### -# generated 2020-04-03, Mozilla Guideline v5.4, Dovecot 2.2.27, OpenSSL 1.1.0l, intermediate configuration -# https://ssl-config.mozilla.org/#server=dovecot&version=2.2.27&config=intermediate&openssl=1.1.0l&guideline=5.4 +# generated 2020-05-02, Mozilla Guideline v5.4, Dovecot 2.3.4.1, OpenSSL 1.1.1d, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4.1&config=intermediate&openssl=1.1.1d&guideline=5.4 ssl = required ssl_cert = Date: Sat, 7 Mar 2020 20:16:03 +0100 Subject: [PATCH 077/138] [conf] Update nslcd and nsswitch stuff using new Buster's default configs + get rid of nslcd service, only keep the regen-conf part --- data/templates/nslcd/nslcd.conf | 12 ++++++++++++ data/templates/nsswitch/nsswitch.conf | 10 +++------- data/templates/yunohost/services.yml | 9 +++++---- locales/en.json | 3 +-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/data/templates/nslcd/nslcd.conf b/data/templates/nslcd/nslcd.conf index 091ecb7cc..7cfe73e07 100644 --- a/data/templates/nslcd/nslcd.conf +++ b/data/templates/nslcd/nslcd.conf @@ -15,6 +15,18 @@ base dc=yunohost,dc=org # The LDAP protocol version to use. #ldap_version 3 +# The DN to bind with for normal lookups. +#binddn cn=annonymous,dc=example,dc=net +#bindpw secret + +# The DN used for password modifications by root. +#rootpwmoddn cn=admin,dc=example,dc=com + +# SSL options +#ssl off +#tls_reqcert never +tls_cacertfile /etc/ssl/certs/ca-certificates.crt + # The search scope. #scope sub diff --git a/data/templates/nsswitch/nsswitch.conf b/data/templates/nsswitch/nsswitch.conf index b55e01b02..8f46e4f5d 100644 --- a/data/templates/nsswitch/nsswitch.conf +++ b/data/templates/nsswitch/nsswitch.conf @@ -1,12 +1,8 @@ # /etc/nsswitch.conf -# -# Example configuration of GNU Name Service Switch functionality. -# If you have the `glibc-doc-reference' and `info' packages installed, try: -# `info libc "Name Service Switch"' for information about this file. -passwd: compat ldap -group: compat ldap -shadow: compat ldap +passwd: files systemd ldap +group: files systemd ldap +shadow: files ldap gshadow: files hosts: files myhostname mdns4_minimal [NOTFOUND=return] dns diff --git a/data/templates/yunohost/services.yml b/data/templates/yunohost/services.yml index e1dd57e55..73ae9403e 100644 --- a/data/templates/yunohost/services.yml +++ b/data/templates/yunohost/services.yml @@ -20,10 +20,9 @@ nginx: test_conf: nginx -t needs_exposed_ports: [80, 443] category: web -nslcd: {} -php7.0-fpm: - log: /var/log/php7.0-fpm.log - test_conf: php-fpm7.0 --test +php7.3-fpm: + log: /var/log/php7.3-fpm.log + test_conf: php-fpm7.3 --test category: web postfix: log: [/var/log/mail.log,/var/log/mail.err] @@ -64,3 +63,5 @@ postgrey: null spamassassin: null rmilter: null php5-fpm: null +php7.0-fpm: null +nslcd: null diff --git a/locales/en.json b/locales/en.json index 3f17b576a..f3daed3ca 100644 --- a/locales/en.json +++ b/locales/en.json @@ -534,8 +534,7 @@ "service_description_metronome": "Manage XMPP instant messaging accounts", "service_description_mysql": "Stores app data (SQL database)", "service_description_nginx": "Serves or provides access to all the websites hosted on your server", - "service_description_nslcd": "Handles YunoHost user shell connection", - "service_description_php7.0-fpm": "Runs apps written in PHP with NGINX", + "service_description_php7.3-fpm": "Runs apps written in PHP with NGINX", "service_description_postfix": "Used to send and receive e-mails", "service_description_redis-server": "A specialized database used for rapid data access, task queue, and communication between programs", "service_description_rspamd": "Filters spam, and other e-mail related features", From 5930b6ddf2619f895c9b2e382f28890b4775a295 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 8 Feb 2020 20:11:31 +0100 Subject: [PATCH 078/138] [php] Now use php7.3 instead of php7.0 + autopatch app scripts like we did for php5 --- data/helpers.d/php | 2 +- locales/en.json | 1 - src/yunohost/app.py | 31 ++++++++++++++++++++----------- src/yunohost/backup.py | 39 +++++++++++++++++---------------------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/data/helpers.d/php b/data/helpers.d/php index 4902e3292..7ff671317 100644 --- a/data/helpers.d/php +++ b/data/helpers.d/php @@ -1,6 +1,6 @@ #!/bin/bash -readonly YNH_DEFAULT_PHP_VERSION=7.0 +readonly YNH_DEFAULT_PHP_VERSION=7.3 # Declare the actual php version to use. # A packager willing to use another version of php can override the variable into its _common.sh. YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} diff --git a/locales/en.json b/locales/en.json index f3daed3ca..da6c759b9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -106,7 +106,6 @@ "backup_output_directory_required": "You must provide an output directory for the backup", "backup_output_symlink_dir_broken": "Your archive directory '{path:s}' is a broken symlink. Maybe you forgot to re/mount or plug in the storage medium it points to.", "backup_permission": "Backup permission for {app:s}", - "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support PHP 7, you may be unable to restore your PHP apps (reason: {error:s})", "backup_running_hooks": "Running backup hooks...", "backup_system_part_failed": "Could not backup the '{part:s}' system part", "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 032536247..56574acf6 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -521,7 +521,7 @@ def app_upgrade(app=[], url=None, file=None): _patch_legacy_helpers(extracted_app_folder) # Apply dirty patch to make php5 apps compatible with php7 - _patch_php5(extracted_app_folder) + _patch_legacy_php_versions(extracted_app_folder) # Start register change on system related_to = [('app', app_instance_name)] @@ -736,7 +736,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu _patch_legacy_helpers(extracted_app_folder) # Apply dirty patch to make php5 apps compatible with php7 - _patch_php5(extracted_app_folder) + _patch_legacy_php_versions(extracted_app_folder) # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) @@ -1033,7 +1033,7 @@ def app_remove(operation_logger, app): # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove # script might date back from jessie install) - _patch_php5(app_setting_path) + _patch_legacy_php_versions(app_setting_path) manifest = _get_manifest_of_app(app_setting_path) @@ -2839,8 +2839,8 @@ def _assert_system_is_sane_for_app(manifest, when): # Some apps use php-fpm or php5-fpm which is now php7.0-fpm def replace_alias(service): - if service in ["php-fpm", "php5-fpm"]: - return "php7.0-fpm" + if service in ["php-fpm", "php5-fpm", "php7.0-fpm"]: + return "php7.3-fpm" else: return service services = [replace_alias(s) for s in services] @@ -2848,7 +2848,7 @@ def _assert_system_is_sane_for_app(manifest, when): # We only check those, mostly to ignore "custom" services # (added by apps) and because those are the most popular # services - service_filter = ["nginx", "php7.0-fpm", "mysql", "postfix"] + service_filter = ["nginx", "php7.3-fpm", "mysql", "postfix"] services = [str(s) for s in services if s in service_filter] if "nginx" not in services: @@ -2873,7 +2873,16 @@ def _assert_system_is_sane_for_app(manifest, when): raise YunohostError("this_action_broke_dpkg") -def _patch_php5(app_folder): +LEGACY_PHP_VERSION_REPLACEMENTS = [ + ("/etc/php5", "/etc/php/7.3"), + ("/etc/php/7.0", "/etc/php/7.3"), + ("/var/run/php5-fpm", "/var/run/php/php7.3-fpm"), + ("/var/run/php/php7.0-fpm", "/var/run/php/php7.3-fpm"), + ("php5", "php7.3"), + ("php7.0", "php7.3") +] + +def _patch_legacy_php_versions(app_folder): files_to_patch = [] files_to_patch.extend(glob.glob("%s/conf/*" % app_folder)) @@ -2888,12 +2897,12 @@ def _patch_php5(app_folder): if not os.path.isfile(filename): continue - c = "sed -i -e 's@/etc/php5@/etc/php/7.0@g' " \ - "-e 's@/var/run/php5-fpm@/var/run/php/php7.0-fpm@g' " \ - "-e 's@php5@php7.0@g' " \ - "%s" % filename + c = "sed -i " \ + + "".join("-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r) for p, r in LEGACY_PHP_VERSION_REPLACEMENTS) \ + + "%s" % filename os.system(c) + def _patch_legacy_helpers(app_folder): files_to_patch = [] diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 8a6ce4e7f..8ba8f2610 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -43,7 +43,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml from yunohost.app import ( - app_info, _is_installed, _parse_app_instance_name, _patch_php5, dump_app_log_extract_for_debugging, _patch_legacy_helpers + app_info, _is_installed, _parse_app_instance_name, _patch_legacy_php_versions, dump_app_log_extract_for_debugging, _patch_legacy_helpers, LEGACY_PHP_VERSION_REPLACEMENTS ) from yunohost.hook import ( hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER @@ -1141,7 +1141,7 @@ class RestoreManager(): self._postinstall_if_needed() # Apply dirty patch to redirect php5 file on php7 - self._patch_backup_csv_file() + self._patch_legacy_php_versions_in_csv_file() self._restore_system() self._restore_apps() @@ -1150,9 +1150,9 @@ class RestoreManager(): finally: self.clean() - def _patch_backup_csv_file(self): + def _patch_legacy_php_versions_in_csv_file(self): """ - Apply dirty patch to redirect php5 file on php7 + Apply dirty patch to redirect php5 and php7.0 files to php7.3 """ backup_csv = os.path.join(self.work_dir, 'backup.csv') @@ -1160,32 +1160,27 @@ class RestoreManager(): if not os.path.isfile(backup_csv): return - contains_php5 = False + replaced_something = False with open(backup_csv) as csvfile: reader = csv.DictReader(csvfile, fieldnames=['source', 'dest']) newlines = [] for row in reader: - if 'php5' in row['source']: - contains_php5 = True - row['source'] = row['source'].replace('/etc/php5', '/etc/php/7.0') \ - .replace('/var/run/php5-fpm', '/var/run/php/php7.0-fpm') \ - .replace('php5', 'php7') + for pattern, replace in LEGACY_PHP_VERSION_REPLACEMENTS: + if pattern in row['source']: + replaced_something = True + row['source'] = row['source'].replace(pattern, replace) newlines.append(row) - if not contains_php5: + if not replaced_something: return - try: - with open(backup_csv, 'w') as csvfile: - writer = csv.DictWriter(csvfile, - fieldnames=['source', 'dest'], - quoting=csv.QUOTE_ALL) - for row in newlines: - writer.writerow(row) - except (IOError, OSError, csv.Error) as e: - logger.warning(m18n.n('backup_php5_to_php7_migration_may_fail', - error=str(e))) + with open(backup_csv, 'w') as csvfile: + writer = csv.DictWriter(csvfile, + fieldnames=['source', 'dest'], + quoting=csv.QUOTE_ALL) + for row in newlines: + writer.writerow(row) def _restore_system(self): """ Restore user and system parts """ @@ -1333,7 +1328,7 @@ class RestoreManager(): _patch_legacy_helpers(app_settings_in_archive) # Apply dirty patch to make php5 apps compatible with php7 - _patch_php5(app_settings_in_archive) + _patch_legacy_php_versions(app_settings_in_archive) # Delete _common.sh file in backup common_file = os.path.join(app_backup_in_archive, '_common.sh') From 2c7b3a1fd1e9f0d28231b0f6675260f124fac7d9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 8 Mar 2020 22:43:52 +0100 Subject: [PATCH 079/138] [php] Add migration procedure for php7.0 -> php7.3 pool files --- .../0016_php70_to_php73_pools.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/yunohost/data_migrations/0016_php70_to_php73_pools.py diff --git a/src/yunohost/data_migrations/0016_php70_to_php73_pools.py b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py new file mode 100644 index 000000000..099c01a8a --- /dev/null +++ b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py @@ -0,0 +1,70 @@ +import os +import glob +from shutil import copy2 + +from moulinette.utils.log import getActionLogger + +from yunohost.tools import Migration +from yunohost.service import _run_service_command + +logger = getActionLogger('yunohost.migration') + +PHP70_POOLS = "/etc/php/7.0/fpm/pool.d" +PHP73_POOLS = "/etc/php/7.3/fpm/pool.d" + +PHP70_SOCKETS_PREFIX = "/run/php/php7.0-fpm" +PHP73_SOCKETS_PREFIX = "/run/php/php7.3-fpm" + +MIGRATION_COMMENT = "; YunoHost note : this file was automatically moved from {}".format(PHP70_POOLS) + + +class MyMigration(Migration): + + "Migrate php7.0-fpm 'pool' conf files to php7.3" + + dependencies = ["migrate_to_buster"] + + def run(self): + # Get list of php7.0 pool files + php70_pool_files = glob.glob("{}/*.conf".format(PHP70_POOLS)) + + # Keep only basenames + php70_pool_files = [os.path.basename(f) for f in php70_pool_files] + + # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) + php70_pool_files = [f for f in php70_pool_files if f != "www.conf"] + + for f in php70_pool_files: + + # Copy the files to the php7.3 pool + src = "{}/{}".format(PHP70_POOLS, f) + dest = "{}/{}".format(PHP73_POOLS, f) + copy2(src, dest) + + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, dest) + os.system(c) + + # Also add a comment that it was automatically moved from php7.0 + # (for human traceability and backward migration) + c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) + os.system(c) + + # Reload/restart the php pools + _run_service_command("restart", "php7.3-fpm") + _run_service_command("enable", "php7.3-fpm") + os.system("systemctl stop php7.0-fpm") + os.system("systemctl disable php7.0-fpm") + os.system("rm /etc/logrotate.d/php7.0-fpm") # We remove this otherwise the logrotate cron will be unhappy + + # Get list of nginx conf file + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, f) + os.system(c) + # FIXME : should tweak the checksum setting in settings.yml of the app so that the file ain't considered manually modified + # Aslo gotta tweak the settings fpm_service and fpm_config_dir ... + + # Reload nginx + _run_service_command("reload", "nginx") From 9dbf626b36d6ff401d15c39bf721fceaeb2aeed1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 9 Mar 2020 20:36:44 +0100 Subject: [PATCH 080/138] [php] More ugly hack for php helpers shipped by apps that assume php 7.0 by default --- src/yunohost/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 56574acf6..8faf587ff 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2879,7 +2879,9 @@ LEGACY_PHP_VERSION_REPLACEMENTS = [ ("/var/run/php5-fpm", "/var/run/php/php7.3-fpm"), ("/var/run/php/php7.0-fpm", "/var/run/php/php7.3-fpm"), ("php5", "php7.3"), - ("php7.0", "php7.3") + ("php7.0", "php7.3"), + ('phpversion="${phpversion:-7.0}"', 'phpversion="${phpversion:-7.3}"'), # Many helpers like the composer ones use 7.0 by default ... + ('"$phpversion" == "7.0"', '$(bc <<< "$phpversion >= 7.3") -eq 1') # patch ynh_install_php to refuse installing/removing php <= 7.3 ] def _patch_legacy_php_versions(app_folder): From b747c17369adb6a8f359059c45d965e7defaca2a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 21 Mar 2020 18:41:06 +0100 Subject: [PATCH 081/138] [php] Also patch stuff in subfolers inside scripts/ (some apps have an experimental_helpers/ folder) --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 8faf587ff..b198ab3b3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2889,6 +2889,7 @@ def _patch_legacy_php_versions(app_folder): files_to_patch = [] files_to_patch.extend(glob.glob("%s/conf/*" % app_folder)) files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder)) files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) files_to_patch.append("%s/manifest.json" % app_folder) files_to_patch.append("%s/manifest.toml" % app_folder) From a90540ddeb7cfd6327857d1cf610658fdc22f9cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 8 Apr 2020 13:01:35 +0200 Subject: [PATCH 082/138] [php] php-mcrypt doesn't exist anymore in buster --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index cd6b6e495..1f2c24678 100644 --- a/debian/control +++ b/debian/control @@ -32,7 +32,7 @@ Depends: ${python:Depends}, ${misc:Depends} Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog - , php-gd, php-curl, php-gettext, php-mcrypt + , php-gd, php-curl, php-gettext , python-pip , unattended-upgrades , libdbd-ldap-perl, libnet-dns-perl From b6d1bb790174f892c82726a7f936f2b57babaad8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 17 May 2020 18:26:18 +0200 Subject: [PATCH 083/138] [php] Also migrate app settings to fix inconsistencies (e.g. during remove of migrated apps) --- .../0016_php70_to_php73_pools.py | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/yunohost/data_migrations/0016_php70_to_php73_pools.py b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py index 099c01a8a..fa2e04a4a 100644 --- a/src/yunohost/data_migrations/0016_php70_to_php73_pools.py +++ b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py @@ -4,6 +4,7 @@ from shutil import copy2 from moulinette.utils.log import getActionLogger +from yunohost.app import _is_installed, _get_app_settings, _set_app_settings from yunohost.tools import Migration from yunohost.service import _run_service_command @@ -50,21 +51,44 @@ class MyMigration(Migration): c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) os.system(c) + app_id = os.path.basename(f)[:-len(".conf")] + self.migrate_app_settings(app_id) + + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" % app_id) + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, f) + os.system(c) + + os.system("rm /etc/logrotate.d/php7.0-fpm") # We remove this otherwise the logrotate cron will be unhappy + # Reload/restart the php pools _run_service_command("restart", "php7.3-fpm") _run_service_command("enable", "php7.3-fpm") os.system("systemctl stop php7.0-fpm") os.system("systemctl disable php7.0-fpm") - os.system("rm /etc/logrotate.d/php7.0-fpm") # We remove this otherwise the logrotate cron will be unhappy - - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, f) - os.system(c) - # FIXME : should tweak the checksum setting in settings.yml of the app so that the file ain't considered manually modified - # Aslo gotta tweak the settings fpm_service and fpm_config_dir ... # Reload nginx _run_service_command("reload", "nginx") + + def migrate_app_settings(self, app_id): + + if not _is_installed(app_id): + return + + settings = _get_app_settings(app_id) + + if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm": + settings["fpm_config_dir"] = "/etc/php/7.3/fpm" + if settings.get("fpm_service") == "php7.0-fpm": + settings["fpm_service"] = "php7.3-fpm" + if settings.get("phpversion") == "7.0": + settings["phpversion"] = "7.3" + + # We delete these checksums otherwise the file will appear as manually modified + list_to_remove = ["checksum__etc_php_7.0_fpm_pool", + "checksum__etc_nginx_conf.d"] + settings = {k: v for k, v in settings.items() + if not any(k.startswith(to_remove) for to_remove in list_to_remove)} + + _set_app_settings(app_id, settings) From 73356eed75033762eb28e3aff1ff1290f01fff13 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 May 2020 16:30:36 +0200 Subject: [PATCH 084/138] [php] Also patch php7.0 settings on the fly during restore --- src/yunohost/app.py | 21 +++++++++++++++ src/yunohost/backup.py | 9 ++++++- .../0016_php70_to_php73_pools.py | 27 +++---------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b198ab3b3..00472744f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2884,6 +2884,7 @@ LEGACY_PHP_VERSION_REPLACEMENTS = [ ('"$phpversion" == "7.0"', '$(bc <<< "$phpversion >= 7.3") -eq 1') # patch ynh_install_php to refuse installing/removing php <= 7.3 ] + def _patch_legacy_php_versions(app_folder): files_to_patch = [] @@ -2906,6 +2907,26 @@ def _patch_legacy_php_versions(app_folder): os.system(c) +def _patch_legacy_php_versions_in_settings(app_folder): + + settings = read_yaml(os.path.join(app_folder, 'settings.yml')) + + if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm": + settings["fpm_config_dir"] = "/etc/php/7.3/fpm" + if settings.get("fpm_service") == "php7.0-fpm": + settings["fpm_service"] = "php7.3-fpm" + if settings.get("phpversion") == "7.0": + settings["phpversion"] = "7.3" + + # We delete these checksums otherwise the file will appear as manually modified + list_to_remove = ["checksum__etc_php_7.0_fpm_pool", + "checksum__etc_nginx_conf.d"] + settings = {k: v for k, v in settings.items() + if not any(k.startswith(to_remove) for to_remove in list_to_remove)} + + write_to_yaml(app_folder + '/settings.yml', settings) + + def _patch_legacy_helpers(app_folder): files_to_patch = [] diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 8ba8f2610..4b55ee83d 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -43,7 +43,13 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml from yunohost.app import ( - app_info, _is_installed, _parse_app_instance_name, _patch_legacy_php_versions, dump_app_log_extract_for_debugging, _patch_legacy_helpers, LEGACY_PHP_VERSION_REPLACEMENTS + app_info, _is_installed, + _parse_app_instance_name, + dump_app_log_extract_for_debugging, + _patch_legacy_helpers, + _patch_legacy_php_versions, + _patch_legacy_php_versions_in_settings, + LEGACY_PHP_VERSION_REPLACEMENTS ) from yunohost.hook import ( hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER @@ -1329,6 +1335,7 @@ class RestoreManager(): # Apply dirty patch to make php5 apps compatible with php7 _patch_legacy_php_versions(app_settings_in_archive) + _patch_legacy_php_versions_in_settings(app_settings_in_archive) # Delete _common.sh file in backup common_file = os.path.join(app_backup_in_archive, '_common.sh') diff --git a/src/yunohost/data_migrations/0016_php70_to_php73_pools.py b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py index fa2e04a4a..1cdb2bc4d 100644 --- a/src/yunohost/data_migrations/0016_php70_to_php73_pools.py +++ b/src/yunohost/data_migrations/0016_php70_to_php73_pools.py @@ -4,7 +4,7 @@ from shutil import copy2 from moulinette.utils.log import getActionLogger -from yunohost.app import _is_installed, _get_app_settings, _set_app_settings +from yunohost.app import _is_installed, _get_app_settings, _set_app_settings, _patch_legacy_php_versions_in_settings from yunohost.tools import Migration from yunohost.service import _run_service_command @@ -52,7 +52,8 @@ class MyMigration(Migration): os.system(c) app_id = os.path.basename(f)[:-len(".conf")] - self.migrate_app_settings(app_id) + if _is_installed(app_id): + _patch_legacy_php_versions_in_settings("/etc/yunohost/apps/%s/" % app_id) nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" % app_id) for f in nginx_conf_files: @@ -70,25 +71,3 @@ class MyMigration(Migration): # Reload nginx _run_service_command("reload", "nginx") - - def migrate_app_settings(self, app_id): - - if not _is_installed(app_id): - return - - settings = _get_app_settings(app_id) - - if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm": - settings["fpm_config_dir"] = "/etc/php/7.3/fpm" - if settings.get("fpm_service") == "php7.0-fpm": - settings["fpm_service"] = "php7.3-fpm" - if settings.get("phpversion") == "7.0": - settings["phpversion"] = "7.3" - - # We delete these checksums otherwise the file will appear as manually modified - list_to_remove = ["checksum__etc_php_7.0_fpm_pool", - "checksum__etc_nginx_conf.d"] - settings = {k: v for k, v in settings.items() - if not any(k.startswith(to_remove) for to_remove in list_to_remove)} - - _set_app_settings(app_id, settings) From 70fab24247786a9331b02df1c3090c3137e9ba73 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 9 Mar 2020 18:32:09 +0100 Subject: [PATCH 085/138] [psql] Upgrade postgresql helper to use version 11 --- data/helpers.d/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 7eb4e7289..11b9c0fed 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -1,7 +1,7 @@ #!/bin/bash PSQL_ROOT_PWD_FILE=/etc/yunohost/psql -PSQL_VERSION=9.6 +PSQL_VERSION=11 # Open a connection as a user # From 50f1e9a6819f79f6fcfb42495303bdd95df9a81c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 12 Mar 2020 20:12:12 +0100 Subject: [PATCH 086/138] [psql] Add migration for Postgresql cluster upgrade from 9.6 to 11 --- locales/en.json | 4 ++ .../0017_postgresql_9p6_to_11.py | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py diff --git a/locales/en.json b/locales/en.json index da6c759b9..7813e4410 100644 --- a/locales/en.json +++ b/locales/en.json @@ -429,6 +429,10 @@ "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently...", "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore...", "migration_0015_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", + "migration_description_0017_postgresql_9p6_to_11": "Migrate databases from PostgreSQL 9.6 to 11", + "migration_0017_postgresql_96_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", + "migration_0017_postgresql_11_not_installed": "PostgreSQL 9.6 is installed, but not postgresql 11‽ Something weird might have happened on your system :(...", + "migration_0017_not_enough_space": "Make sufficient space available in {path} to run the migration.", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.", diff --git a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py new file mode 100644 index 000000000..f03cd9c8c --- /dev/null +++ b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py @@ -0,0 +1,41 @@ +import subprocess + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger + +from yunohost.tools import Migration +from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory + +logger = getActionLogger('yunohost.migration') + + +class MyMigration(Migration): + + "Migrate DBs from Postgresql 9.6 to 11 after migrating to Buster" + + dependencies = ["migrate_to_buster"] + + def run(self): + + if not self.package_is_installed("postgresql-9.6"): + logger.warning(m18n.n("migration_0017_postgresql_96_not_installed")) + return + + if not self.package_is_installed("postgresql-11"): + raise YunohostError("migration_0017_postgresql_11_not_installed") + + if not space_used_by_directory("/var/lib/postgresql/9.6") > free_space_in_directory("/var/lib/postgresql"): + raise YunohostError("migration_0017_not_enough_space", path="/var/lib/postgresql/") + + subprocess.check_call("systemctl stop postgresql", shell=True) + subprocess.check_call("pg_dropcluster --stop 11 main", shell=True) + subprocess.check_call("pg_upgradecluster -m upgrade 9.6 main", shell=True) + subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True) + subprocess.check_call("systemctl start postgresql", shell=True) + + def package_is_installed(self, package_name): + + p = subprocess.Popen("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), shell=True) + p.communicate() + return p.returncode == 0 From 56a5b5aae2a64d56e1bde6ecd9d69940bd308cd8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 18 May 2020 02:11:12 +0200 Subject: [PATCH 087/138] [psql] Fix hardcoded fix for postgresql status check --- src/yunohost/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index f12c0743c..2eeb078bc 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -618,7 +618,7 @@ def _get_services(): if "postgresql" in services: if "description" in services["postgresql"]: del services["postgresql"]["description"] - services["postgresql"]["actual_systemd_service"] = "postgresql@9.6-main" + services["postgresql"]["actual_systemd_service"] = "postgresql@11-main" return services From bbb6a4ce91e90d083c99176130e400d695208a43 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 17 May 2020 16:41:54 +0200 Subject: [PATCH 088/138] [firewall] Add draft of migration for xtable->nftable --- locales/en.json | 3 +- .../data_migrations/0018_xtable_to_nftable.py | 92 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 src/yunohost/data_migrations/0018_xtable_to_nftable.py diff --git a/locales/en.json b/locales/en.json index 7813e4410..d40fcfb14 100644 --- a/locales/en.json +++ b/locales/en.json @@ -429,10 +429,11 @@ "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently...", "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore...", "migration_0015_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", - "migration_description_0017_postgresql_9p6_to_11": "Migrate databases from PostgreSQL 9.6 to 11", "migration_0017_postgresql_96_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", "migration_0017_postgresql_11_not_installed": "PostgreSQL 9.6 is installed, but not postgresql 11‽ Something weird might have happened on your system :(...", "migration_0017_not_enough_space": "Make sufficient space available in {path} to run the migration.", + "migration_0018_failed_to_migrate_iptables_rules": "Failed to migrate legacy iptables rules to nftables: {error}", + "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.", diff --git a/src/yunohost/data_migrations/0018_xtable_to_nftable.py b/src/yunohost/data_migrations/0018_xtable_to_nftable.py new file mode 100644 index 000000000..7d3810518 --- /dev/null +++ b/src/yunohost/data_migrations/0018_xtable_to_nftable.py @@ -0,0 +1,92 @@ +import os +import subprocess + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger + +from yunohost.firewall import firewall_reload +from yunohost.service import service_restart +from yunohost.tools import Migration + +logger = getActionLogger('yunohost.migration') + + +class MyMigration(Migration): + + "Migrate legacy iptables rules from stretch that relied on xtable and should now rely on nftable" + + dependencies = ["migrate_to_buster"] + + def run(self): + + self.do_ipv4 = os.system("iptables -w -L >/dev/null") == 0 + self.do_ipv6 = os.system("ip6tables -w -L >/dev/null") == 0 + + if not self.do_ipv4: + logger.warning(m18n.n('iptables_unavailable')) + if not self.do_ipv6: + logger.warning(m18n.n('ip6tables_unavailable')) + + backup_folder = "/home/yunohost.backup/premigration/xtable_to_nftable/" + if not os.path.exists(backup_folder): + os.makedirs(backup_folder, 0o750) + self.backup_rules_ipv4 = os.path.join(backup_folder, "legacy_rules_ipv4") + self.backup_rules_ipv6 = os.path.join(backup_folder, "legacy_rules_ipv6") + + # Backup existing legacy rules to be able to rollback + if self.do_ipv4 and not os.path.exists(self.backup_rules_ipv4): + os.system("iptables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? + subprocess.check_call("iptables-legacy-save > %s" % self.backup_rules_ipv4, shell=True) + assert subprocess.check_output("cat %s" % self.backup_rules_ipv4, shell=True).strip(), "Uhoh backup of legacy ipv4 rules is empty !?" + if self.do_ipv6 and not os.path.exists(self.backup_rules_ipv6): + os.system("ip6tables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? + subprocess.check_call("ip6tables-legacy-save > %s" % self.backup_rules_ipv6, shell=True) + assert subprocess.check_output("cat %s" % self.backup_rules_ipv6, shell=True).strip(), "Uhoh backup of legacy ipv6 rules is empty !?" + + # We inject the legacy rules (iptables-legacy) into the new iptable (just "iptables") + try: + if self.do_ipv4: + subprocess.check_call("iptables-legacy-save | iptables-restore", shell=True) + if self.do_ipv6: + subprocess.check_call("ip6tables-legacy-save | ip6tables-restore", shell=True) + except Exception as e: + self.rollback() + raise YunohostError("migration_0018_failed_to_migrate_iptables_rules", error=e) + + # Reset everything in iptables-legacy + # Stolen from https://serverfault.com/a/200642 + try: + if self.do_ipv4: + subprocess.check_call( + "iptables-legacy-save | awk '/^[*]/ { print $1 }" # Keep lines like *raw, *filter and *nat + " /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }" # Turn all policies to accept + " /COMMIT/ { print $0; }'" # Keep the line COMMIT + " | iptables-legacy-restore", + shell=True) + if self.do_ipv6: + subprocess.check_call( + "ip6tables-legacy-save | awk '/^[*]/ { print $1 }" # Keep lines like *raw, *filter and *nat + " /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }" # Turn all policies to accept + " /COMMIT/ { print $0; }'" # Keep the line COMMIT + " | ip6tables-legacy-restore", + shell=True) + except Exception as e: + self.rollback() + raise YunohostError("migration_0018_failed_to_reset_legacy_rules", error=e) + + # You might be wondering "uh but is it really useful to + # iptables-legacy-save | iptables-restore considering firewall_reload() + # flush/resets everything anyway ?" + # But the answer is : firewall_reload() only resets the *filter table. + # On more complex setups (e.g. internet cube or docker) you will also + # have rules in the *nat (or maybe *raw?) sections of iptables. + firewall_reload() + service_restart("fail2ban") + + def rollback(self): + + if self.do_ipv4: + subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv4, shell=True) + if self.do_ipv6: + subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv6, shell=True) From 591474095760771f4a06f0e8f878f233bef23915 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 17 May 2020 16:55:41 +0200 Subject: [PATCH 089/138] [firewall] nftables is the new iptables-persistent and it should conflict with yunohost-firewall --- data/hooks/conf_regen/01-yunohost | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index 61951fe5f..1081def17 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -75,7 +75,16 @@ EOF ConditionCapability=CAP_SYS_TIME ConditionVirtualization=!container " > ${pending_dir}/etc/systemd/system/ntp.service.d/ynh-override.conf - + + # Make nftable conflict with yunohost-firewall + mkdir -p ${pending_dir}/etc/systemd/system/nftables.service.d/ + cat > ${pending_dir}/etc/systemd/system/nftables.service.d/ynh-override.conf << EOF +[Unit] +# yunohost-firewall and nftables conflict with each other +Conflicts=yunohost-firewall.service +ConditionFileIsExecutable=!/etc/init.d/yunohost-firewall +ConditionPathExists=!/etc/systemd/system/multi-user.target.wants/yunohost-firewall.service +EOF } do_post_regen() { @@ -100,7 +109,9 @@ do_post_regen() { [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d) [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps) + # Propagates changes in systemd service config overrides [[ ! "$regen_conf_files" =~ "ntp.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl restart ntp; } + [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl disable nftables -q; } } _update_services() { From ca20eda9caf3849a9b7391e0f0fc8d4eba751874 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 17 May 2020 20:09:50 +0200 Subject: [PATCH 090/138] [firewall] Let's not disable nftables that makes the script crash if it's not installed and we don't really need to disable it anyway since it's already configured to conflict --- data/hooks/conf_regen/01-yunohost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index 1081def17..c4120d487 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -111,7 +111,7 @@ do_post_regen() { # Propagates changes in systemd service config overrides [[ ! "$regen_conf_files" =~ "ntp.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl restart ntp; } - [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl disable nftables -q; } + [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload } _update_services() { From 70997503239a0e725d44a2d43f31b2ddc6f09008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 7 May 2020 00:22:53 +0200 Subject: [PATCH 091/138] Change file type of LDAP schema from 'schema' to ldif --- .../{mailserver.schema => mailserver.ldif} | 42 +++++++++------- .../{yunohost.schema => permission.ldif} | 22 ++++---- .../slapd/{sudo.schema => sudo.ldif} | 50 ++++++++++--------- 3 files changed, 61 insertions(+), 53 deletions(-) rename data/templates/slapd/{mailserver.schema => mailserver.ldif} (79%) rename data/templates/slapd/{yunohost.schema => permission.ldif} (55%) rename data/templates/slapd/{sudo.schema => sudo.ldif} (72%) diff --git a/data/templates/slapd/mailserver.schema b/data/templates/slapd/mailserver.ldif similarity index 79% rename from data/templates/slapd/mailserver.schema rename to data/templates/slapd/mailserver.ldif index 23d0d24bd..849d1d9e1 100644 --- a/data/templates/slapd/mailserver.schema +++ b/data/templates/slapd/mailserver.ldif @@ -2,58 +2,62 @@ ## Version 0.1 ## Adrien Beudin +dn: cn=mailserver,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: mailserver +# # Attributes -attributetype ( 1.3.6.1.4.1.40328.1.20.2.1 +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.1 NAME 'maildrop' DESC 'Mail addresses where mails are forwarded -- ie forwards' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.2 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.2 NAME 'mailalias' DESC 'Mail addresses accepted by this account -- ie aliases' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.3 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.3 NAME 'mailenable' DESC 'Mail Account validity' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{8}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.4 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.4 NAME 'mailbox' DESC 'Mailbox path where mails are delivered' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.5 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.5 NAME 'virtualdomain' DESC 'A mail domain name' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.6 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.6 NAME 'virtualdomaindescription' DESC 'Virtual domain description' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512}) - -attributetype ( 1.3.6.1.4.1.40328.1.20.2.7 +# +olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.7 NAME 'mailuserquota' DESC 'Mailbox quota for a user' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{16} SINGLE-VALUE ) - +# # Mail Account Objectclass -objectclass ( 1.3.6.1.4.1.40328.1.1.2.1 +olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.1 NAME 'mailAccount' DESC 'Mail Account' SUP top @@ -65,9 +69,9 @@ objectclass ( 1.3.6.1.4.1.40328.1.1.2.1 mailalias $ maildrop $ mailenable $ mailbox $ mailuserquota ) ) - +# # Mail Domain Objectclass -objectclass ( 1.3.6.1.4.1.40328.1.1.2.2 +olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.2 NAME 'mailDomain' DESC 'Domain mail entry' SUP top @@ -79,9 +83,9 @@ objectclass ( 1.3.6.1.4.1.40328.1.1.2.2 virtualdomaindescription $ mailuserquota ) ) - +# # Mail Group Objectclass -objectclass ( 1.3.6.1.4.1.40328.1.1.2.3 +olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.3 NAME 'mailGroup' SUP top AUXILIARY DESC 'Mail Group' MUST ( mail ) diff --git a/data/templates/slapd/yunohost.schema b/data/templates/slapd/permission.ldif similarity index 55% rename from data/templates/slapd/yunohost.schema rename to data/templates/slapd/permission.ldif index 7da60a20c..a97249d07 100644 --- a/data/templates/slapd/yunohost.schema +++ b/data/templates/slapd/permission.ldif @@ -1,33 +1,35 @@ -#dn: cn=yunohost,cn=schema,cn=config -#objectClass: olcSchemaConfig -#cn: yunohost +# Yunohost schema for group and permission support + +dn: cn=yunohost,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: yunohost # ATTRIBUTES # For Permission -attributetype ( 1.3.6.1.4.1.17953.9.1.1 NAME 'permission' +olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.1 NAME 'permission' DESC 'Yunohost permission on user and group side' SUP distinguishedName ) -attributetype ( 1.3.6.1.4.1.17953.9.1.2 NAME 'groupPermission' +olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.2 NAME 'groupPermission' DESC 'Yunohost permission for a group on permission side' SUP distinguishedName ) -attributetype ( 1.3.6.1.4.1.17953.9.1.3 NAME 'inheritPermission' +olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.3 NAME 'inheritPermission' DESC 'Yunohost permission for user on permission side' SUP distinguishedName ) -attributetype ( 1.3.6.1.4.1.17953.9.1.4 NAME 'URL' +olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.4 NAME 'URL' DESC 'Yunohost application URL' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} ) # OBJECTCLASS # For Applications -objectclass ( 1.3.6.1.4.1.17953.9.2.1 NAME 'groupOfNamesYnh' +olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.1 NAME 'groupOfNamesYnh' DESC 'Yunohost user group' SUP top AUXILIARY MAY ( member $ businessCategory $ seeAlso $ owner $ ou $ o $ permission ) ) -objectclass ( 1.3.6.1.4.1.17953.9.2.2 NAME 'permissionYnh' +olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.2 NAME 'permissionYnh' DESC 'a Yunohost application' SUP top AUXILIARY MUST cn MAY ( groupPermission $ inheritPermission $ URL ) ) # For User -objectclass ( 1.3.6.1.4.1.17953.9.2.3 NAME 'userPermissionYnh' +olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.3 NAME 'userPermissionYnh' DESC 'a Yunohost application' SUP top AUXILIARY MAY ( permission ) ) diff --git a/data/templates/slapd/sudo.schema b/data/templates/slapd/sudo.ldif similarity index 72% rename from data/templates/slapd/sudo.schema rename to data/templates/slapd/sudo.ldif index d3e95e00c..a7088c855 100644 --- a/data/templates/slapd/sudo.schema +++ b/data/templates/slapd/sudo.ldif @@ -1,76 +1,78 @@ # # OpenLDAP schema file for Sudo -# Save as /etc/openldap/schema/sudo.schema +# Save as /etc/openldap/schema/sudo.ldif # -attributetype ( 1.3.6.1.4.1.15953.9.1.1 +dn: cn=sudo,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: sudo +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.1 NAME 'sudoUser' DESC 'User(s) who may run sudo' EQUALITY caseExactIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.2 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.2 NAME 'sudoHost' DESC 'Host(s) who may run sudo' EQUALITY caseExactIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.3 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.3 NAME 'sudoCommand' DESC 'Command(s) to be executed by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.4 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.4 NAME 'sudoRunAs' DESC 'User(s) impersonated by sudo (deprecated)' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.5 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.5 NAME 'sudoOption' DESC 'Options(s) followed by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.6 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.6 NAME 'sudoRunAsUser' DESC 'User(s) impersonated by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.7 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.7 NAME 'sudoRunAsGroup' DESC 'Group(s) impersonated by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.8 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.8 NAME 'sudoNotBefore' DESC 'Start of time interval for which the entry is valid' EQUALITY generalizedTimeMatch ORDERING generalizedTimeOrderingMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 ) - -attributetype ( 1.3.6.1.4.1.15953.9.1.9 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.9 NAME 'sudoNotAfter' DESC 'End of time interval for which the entry is valid' EQUALITY generalizedTimeMatch ORDERING generalizedTimeOrderingMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 ) - -attributeTypes ( 1.3.6.1.4.1.15953.9.1.10 +# +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.10 NAME 'sudoOrder' DESC 'an integer to order the sudoRole entries' EQUALITY integerMatch ORDERING integerOrderingMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 ) - -objectclass ( 1.3.6.1.4.1.15953.9.2.1 NAME 'sudoRole' SUP top STRUCTURAL +# +olcObjectClasses: ( 1.3.6.1.4.1.15953.9.2.1 NAME 'sudoRole' SUP top STRUCTURAL DESC 'Sudoer Entries' MUST ( cn ) - MAY ( sudoUser $ sudoHost $ sudoCommand $ sudoRunAs $ sudoRunAsUser $ sudoRunAsGroup $ sudoOption $ sudoOrder $ sudoNotBefore $ sudoNotAfter $ - description ) + MAY ( sudoUser $ sudoHost $ sudoCommand $ sudoRunAs $ sudoRunAsUser $ sudoRunAsGroup $ sudoOption $ sudoOrder $ sudoNotBefore $ sudoNotAfter $ description ) ) From d73a71fa610fb28488a4af19cde08a8bc8cd0f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 7 May 2020 00:24:53 +0200 Subject: [PATCH 092/138] Rewrite LDAP configuration in LDIF format --- data/templates/slapd/slapd.conf | 154 --------------------- data/templates/slapd/slapd.ldif | 229 ++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 154 deletions(-) delete mode 100644 data/templates/slapd/slapd.conf create mode 100644 data/templates/slapd/slapd.ldif diff --git a/data/templates/slapd/slapd.conf b/data/templates/slapd/slapd.conf deleted file mode 100644 index 0750b43aa..000000000 --- a/data/templates/slapd/slapd.conf +++ /dev/null @@ -1,154 +0,0 @@ -# This is the main slapd configuration file. See slapd.conf(5) for more -# info on the configuration options. - -####################################################################### -# Global Directives: - -# Features to permit -#allow bind_v2 - -# Schema and objectClass definitions -include /etc/ldap/schema/core.schema -include /etc/ldap/schema/cosine.schema -include /etc/ldap/schema/nis.schema -include /etc/ldap/schema/inetorgperson.schema -include /etc/ldap/schema/mailserver.schema -include /etc/ldap/schema/sudo.schema -include /etc/ldap/schema/yunohost.schema - -# Where the pid file is put. The init.d script -# will not stop the server if you change this. -pidfile /var/run/slapd/slapd.pid - -# List of arguments that were passed to the server -argsfile /var/run/slapd/slapd.args - -# Read slapd.conf(5) for possible values -loglevel none - -# Hashes to be used in generation of user passwords -password-hash {SSHA} - -# Where the dynamically loaded modules are stored -modulepath /usr/lib/ldap -moduleload back_mdb -moduleload memberof - -# The maximum number of entries that is returned for a search operation -sizelimit 500 - -# The tool-threads parameter sets the actual amount of cpu's that is used -# for indexing. -tool-threads 1 - -# TLS Support -TLSCertificateFile /etc/yunohost/certs/yunohost.org/crt.pem -TLSCertificateKeyFile /etc/yunohost/certs/yunohost.org/key.pem - -####################################################################### -# Specific Backend Directives for mdb: -# Backend specific directives apply to this backend until another -# 'backend' directive occurs -backend mdb - -####################################################################### -# Specific Directives for database #1, of type mdb: -# Database specific directives apply to this databasse until another -# 'database' directive occurs -database mdb - -# The base of your directory in database #1 -suffix "dc=yunohost,dc=org" - -# rootdn directive for specifying a superuser on the database. This is needed -# for syncrepl. -# rootdn "cn=admin,dc=yunohost,dc=org" - -# Where the database file are physically stored for database #1 -directory "/var/lib/ldap" - -# Indexing options for database #1 -index objectClass eq -index uid,sudoUser eq,sub -index entryCSN,entryUUID eq -index cn,mail eq -index gidNumber,uidNumber eq -index member,memberUid,uniqueMember eq -index virtualdomain eq -index permission eq - -# Save the time that the entry gets modified, for database #1 -lastmod on - -# Checkpoint the BerkeleyDB database periodically in case of system -# failure and to speed slapd shutdown. -checkpoint 512 30 - -# The userPassword by default can be changed -# by the entry owning it if they are authenticated. -# Others should not be able to see it, except the -# admin entry below -# These access lines apply to database #1 only -access to attrs=userPassword,shadowLastChange - by dn="cn=admin,dc=yunohost,dc=org" write - by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write - by anonymous auth - by self write - by * none - -# Personnal information can be changed by the entry -# owning it if they are authenticated. -# Others should be able to see it. -access to attrs=cn,gecos,givenName,mail,maildrop,displayName,sn - by dn="cn=admin,dc=yunohost,dc=org" write - by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write - by self write - by * read - -# Ensure read access to the base for things like -# supportedSASLMechanisms. Without this you may -# have problems with SASL not knowing what -# mechanisms are available and the like. -# Note that this is covered by the 'access to *' -# ACL below too but if you change that as people -# are wont to do you'll still need this if you -# want SASL (and possible other things) to work -# happily. -access to dn.base="" by * read - -# The admin dn has full write access, everyone else -# can read everything. -access to * - by dn="cn=admin,dc=yunohost,dc=org" write - by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write - by group/groupOfNames/Member="cn=admin,ou=groups,dc=yunohost,dc=org" write - by * read - -# Configure Memberof Overlay (used for Yunohost permission) - -# Link user <-> group -#dn: olcOverlay={0}memberof,olcDatabase={1}mdb,cn=config -overlay memberof -memberof-group-oc groupOfNamesYnh -memberof-member-ad member -memberof-memberof-ad memberOf -memberof-dangling error -memberof-refint TRUE - -# Link permission <-> groupes -#dn: olcOverlay={1}memberof,olcDatabase={1}mdb,cn=config -overlay memberof -memberof-group-oc permissionYnh -memberof-member-ad groupPermission -memberof-memberof-ad permission -memberof-dangling error -memberof-refint TRUE - -# Link permission <-> user -#dn: olcOverlay={2}memberof,olcDatabase={1}mdb,cn=config -overlay memberof -memberof-group-oc permissionYnh -memberof-member-ad inheritPermission -memberof-memberof-ad permission -memberof-dangling error -memberof-refint TRUE diff --git a/data/templates/slapd/slapd.ldif b/data/templates/slapd/slapd.ldif new file mode 100644 index 000000000..c9b6c581f --- /dev/null +++ b/data/templates/slapd/slapd.ldif @@ -0,0 +1,229 @@ +# OpenLDAP server configuration for Yunohost +# ------------------------------------------ +# +# By the Yunohost regen-conf tools it's NOT possible to edit the config database by a LDAP request. +# The way to to edit the config database is to edit THIS file +# and after update the config database based on this file. +# +# Config database customization: +# 1. Edit this file as you want. +# 2. Apply your modifications. For this just run this following command in a shell: +# $ /usr/share/yunohost/hooks/conf_regen/06-slapd apply_config +# +# Note that if you customize this file after all modifications my Yunohost won't be applied. + +# +# Main configuration +# +dn: cn=config +objectClass: olcGlobal +cn: config +olcConfigFile: /etc/ldap/slapd.conf +olcConfigDir: /etc/ldap/slapd.d/ +# List of arguments that were passed to the server +olcArgsFile: /var/run/slapd/slapd.args +# +olcAttributeOptions: lang- +olcAuthzPolicy: none +olcConcurrency: 0 +olcConnMaxPending: 100 +olcConnMaxPendingAuth: 1000 +olcIdleTimeout: 0 +olcIndexSubstrIfMaxLen: 4 +olcIndexSubstrIfMinLen: 2 +olcIndexSubstrAnyLen: 4 +olcIndexSubstrAnyStep: 2 +olcIndexIntLen: 4 +olcListenerThreads: 1 +olcLocalSSF: 71 +# Read slapd.conf(5) for possible values +olcLogLevel: None +# Where the pid file is put. The init.d script +# will not stop the server if you change this. +olcPidFile: /var/run/slapd/slapd.pid +olcReverseLookup: FALSE +olcThreads: 16 +# TLS Support +olcTLSCertificateFile: /etc/yunohost/certs/yunohost.org/crt.pem +olcTLSCertificateKeyFile: /etc/yunohost/certs/yunohost.org/key.pem +olcTLSVerifyClient: never +olcTLSProtocolMin: 0.0 +# The tool-threads parameter sets the actual amount of cpu's that is used +# for indexing. +olcToolThreads: 1 +structuralObjectClass: olcGlobal + +# +# Schema and objectClass definitions +# +dn: cn=schema,cn=config +objectClass: olcSchemaConfig +cn: schema + +include: file:///etc/ldap/schema/core.ldif +include: file:///etc/ldap/schema/cosine.ldif +include: file:///etc/ldap/schema/nis.ldif +include: file:///etc/ldap/schema/inetorgperson.ldif +include: file:///etc/ldap/schema/mailserver.ldif +include: file:///etc/ldap/schema/sudo.ldif +include: file:///etc/ldap/schema/permission.ldif + +# +# Module management +# +dn: cn=module{0},cn=config +objectClass: olcModuleList +cn: module{0} +# Where the dynamically loaded modules are stored +olcModulePath: /usr/lib/ldap +olcModuleLoad: {0}back_mdb +olcModuleLoad: {1}memberof +structuralObjectClass: olcModuleList + +# +# Frontend database +# +dn: olcDatabase={-1}frontend,cn=config +objectClass: olcDatabaseConfig +objectClass: olcFrontendConfig +olcDatabase: {-1}frontend +olcAddContentAcl: FALSE +olcLastMod: TRUE +olcSchemaDN: cn=Subschema +# Hashes to be used in generation of user passwords +olcPasswordHash: {SSHA} +structuralObjectClass: olcDatabaseConfig + +# +# Config database Configuration (#0) +# +dn: olcDatabase={0}config,cn=config +objectClass: olcDatabaseConfig +olcDatabase: {0}config +# Give access to root user. +# This give the possiblity to the admin to customize the LDAP configuration +olcAccess: {0}to * by * none +olcAddContentAcl: TRUE +olcLastMod: TRUE +olcRootDN: cn=config +structuralObjectClass: olcDatabaseConfig + +# +# Main database Configuration (#1) +# +dn: olcDatabase={1}mdb,cn=config +objectClass: olcDatabaseConfig +objectClass: olcMdbConfig +olcDatabase: {1}mdb +# The base of your directory in database #1 +olcSuffix: dc=yunohost,dc=org +# +# The userPassword by default can be changed +# by the entry owning it if they are authenticated. +# Others should not be able to see it, except the +# admin entry below +# These access lines apply to database #1 only +olcAccess: {0}to attrs=userPassword,shadowLastChange + by dn.base="cn=admin,dc=yunohost,dc=org" write + by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write + by anonymous auth + by self write + by * none +# +# Personnal information can be changed by the entry +# owning it if they are authenticated. +# Others should be able to see it. +olcAccess: {1}to attrs=cn,gecos,givenName,mail,maildrop,displayName,sn + by dn.base="cn=admin,dc=yunohost,dc=org" write + by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write + by self write + by * read +# +# Ensure read access to the base for things like +# supportedSASLMechanisms. Without this you may +# have problems with SASL not knowing what +# mechanisms are available and the like. +# Note that this is covered by the 'access to *' +# ACL below too but if you change that as people +# are wont to do you'll still need this if you +# want SASL (and possible other things) to work +# happily. +olcAccess: {2}to dn.base="" + by * read +# +# The admin dn has full write access, everyone else +# can read everything. +olcAccess: {3}to * + by dn.base="cn=admin,dc=yunohost,dc=org" write + by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write + by group/groupOfNames/member.exact="cn=admin,ou=groups,dc=yunohost,dc=org" write + by * read +# +olcAddContentAcl: FALSE +# Save the time that the entry gets modified, for database #1 +olcLastMod: TRUE +# Where the database file are physically stored for database #1 +olcDbDirectory: /var/lib/ldap +# Checkpoint the BerkeleyDB database periodically in case of system +# failure and to speed slapd shutdown. +olcDbCheckpoint: 512 30 +olcDbNoSync: FALSE +# Indexing options for database #1 +olcDbIndex: objectClass eq +olcDbIndex: entryUUID eq +olcDbIndex: entryCSN eq +olcDbIndex: cn eq +olcDbIndex: uid eq,sub +olcDbIndex: uidNumber eq +olcDbIndex: gidNumber eq +olcDbIndex: sudoUser eq,sub +olcDbIndex: member eq +olcDbIndex: mail eq +olcDbIndex: memberUid eq +olcDbIndex: uniqueMember eq +olcDbIndex: virtualdomain eq +olcDbMaxSize: 10485760 +structuralObjectClass: olcMdbConfig + +# +# Configure Memberof Overlay (used for Yunohost permission) +# + +# Link user <-> group +dn: olcOverlay={0}memberof,olcDatabase={1}mdb,cn=config +objectClass: olcOverlayConfig +objectClass: olcMemberOf +olcOverlay: {0}memberof +olcMemberOfDangling: error +olcMemberOfDanglingError: constraintViolation +olcMemberOfRefInt: TRUE +olcMemberOfGroupOC: groupOfNamesYnh +olcMemberOfMemberAD: member +olcMemberOfMemberOfAD: memberOf +structuralObjectClass: olcMemberOf + +# Link permission <-> groupes +dn: olcOverlay={1}memberof,olcDatabase={1}mdb,cn=config +objectClass: olcOverlayConfig +objectClass: olcMemberOf +olcOverlay: {1}memberof +olcMemberOfDangling: error +olcMemberOfDanglingError: constraintViolation +olcMemberOfRefInt: TRUE +olcMemberOfGroupOC: permissionYnh +olcMemberOfMemberAD: groupPermission +olcMemberOfMemberOfAD: permission +structuralObjectClass: olcMemberOf + +# Link permission <-> user +dn: olcOverlay={2}memberof,olcDatabase={1}mdb,cn=config +objectClass: olcOverlayConfig +objectClass: olcMemberOf +olcOverlay: {2}memberof +olcMemberOfDangling: error +olcMemberOfDanglingError: constraintViolation +olcMemberOfRefInt: TRUE +olcMemberOfGroupOC: permissionYnh +olcMemberOfMemberAD: inheritPermission +olcMemberOfMemberOfAD: permission +structuralObjectClass: olcMemberOf From 9bb8c0437dd6059effe6bc4e698518d1a57dfb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 7 May 2020 00:26:21 +0200 Subject: [PATCH 093/138] Rewrite slapd regen-conf for new config file --- data/hooks/conf_regen/06-slapd | 118 ++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index c8fba52fc..26ff325f5 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -14,19 +14,25 @@ do_init_regen() { systemctl daemon-reload - # fix some permissions - chown root:openldap /etc/ldap/slapd.conf - chown -R openldap:openldap /etc/ldap/schema/ - usermod -aG ssl-cert openldap + # Because slaptest can't test the LDAP config file + # we need to regenerate the new config and after validate it + # regenerate LDAP config directory from slapd.ldif + rm -Rf /etc/ldap/slapd_new.d + mkdir /etc/ldap/slapd_new.d + slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1 # check the slapd config file at first - slaptest -Q -u -f /etc/ldap/slapd.conf + slaptest -Q -u -F /etc/ldap/slapd_new.d - # regenerate LDAP config directory from slapd.conf + # Move to the new config rm -Rf /etc/ldap/slapd.d - mkdir /etc/ldap/slapd.d - slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1 + mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d + + # fix some permissions + chown root:openldap /etc/ldap/slapd.ldif + chown -R openldap:openldap /etc/ldap/schema/ chown -R openldap:openldap /etc/ldap/slapd.d/ + usermod -aG ssl-cert openldap service slapd restart } @@ -34,6 +40,28 @@ do_init_regen() { do_pre_regen() { pending_dir=$1 + # remove temporary backup file + rm -f "$tmp_backup_dir_file" + + # Define if we need to migrate from hdb to mdb + curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}') + if [ -e /etc/ldap/slapd.conf ] && [ -n "$curr_backend" ] && \ + [ $curr_backend != 'mdb' ]; then + backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)" + mkdir -p "$backup_dir" + slapcat -b dc=yunohost,dc=org \ + -l "${backup_dir}/dc=yunohost-dc=org.ldif" + echo "$backup_dir" > "$tmp_backup_dir_file" + fi + + # remove legacy configuration file + [ ! -f /etc/ldap/slapd-yuno.conf ] \ + || touch "${pending_dir}/etc/ldap/slapd-yuno.conf" + [ ! -f /etc/ldap/slapd.conf ] \ + || touch "${pending_dir}/etc/ldap/slapd.conf" + [ ! -f /etc/ldap/schema/yunohost.schema ] \ + || touch "${pending_dir}/etc/ldap/schema/yunohost.schema" + cd /usr/share/yunohost/templates/slapd # create needed directories @@ -41,29 +69,9 @@ do_pre_regen() { schema_dir="${ldap_dir}/schema" mkdir -p "$ldap_dir" "$schema_dir" - # remove legacy configuration file - [ ! -f /etc/ldap/slapd-yuno.conf ] \ - || touch "${pending_dir}/etc/ldap/slapd-yuno.conf" - - # remove temporary backup file - rm -f "$tmp_backup_dir_file" - - # retrieve current and new backends - curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}') - new_backend=$(grep '^database' slapd.conf | awk '{print $2}') - - # save current database before any conf changes - if [[ -n "$curr_backend" && "$curr_backend" != "$new_backend" ]]; then - backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)" - mkdir -p "$backup_dir" - slapcat -b dc=yunohost,dc=org \ - -l "${backup_dir}/dc=yunohost-dc=org.ldif" - echo "$backup_dir" > "$tmp_backup_dir_file" - fi - # copy configuration files - cp -a ldap.conf slapd.conf "$ldap_dir" - cp -a sudo.schema mailserver.schema yunohost.schema "$schema_dir" + cp -a ldap.conf slapd.ldif "$ldap_dir" + cp -a sudo.ldif mailserver.ldif permission.ldif "$schema_dir" mkdir -p ${pending_dir}/etc/systemd/system/slapd.service.d/ cp systemd-override.conf ${pending_dir}/etc/systemd/system/slapd.service.d/ynh-override.conf @@ -74,14 +82,11 @@ do_pre_regen() { do_post_regen() { regen_conf_files=$1 - # ensure that slapd.d exists - mkdir -p /etc/ldap/slapd.d - # fix some permissions echo "Making sure we have the right permissions needed ..." # penldap user should be in the ssl-cert group to let it access the certificate for TLS usermod -aG ssl-cert openldap - chown root:openldap /etc/ldap/slapd.conf + chown root:openldap /etc/ldap/slapd.ldif chown -R openldap:openldap /etc/ldap/schema/ chown -R openldap:openldap /etc/ldap/slapd.d/ @@ -94,29 +99,33 @@ do_post_regen() { [ -z "$regen_conf_files" ] && exit 0 - # check the slapd config file at first - slaptest -Q -u -f /etc/ldap/slapd.conf - # check if a backup should be restored backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true) + + # regenerate LDAP config directory from slapd.conf + echo "Regenerate LDAP config directory from slapd.conf" + + # Because slaptest can't test the LDAP config file + # we need to regenerate the new config and after validate it + # regenerate LDAP config directory from slapd.ldif + rm -Rf /etc/ldap/slapd_new.d + mkdir /etc/ldap/slapd_new.d + slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1 + + # check the slapd config file at first + slaptest -Q -u -F /etc/ldap/slapd_new.d + + # Move to the new config + rm -Rf /etc/ldap/slapd.d + mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d + chown -R openldap:openldap /etc/ldap/slapd.d/ + if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then # regenerate LDAP config directory and import database as root - # since the admin user may be unavailable - echo "Regenerate LDAP config directory and import the database using slapadd" - sh -c "rm -Rf /etc/ldap/slapd.d; - mkdir /etc/ldap/slapd.d; - slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d; - chown -R openldap:openldap /etc/ldap/slapd.d; - slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \ - -l '${backup_dir}/dc=yunohost-dc=org.ldif'; - chown -R openldap:openldap /var/lib/ldap" 2>&1 - else - # regenerate LDAP config directory from slapd.conf - echo "Regenerate LDAP config directory from slapd.conf" - rm -Rf /etc/ldap/slapd.d - mkdir /etc/ldap/slapd.d - slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1 - chown -R openldap:openldap /etc/ldap/slapd.d/ + echo "Import the database using slapadd" + slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \ + -l '${backup_dir}/dc=yunohost-dc=org.ldif' + chown -R openldap:openldap /var/lib/ldap 2>&1 fi echo "Running slapdindex" @@ -156,6 +165,9 @@ case "$1" in init) do_init_regen ;; + apply_config) + do_post_regen /etc/ldap/slapd.ldif + ;; *) echo "hook called with unknown argument \`$1'" >&2 exit 1 From e6a52f09bd6acd3833f0901830fad4e322ab8b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 7 May 2020 00:40:24 +0200 Subject: [PATCH 094/138] [fix] Clean old files --- data/hooks/conf_regen/06-slapd | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 26ff325f5..6759cef1a 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -55,12 +55,18 @@ do_pre_regen() { fi # remove legacy configuration file - [ ! -f /etc/ldap/slapd-yuno.conf ] \ - || touch "${pending_dir}/etc/ldap/slapd-yuno.conf" - [ ! -f /etc/ldap/slapd.conf ] \ - || touch "${pending_dir}/etc/ldap/slapd.conf" - [ ! -f /etc/ldap/schema/yunohost.schema ] \ - || touch "${pending_dir}/etc/ldap/schema/yunohost.schema" + if [ -f /etc/ldap/slapd-yuno.conf ]; then + mkdir -p ${pending_dir}/etc/ldap + touch "${pending_dir}/etc/ldap/slapd-yuno.conf" + fi + if [ -f /etc/ldap/slapd.conf ]; then + mkdir -p ${pending_dir}/etc/ldap + touch "${pending_dir}/etc/ldap/slapd.conf" + fi + if [ -f /etc/ldap/schema/yunohost.schema ]; then + mkdir -p ${pending_dir}/etc/ldap/schema + touch "${pending_dir}/etc/ldap/schema/yunohost.schema" + fi cd /usr/share/yunohost/templates/slapd From c0d3a361e362e83c52fe7d1e1707254b4da26271 Mon Sep 17 00:00:00 2001 From: Josue-T Date: Thu, 7 May 2020 10:48:49 +0200 Subject: [PATCH 095/138] Update data/hooks/conf_regen/06-slapd Co-authored-by: Alexandre Aubin --- data/hooks/conf_regen/06-slapd | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 6759cef1a..38d3adfc5 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -54,27 +54,18 @@ do_pre_regen() { echo "$backup_dir" > "$tmp_backup_dir_file" fi - # remove legacy configuration file - if [ -f /etc/ldap/slapd-yuno.conf ]; then - mkdir -p ${pending_dir}/etc/ldap - touch "${pending_dir}/etc/ldap/slapd-yuno.conf" - fi - if [ -f /etc/ldap/slapd.conf ]; then - mkdir -p ${pending_dir}/etc/ldap - touch "${pending_dir}/etc/ldap/slapd.conf" - fi - if [ -f /etc/ldap/schema/yunohost.schema ]; then - mkdir -p ${pending_dir}/etc/ldap/schema - touch "${pending_dir}/etc/ldap/schema/yunohost.schema" - fi - - cd /usr/share/yunohost/templates/slapd - # create needed directories ldap_dir="${pending_dir}/etc/ldap" schema_dir="${ldap_dir}/schema" mkdir -p "$ldap_dir" "$schema_dir" + # remove legacy configuration file + [ ! -f /etc/ldap/slapd-yuno.conf ] || touch "${ldap_dir}/slapd-yuno.conf" + [ ! -f /etc/ldap/slapd.conf ] || touch "${ldap_dir}/slapd.conf" + [ ! -f /etc/ldap/schema/yunohost.schema ] || touch "${schema_dir}/yunohost.schema" + + cd /usr/share/yunohost/templates/slapd + # copy configuration files cp -a ldap.conf slapd.ldif "$ldap_dir" cp -a sudo.ldif mailserver.ldif permission.ldif "$schema_dir" From 4eb93d62e90953771acc86505e4ff4696b69f7de Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 May 2020 21:43:21 +0200 Subject: [PATCH 096/138] Improve comment about editing slapd.ldif --- data/templates/slapd/slapd.ldif | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/data/templates/slapd/slapd.ldif b/data/templates/slapd/slapd.ldif index c9b6c581f..44dde192e 100644 --- a/data/templates/slapd/slapd.ldif +++ b/data/templates/slapd/slapd.ldif @@ -1,16 +1,21 @@ # OpenLDAP server configuration for Yunohost # ------------------------------------------ # -# By the Yunohost regen-conf tools it's NOT possible to edit the config database by a LDAP request. -# The way to to edit the config database is to edit THIS file -# and after update the config database based on this file. +# Because of the Yunohost's regen-conf mechanism, it is NOT POSSIBLE to +# edit the config database using an LDAP request. +# +# If you wish to edit the config database, you should edit THIS file +# and update the config database based on this file. # # Config database customization: # 1. Edit this file as you want. # 2. Apply your modifications. For this just run this following command in a shell: # $ /usr/share/yunohost/hooks/conf_regen/06-slapd apply_config # -# Note that if you customize this file after all modifications my Yunohost won't be applied. +# Note that if you customize this file, YunoHost's regen-conf will NOT +# overwrite this file. But that also means that you should be careful about +# upgrades, because they may ship important/necessary changes to this +# configuration that you will have to propagate yourself. # # Main configuration From a179e9107a60d8219b3c883eb7275796c4bd9607 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 May 2020 21:44:38 +0200 Subject: [PATCH 097/138] Misc comment improvements? --- data/hooks/conf_regen/06-slapd | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 38d3adfc5..0f465e50d 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -14,21 +14,21 @@ do_init_regen() { systemctl daemon-reload - # Because slaptest can't test the LDAP config file - # we need to regenerate the new config and after validate it - # regenerate LDAP config directory from slapd.ldif + # Validate the new slapd config + # To do so, we have to use the .ldif to generate the config directory + # so we use a temporary directory slapd_new.d rm -Rf /etc/ldap/slapd_new.d mkdir /etc/ldap/slapd_new.d slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1 - - # check the slapd config file at first + # Actual validation (-Q is for quiet, -u is for dry-run) slaptest -Q -u -F /etc/ldap/slapd_new.d - # Move to the new config + # "Commit" / apply the new config (meaning we delete the old one and replace + # it with the new one) rm -Rf /etc/ldap/slapd.d mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d - # fix some permissions + # Enforce permissions chown root:openldap /etc/ldap/slapd.ldif chown -R openldap:openldap /etc/ldap/schema/ chown -R openldap:openldap /etc/ldap/slapd.d/ @@ -50,7 +50,7 @@ do_pre_regen() { backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)" mkdir -p "$backup_dir" slapcat -b dc=yunohost,dc=org \ - -l "${backup_dir}/dc=yunohost-dc=org.ldif" + -l "${backup_dir}/dc=yunohost-dc=org.ldif" echo "$backup_dir" > "$tmp_backup_dir_file" fi @@ -80,7 +80,7 @@ do_post_regen() { regen_conf_files=$1 # fix some permissions - echo "Making sure we have the right permissions needed ..." + echo "Enforce permissions on ldap/slapd directories and certs ..." # penldap user should be in the ssl-cert group to let it access the certificate for TLS usermod -aG ssl-cert openldap chown root:openldap /etc/ldap/slapd.ldif @@ -100,16 +100,15 @@ do_post_regen() { backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true) # regenerate LDAP config directory from slapd.conf - echo "Regenerate LDAP config directory from slapd.conf" + echo "Regenerate LDAP config directory from slapd.ldif" - # Because slaptest can't test the LDAP config file - # we need to regenerate the new config and after validate it - # regenerate LDAP config directory from slapd.ldif + # Validate the new slapd config + # To do so, we have to use the .ldif to generate the config directory + # so we use a temporary directory slapd_new.d rm -Rf /etc/ldap/slapd_new.d mkdir /etc/ldap/slapd_new.d slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1 - - # check the slapd config file at first + # Actual validation (-Q is for quiet, -u is for dry-run) slaptest -Q -u -F /etc/ldap/slapd_new.d # Move to the new config From 2bbff1d26468f3b52c69cbf0e4d377711f157c43 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 May 2020 21:53:21 +0200 Subject: [PATCH 098/138] Factorize the actual slapd regeneration --- data/hooks/conf_regen/06-slapd | 36 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 0f465e50d..8d79335a2 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -13,6 +13,18 @@ do_init_regen() { do_pre_regen "" systemctl daemon-reload + + _regenerate_slapd_conf + + # Enforce permissions + chown root:openldap /etc/ldap/slapd.ldif + chown -R openldap:openldap /etc/ldap/schema/ + usermod -aG ssl-cert openldap + + service slapd restart +} + +_regenerate_slapd_conf() { # Validate the new slapd config # To do so, we have to use the .ldif to generate the config directory @@ -28,13 +40,7 @@ do_init_regen() { rm -Rf /etc/ldap/slapd.d mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d - # Enforce permissions - chown root:openldap /etc/ldap/slapd.ldif - chown -R openldap:openldap /etc/ldap/schema/ chown -R openldap:openldap /etc/ldap/slapd.d/ - usermod -aG ssl-cert openldap - - service slapd restart } do_pre_regen() { @@ -49,8 +55,7 @@ do_pre_regen() { [ $curr_backend != 'mdb' ]; then backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)" mkdir -p "$backup_dir" - slapcat -b dc=yunohost,dc=org \ - -l "${backup_dir}/dc=yunohost-dc=org.ldif" + slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif" echo "$backup_dir" > "$tmp_backup_dir_file" fi @@ -101,20 +106,7 @@ do_post_regen() { # regenerate LDAP config directory from slapd.conf echo "Regenerate LDAP config directory from slapd.ldif" - - # Validate the new slapd config - # To do so, we have to use the .ldif to generate the config directory - # so we use a temporary directory slapd_new.d - rm -Rf /etc/ldap/slapd_new.d - mkdir /etc/ldap/slapd_new.d - slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1 - # Actual validation (-Q is for quiet, -u is for dry-run) - slaptest -Q -u -F /etc/ldap/slapd_new.d - - # Move to the new config - rm -Rf /etc/ldap/slapd.d - mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d - chown -R openldap:openldap /etc/ldap/slapd.d/ + _regenerate_slapd_conf if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then # regenerate LDAP config directory and import database as root From e28c618e677d491b99fc1e00d2fe9e09d4f99e53 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 May 2020 21:54:07 +0200 Subject: [PATCH 099/138] Those single quotes were probably meant to be double quotes? --- data/hooks/conf_regen/06-slapd | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 8d79335a2..9f808b58e 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -101,18 +101,16 @@ do_post_regen() { [ -z "$regen_conf_files" ] && exit 0 - # check if a backup should be restored - backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true) - # regenerate LDAP config directory from slapd.conf echo "Regenerate LDAP config directory from slapd.ldif" _regenerate_slapd_conf + # If there's a backup, re-import its data + backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true) if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then # regenerate LDAP config directory and import database as root echo "Import the database using slapadd" - slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \ - -l '${backup_dir}/dc=yunohost-dc=org.ldif' + slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif" chown -R openldap:openldap /var/lib/ldap 2>&1 fi From 8af56c82efc06843c6c6b5a5f13927ae064c888e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 28 May 2020 21:10:15 +0200 Subject: [PATCH 100/138] slapd.conf is no more used, but ldap.conf need to be saved/restored --- data/hooks/backup/05-conf_ldap | 2 +- data/hooks/restore/05-conf_ldap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index 75b4c2075..b28ea39ca 100755 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -10,7 +10,7 @@ source /usr/share/yunohost/helpers backup_dir="${1}/conf/ldap" # Backup the configuration -ynh_backup "/etc/ldap/slapd.conf" "${backup_dir}/slapd.conf" +ynh_backup "/etc/ldap/ldap.conf" "${backup_dir}/ldap.conf" slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif" # Backup the database diff --git a/data/hooks/restore/05-conf_ldap b/data/hooks/restore/05-conf_ldap index 74093136d..23cf98887 100644 --- a/data/hooks/restore/05-conf_ldap +++ b/data/hooks/restore/05-conf_ldap @@ -39,7 +39,7 @@ else # Restore the configuration mv /etc/ldap/slapd.d "$TMPDIR" mkdir -p /etc/ldap/slapd.d - cp -a "${backup_dir}/slapd.conf" /etc/ldap/slapd.conf + cp -a "${backup_dir}/ldap.conf" /etc/ldap/ldap.conf slapadd -F /etc/ldap/slapd.d -b cn=config \ -l "${backup_dir}/cn=config.master.ldif" \ || die 1 "Unable to restore LDAP configuration" From 7c10ea6b9331a30a25ec7a9a287254f212ad08ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Fri, 29 May 2020 10:59:51 +0200 Subject: [PATCH 101/138] Fix index config --- data/templates/slapd/slapd.ldif | 1 + 1 file changed, 1 insertion(+) diff --git a/data/templates/slapd/slapd.ldif b/data/templates/slapd/slapd.ldif index 44dde192e..d3ed2e053 100644 --- a/data/templates/slapd/slapd.ldif +++ b/data/templates/slapd/slapd.ldif @@ -187,6 +187,7 @@ olcDbIndex: mail eq olcDbIndex: memberUid eq olcDbIndex: uniqueMember eq olcDbIndex: virtualdomain eq +olcDbIndex: permission eq olcDbMaxSize: 10485760 structuralObjectClass: olcMdbConfig From f22f64103abed25a119329f48729d9a4159c86be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Fri, 29 May 2020 13:38:29 +0200 Subject: [PATCH 102/138] Backup/restore also slapd.ldif We need this because the regen-conf need to get the state of the slapd config database if it is customized or not and if some update are need. --- data/hooks/backup/05-conf_ldap | 1 + data/hooks/restore/05-conf_ldap | 3 +++ 2 files changed, 4 insertions(+) diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index b28ea39ca..e3e8e455d 100755 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -11,6 +11,7 @@ backup_dir="${1}/conf/ldap" # Backup the configuration ynh_backup "/etc/ldap/ldap.conf" "${backup_dir}/ldap.conf" +ynh_backup "/etc/ldap/slapd.ldif" "${backup_dir}/slapd.ldif" slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif" # Backup the database diff --git a/data/hooks/restore/05-conf_ldap b/data/hooks/restore/05-conf_ldap index 23cf98887..bdc1ebcdf 100644 --- a/data/hooks/restore/05-conf_ldap +++ b/data/hooks/restore/05-conf_ldap @@ -40,6 +40,9 @@ else mv /etc/ldap/slapd.d "$TMPDIR" mkdir -p /etc/ldap/slapd.d cp -a "${backup_dir}/ldap.conf" /etc/ldap/ldap.conf + cp -a "${backup_dir}/slapd.ldif" /etc/ldap/slapd.ldif + # Legacy thing but we need it to force the regen-conf in case of it exist + cp -a "${backup_dir}/slapd.conf" /etc/ldap/slapd.conf slapadd -F /etc/ldap/slapd.d -b cn=config \ -l "${backup_dir}/cn=config.master.ldif" \ || die 1 "Unable to restore LDAP configuration" From 5a6dca897ba94a2cd777fc6707e7e2667df75243 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jun 2020 03:35:41 +0200 Subject: [PATCH 103/138] Update changelog for 4.0.0-alpha1 --- debian/changelog | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7f1e50190..d87b0b669 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,24 @@ -yunohost (4.0.0~alpha) unstable; urgency=low +yunohost (4.0.0-alpha1) testing; urgency=low - - Tmp version for debian builds during experimental/alpha dev for Buster + - [fix] It just make no sense to backup/restore the mysql password... (#911) + - [fix] Fix getopts and helpers (#885, #886) + - [fix] Explicitly create home using mkhomedir_helper instead of obscure pam rule that doesn't work anymore (b67ff314) + - [fix] Ldap interface seems to expect lists everywhere now? (fb8c2b7b) + - [deb] Clean control file, remove some legacy Conflicts and Replaces (ca0d4933) + - [deb] Add conflicts with versions from backports for critical dependencies (#967) + - [cleanup] Stale / legacy code (217aaa36, d77da6a0, af047468, 82d468a3) + - [conf] Automatically disable/stop systemd-resolved that conflicts with dnsmasq on fresh setups ... (e7214b37) + - [conf] Remove deprecated option in sshd conf, c.f. https://patchwork.openembedded.org/patch/139981/ (2723d245) + - [conf] Small tweak in dovecot conf (deprecated settings) (dc0481e2) + - [conf] Update nslcd and nsswitch stuff using new Buster's default configs + get rid of nslcd service, only keep the regen-conf part (6ef3520f) + - [php] Migrate from php7.0 to php7.3 (3374e653, 9be10506, dd9564d3, 9679c291, 212a15e4, 25fcaa19, c4ad66f5) + - [psql] Migrate from psql 11 to 9.6 (e88aed72, 4920d4f9, c70b0ae4) + - [firewall] Migrate from xtable to nftable (05fb58f2, 2c4a8b73, 625d5372) + - [slapd] Rework slapd regenconf to use new backend (#984) - -- Alexandre Aubin Fri, 7 Feb 2020 21:15:00 +0000 + Thanks to all contributors <3 ! (Étienne M., Josué, Kay0u) + + -- Alexandre Aubin Fri, 05 Jun 2020 03:10:09 +0200 yunohost (3.8.5.5) stable; urgency=low From c5ae774e8d8b6751da8ebae531f4314581b97f17 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jun 2020 04:02:16 +0200 Subject: [PATCH 104/138] Funny jokes with char meanings in deb versions, apparently using a dash makes everything explode suddently >.> --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d87b0b669..5e585731a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -yunohost (4.0.0-alpha1) testing; urgency=low +yunohost (4.0.0~alpha1) testing; urgency=low - [fix] It just make no sense to backup/restore the mysql password... (#911) - [fix] Fix getopts and helpers (#885, #886) From 1f2061c9ee0060b73d66b89b496d49465adb3b71 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jun 2020 17:01:03 +0200 Subject: [PATCH 105/138] Fix version number >.> --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 5e585731a..f8f76c3a8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -yunohost (4.0.0~alpha1) testing; urgency=low +yunohost (4.0.1~alpha) testing; urgency=low - [fix] It just make no sense to backup/restore the mysql password... (#911) - [fix] Fix getopts and helpers (#885, #886) From 509d417070b37b1d090775f423673ae2454505f6 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 Jun 2020 22:50:02 +0200 Subject: [PATCH 106/138] create admin home folder --- src/yunohost/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 75a27aa66..e341c76f1 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -104,6 +104,14 @@ def tools_ldapinit(): # Force nscd to refresh cache to take admin creation into account subprocess.call(['nscd', '-i', 'passwd']) + try: + # Attempt to create user home folder + subprocess.check_call(["mkhomedir_helper", "admin"]) + except subprocess.CalledProcessError: + if not os.path.isdir('/home/{0}'.format("admin")): + logger.warning(m18n.n('user_home_creation_failed'), + exc_info=1) + # Check admin actually exists now try: pwd.getpwnam("admin") From c2422d8c3eadba4ba655be98d57028d34ac7dc74 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 18 Jun 2020 19:14:30 +0200 Subject: [PATCH 107/138] Let's create the home after we validate that admin actually does exists --- src/yunohost/tools.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index e341c76f1..b6d45b03c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -104,14 +104,6 @@ def tools_ldapinit(): # Force nscd to refresh cache to take admin creation into account subprocess.call(['nscd', '-i', 'passwd']) - try: - # Attempt to create user home folder - subprocess.check_call(["mkhomedir_helper", "admin"]) - except subprocess.CalledProcessError: - if not os.path.isdir('/home/{0}'.format("admin")): - logger.warning(m18n.n('user_home_creation_failed'), - exc_info=1) - # Check admin actually exists now try: pwd.getpwnam("admin") @@ -119,6 +111,14 @@ def tools_ldapinit(): logger.error(m18n.n('ldap_init_failed_to_create_admin')) raise YunohostError('installation_failed') + try: + # Attempt to create user home folder + subprocess.check_call(["mkhomedir_helper", "admin"]) + except subprocess.CalledProcessError: + if not os.path.isdir('/home/{0}'.format("admin")): + logger.warning(m18n.n('user_home_creation_failed'), + exc_info=1) + logger.success(m18n.n('ldap_initialized')) From 6a9a7b1cf6c7b6f448ac3116622278ebe8b5d54c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Jun 2020 15:16:54 +0200 Subject: [PATCH 108/138] Update changelog for 4.0.2~beta --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index f8f76c3a8..cc41be2d3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +yunohost (4.0.2~beta) testing; urgency=low + + - [mod] Rebase on stretch-unstable to include recent changes + - [fix] Create admin's home during postinstall (#1021) + + Thanks to all contributors <3 ! (Kay0u) + + -- Alexandre Aubin Fri, 19 Jun 2020 15:16:26 +0200 + yunohost (4.0.1~alpha) testing; urgency=low - [fix] It just make no sense to backup/restore the mysql password... (#911) From b4ced44ad4068c85e67db4d7f507b72aa9dd5c7a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Jul 2020 17:10:41 +0200 Subject: [PATCH 109/138] Update changelog for 4.0.3 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index cc41be2d3..01db3b109 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (4.0.3) stable; urgency=low + + - Bump version number for stable release + + -- Alexandre Aubin Wed, 29 Jul 2020 17:00:00 +0200 + yunohost (4.0.2~beta) testing; urgency=low - [mod] Rebase on stretch-unstable to include recent changes From fed5c2c1e01cc8ae65c3f6291031615d532b1f2a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Jul 2020 18:49:39 +0200 Subject: [PATCH 110/138] Stupid fr wording ~_~ --- locales/fr.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index 4b229d470..72eec2458 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -89,7 +89,7 @@ "mail_domain_unknown": "Le domaine '{domain:s}' de cette adresse de courriel n’est pas valide. Merci d’utiliser un domaine administré par ce serveur.", "mail_forward_remove_failed": "Impossible de supprimer le courriel de transfert '{mail:s}'", "main_domain_change_failed": "Impossible de modifier le domaine principal", - "main_domain_changed": "Le domaine principal modifié", + "main_domain_changed": "Le domaine principal a été modifié", "no_internet_connection": "Le serveur n’est pas connecté à Internet", "not_enough_disk_space": "L’espace disque est insuffisant sur '{path:s}'", "package_unknown": "Le paquet '{pkgname}' est inconnu", @@ -188,7 +188,7 @@ "certmanager_http_check_timeout": "Expiration du délai lorsque le serveur a essayé de se contacter lui-même via HTTP en utilisant l’adresse IP public {ip:s} du domaine {domain:s}. Vous rencontrez peut-être un problème d’hairpinning ou alors le pare-feu/routeur en amont de votre serveur est mal configuré.", "certmanager_couldnt_fetch_intermediate_cert": "Expiration du délai lors de la tentative de récupération du certificat intermédiaire depuis Let’s Encrypt. L’installation ou le renouvellement du certificat a été annulé. Veuillez réessayer plus tard.", "domain_hostname_failed": "Échec de l’utilisation d’un nouveau nom d’hôte. Cela pourrait causer des soucis plus tard (cela n’en causera peut-être pas).", - "yunohost_ca_creation_success": "L’autorité de certification locale créée.", + "yunohost_ca_creation_success": "L'autorité de certification locale a été créée.", "app_already_installed_cant_change_url": "Cette application est déjà installée. L’URL ne peut pas être changé simplement par cette fonction. Vérifiez si cela est disponible avec `app changeurl`.", "app_change_url_failed_nginx_reload": "Le redémarrage de Nginx a échoué. Voici la sortie de 'nginx -t' :\n{nginx_errors:s}", "app_change_url_identical_domains": "L’ancien et le nouveau couple domaine/chemin_de_l’URL sont identiques pour ('{domain:s}{path:s}'), rien à faire.", From 05ff84e5af4228f762e99a8c1c01a6fee33c81ff Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Tue, 4 Aug 2020 18:36:19 +0200 Subject: [PATCH 111/138] Update php --- data/helpers.d/php | 106 ++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/data/helpers.d/php b/data/helpers.d/php index 7ff671317..0fe118fde 100644 --- a/data/helpers.d/php +++ b/data/helpers.d/php @@ -1,37 +1,37 @@ #!/bin/bash readonly YNH_DEFAULT_PHP_VERSION=7.3 -# Declare the actual php version to use. -# A packager willing to use another version of php can override the variable into its _common.sh. +# Declare the actual PHP version to use. +# A packager willing to use another version of PHP can override the variable into its _common.sh. YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} -# Create a dedicated php-fpm config +# Create a dedicated PHP-FPM config # # usage 1: ynh_add_fpm_config [--phpversion=7.X] [--use_template] [--package=packages] [--dedicated_service] -# | arg: -v, --phpversion= - Version of php to use. +# | arg: -v, --phpversion= - Version of PHP to use. # | arg: -t, --use_template - Use this helper in template mode. -# | arg: -p, --package= - Additionnal php packages to install -# | arg: -d, --dedicated_service - Use a dedicated php-fpm service instead of the common one. +# | arg: -p, --package= - Additionnal PHP packages to install +# | arg: -d, --dedicated_service - Use a dedicated PHP-FPM service instead of the common one. # # ----------------------------------------------------------------------------- # # usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service] -# | arg: -v, --phpversion= - Version of php to use. +# | arg: -v, --phpversion= - Version of PHP to use. # | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). -# low - Less than 20Mb of ram by pool. -# medium - Between 20Mb and 40Mb of ram by pool. -# high - More than 40Mb of ram by pool. -# Or specify exactly the footprint, the load of the service as Mb by pool instead of having a standard value. +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. # To have this value, use the following command and stress the service. # watch -n0.5 ps -o user,cmd,%cpu,rss -u APP # # | arg: -u, --usage= - Expected usage of the service (low/medium/high). -# low - Personal usage, behind the sso. +# low - Personal usage, behind the SSO. # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # -# | arg: -p, --package= - Additionnal php packages to install for a specific version of php -# | arg: -d, --dedicated_service - Use a dedicated php-fpm service instead of the common one. +# | arg: -p, --package= - Additionnal PHP packages to install for a specific version of PHP +# | arg: -d, --dedicated_service - Use a dedicated PHP-FPM service instead of the common one. # # # The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. @@ -85,7 +85,7 @@ ynh_add_fpm_config () { # Set the default PHP-FPM version by default phpversion="${phpversion:-$YNH_PHP_VERSION}" - # If the requested php version is not the default version for YunoHost + # If the requested PHP version is not the default version for YunoHost if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] then # If the argument --package is used, add the packages to ynh_install_php to install them from sury @@ -95,7 +95,7 @@ ynh_add_fpm_config () { else local additionnal_packages="" fi - # Install this specific version of php. + # Install this specific version of PHP. ynh_install_php --phpversion="$phpversion" "$additionnal_packages" elif [ -n "$package" ] then @@ -118,7 +118,7 @@ ynh_add_fpm_config () { fpm_service="php5-fpm" fi - # Create the directory for fpm pools + # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" @@ -127,7 +127,7 @@ ynh_add_fpm_config () { ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion finalphpconf="$fpm_config_dir/pool.d/$app.conf" - # Migrate from mutual php service to dedicated one. + # Migrate from mutual PHP service to dedicated one. if [ $dedicated_service -eq 1 ] then local old_fpm_config_dir="/etc/php/$phpversion/fpm" @@ -137,9 +137,9 @@ ynh_add_fpm_config () { ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." # Create a backup of the old file before migration ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" - # Remove the old php config file + # Remove the old PHP config file ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" - # Reload php to release the socket and allow the dedicated service to use it + # Reload PHP to release the socket and allow the dedicated service to use it ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload fi fi @@ -151,10 +151,10 @@ ynh_add_fpm_config () { # Usage 1, use the template in conf/php-fpm.conf local phpfpm_path="../conf/php-fpm.conf" if [ ! -e "$phpfpm_path" ]; then - phpfpm_path="../settings/conf/php-fpm.conf" # Into the restore script, the php-fpm template is not at the same place + phpfpm_path="../settings/conf/php-fpm.conf" # Into the restore script, the PHP-FPM template is not at the same place fi # Make sure now that the template indeed exists - [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure php-fpm." + [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM." cp "$phpfpm_path" "$finalphpconf" ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" @@ -162,13 +162,13 @@ ynh_add_fpm_config () { ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf" else - # Usage 2, generate a php-fpm config file with ynh_get_scalable_phpfpm + # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm # Store settings ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage - # Define the values to use for the configuration of php. + # Define the values to use for the configuration of PHP. ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint # Copy the default file @@ -181,7 +181,7 @@ ynh_add_fpm_config () { ynh_replace_string --match_string="^group = .*" --replace_string="group = $app" --target_file="$finalphpconf" ynh_replace_string --match_string=".*chdir = .*" --replace_string="chdir = $final_path" --target_file="$finalphpconf" - # Configure fpm children + # Configure FPM children ynh_replace_string --match_string=".*pm = .*" --replace_string="pm = $php_pm" --target_file="$finalphpconf" ynh_replace_string --match_string=".*pm.max_children = .*" --replace_string="pm.max_children = $php_max_children" --target_file="$finalphpconf" ynh_replace_string --match_string=".*pm.max_requests = .*" --replace_string="pm.max_requests = 500" --target_file="$finalphpconf" @@ -238,7 +238,7 @@ ynh_add_fpm_config () { ynh_replace_string --match_string="^[; ]*syslog.ident *=.*" --replace_string="syslog.ident = php-fpm-$app" --target_file="$globalphpconf" ynh_replace_string --match_string="^[; ]*include *=.*" --replace_string="include = $finalphpconf" --target_file="$globalphpconf" - # Create a config for a dedicated php-fpm service for the app + # Create a config for a dedicated PHP-FPM service for the app echo "[Unit] Description=PHP $phpversion FastCGI Process Manager for $app After=network.target @@ -253,7 +253,7 @@ ExecReload=/bin/kill -USR2 \$MAINPID WantedBy=multi-user.target " > ../conf/$fpm_service - # Create this dedicated php-fpm service + # Create this dedicated PHP-FPM service ynh_add_systemd_config --service=$fpm_service --template=$fpm_service # Integrate the service in YunoHost admin panel yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --log_type file --description "Php-fpm dedicated to $app" @@ -262,12 +262,12 @@ WantedBy=multi-user.target # Restart the service, as this service is either stopped or only for this app ynh_systemd_action --service_name=$fpm_service --action=restart else - # Reload php, to not impact other parts of the system using php + # Reload PHP, to not impact other parts of the system using PHP ynh_systemd_action --service_name=$fpm_service --action=reload fi } -# Remove the dedicated php-fpm config +# Remove the dedicated PHP-FPM config # # usage: ynh_remove_fpm_config # @@ -277,13 +277,13 @@ ynh_remove_fpm_config () { local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) dedicated_service=${dedicated_service:-0} - # Get the version of php used by this app + # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get $app phpversion) # Assume default PHP-FPM version by default phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" - # Assume default php files if not set + # Assume default PHP files if not set if [ -z "$fpm_config_dir" ] then fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm" @@ -292,11 +292,11 @@ ynh_remove_fpm_config () { if [ $dedicated_service -eq 1 ] then - # Remove the dedicated service php-fpm service for the app + # Remove the dedicated service PHP-FPM service for the app ynh_remove_systemd_config --service=$fpm_service - # Remove the global php-fpm conf + # Remove the global PHP-FPM conf ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" - # Remove the service from the list of services known by Yunohost + # Remove the service from the list of services known by YunoHost yunohost service remove $fpm_service elif ynh_package_is_installed --package="php${phpversion}-fpm"; then ynh_systemd_action --service_name=$fpm_service --action=reload @@ -308,21 +308,21 @@ ynh_remove_fpm_config () { ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" fi - # If the php version used is not the default version for YunoHost + # If the PHP version used is not the default version for YunoHost if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] then - # Remove this specific version of php + # Remove this specific version of PHP ynh_remove_php fi } -# Install another version of php. +# Install another version of PHP. # # [internal] # # usage: ynh_install_php --phpversion=phpversion [--package=packages] -# | arg: -v, --phpversion= - Version of php to install. -# | arg: -p, --package= - Additionnal php packages to install +# | arg: -v, --phpversion= - Version of PHP to install. +# | arg: -p, --package= - Additionnal PHP packages to install # # Requires YunoHost version 3.8.1 or higher. ynh_install_php () { @@ -349,7 +349,7 @@ ynh_install_php () { # Do not add twice the same line if ! grep --quiet "$YNH_APP_INSTANCE_NAME:" "/etc/php/ynh_app_version" then - # Store the ID of this app and the version of php requested for it + # Store the ID of this app and the version of PHP requested for it echo "$YNH_APP_INSTANCE_NAME:$phpversion" | tee --append "/etc/php/ynh_app_version" fi @@ -357,11 +357,11 @@ ynh_install_php () { ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --priority=995 --name=extra_php_version --priority=600 # Install requested dependencies from this extra repository. - # Install php-fpm first, otherwise php will install apache as a dependency. + # Install PHP-FPM first, otherwise PHP will install apache as a dependency. ynh_add_app_dependencies --package="php${phpversion}-fpm" ynh_add_app_dependencies --package="php$phpversion php${phpversion}-common $package" - # Set the default php version back as the default version for php-cli. + # Set the default PHP version back as the default version for php-cli. update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION # Pin this extra repository after packages are installed to prevent sury of doing shit @@ -374,7 +374,7 @@ ynh_install_php () { yunohost service add php${phpversion}-fpm --log "/var/log/php${phpversion}-fpm.log" } -# Remove the specific version of php used by the app. +# Remove the specific version of PHP used by the app. # # [internal] # @@ -382,7 +382,7 @@ ynh_install_php () { # # Requires YunoHost version 3.8.1 or higher. ynh_remove_php () { - # Get the version of php used by this app + # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get $app phpversion) if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] || [ -z "$phpversion" ] @@ -400,7 +400,7 @@ ynh_remove_php () { # Remove the line for this app sed --in-place "/$YNH_APP_INSTANCE_NAME:$phpversion/d" "/etc/php/ynh_app_version" - # If no other app uses this version of php, remove it. + # If no other app uses this version of PHP, remove it. if ! grep --quiet "$phpversion" "/etc/php/ynh_app_version" then # Remove the service from the admin panel @@ -408,26 +408,26 @@ ynh_remove_php () { yunohost service remove php${phpversion}-fpm fi - # Purge php dependencies for this version. + # Purge PHP dependencies for this version. ynh_package_autopurge "php$phpversion php${phpversion}-fpm php${phpversion}-common" fi } -# Define the values to configure php-fpm +# Define the values to configure PHP-FPM # # [internal] # # usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print] # | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). -# low - Less than 20Mb of ram by pool. -# medium - Between 20Mb and 40Mb of ram by pool. -# high - More than 40Mb of ram by pool. -# Or specify exactly the footprint, the load of the service as Mb by pool instead of having a standard value. +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. # To have this value, use the following command and stress the service. # watch -n0.5 ps -o user,cmd,%cpu,rss -u APP # # | arg: -u, --usage= - Expected usage of the service (low/medium/high). -# low - Personal usage, behind the sso. +# low - Personal usage, behind the SSO. # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # @@ -498,7 +498,7 @@ ynh_get_scalable_phpfpm () { # Define pm.max_children # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. - # So if php-fpm start the maximum of children, it won't exceed half of the ram. + # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. php_max_children=$(( $max_ram / 2 / $footprint )) # If process manager is set as static, use half less children. # Used as static, there's always as many children as the value of pm.max_children From 53bc87149a920b3b72df877482e94a067fbcd3ab Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 10 Aug 2020 11:40:53 +0200 Subject: [PATCH 112/138] Update lint.gitlab-ci.yml --- .gitlab/ci/lint.gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index c6967d5a5..a7dbe3aab 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -17,8 +17,9 @@ invalidcode: script: - tox -e invalidcode -# Disabled, waiting for buster -#format-check: -# extends: .lint-stage -# script: -# - black --check --diff +format-check: + stage: lint + image: "before-install" + needs: [] + script: + - black --check --diff From e35d806ae1e89062250fd72b91625ce5880c1989 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 10 Aug 2020 12:04:34 +0200 Subject: [PATCH 113/138] Update lint.gitlab-ci.yml --- .gitlab/ci/lint.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index a7dbe3aab..9a2d1d95f 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -1,6 +1,7 @@ ######################################## # LINTER ######################################## +# later we must fix lint and format-check jobs and remove "allow_failure" lint: stage: lint @@ -21,5 +22,6 @@ format-check: stage: lint image: "before-install" needs: [] + allow_failure: true script: - - black --check --diff + - black --check --diff src doc data tests From accc2da4e3e3b80f8471ba4c57cf0c31da4886da Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Aug 2020 16:55:54 +0200 Subject: [PATCH 114/138] Improve debugging of postgresql 9.6 -> 11 migration --- .../0017_postgresql_9p6_to_11.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py index f03cd9c8c..b0b527a73 100644 --- a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py +++ b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py @@ -28,14 +28,29 @@ class MyMigration(Migration): if not space_used_by_directory("/var/lib/postgresql/9.6") > free_space_in_directory("/var/lib/postgresql"): raise YunohostError("migration_0017_not_enough_space", path="/var/lib/postgresql/") - subprocess.check_call("systemctl stop postgresql", shell=True) - subprocess.check_call("pg_dropcluster --stop 11 main", shell=True) - subprocess.check_call("pg_upgradecluster -m upgrade 9.6 main", shell=True) - subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True) - subprocess.check_call("systemctl start postgresql", shell=True) + self.runcmd("systemctl stop postgresql") + self.runcmd("pg_dropcluster --stop 11 main") + self.runcmd("pg_upgradecluster -m upgrade 9.6 main") + self.runcmd("pg_dropcluster --stop 9.6 main") + self.runcmd("systemctl start postgresql") def package_is_installed(self, package_name): - p = subprocess.Popen("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), shell=True) - p.communicate() - return p.returncode == 0 + (returncode, out, err) = self.runcmd("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), raise_on_errors=False) + return returncode == 0 + + def runcmd(self, cmd, raise_on_errors=True): + p = subprocess.Popen(cmd, + shell=True, + executable='/bin/bash', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + out, err = p.communicate() + returncode = p.returncode + if raise_on_errors and returncode != 0: + raise YunohostError("Failed to run command '{}'.\nreturncode: {}\nstdout:\n{}\nstderr:\n{}\n".format(cmd, returncode, out, err)) + + out = out.strip().split("\n") + return (returncode, out, err) + From 59bd7d6664217968faec74ac339c228e745a1ab3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Aug 2020 17:12:21 +0200 Subject: [PATCH 115/138] Improve robustness of postgresql 9.6 -> 11 migration (in particular dropcluster in case cluster 11 doesn't exists yet) --- .../data_migrations/0017_postgresql_9p6_to_11.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py index b0b527a73..e39cd84ad 100644 --- a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py +++ b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py @@ -25,11 +25,18 @@ class MyMigration(Migration): if not self.package_is_installed("postgresql-11"): raise YunohostError("migration_0017_postgresql_11_not_installed") + # Make sure there's a 9.6 cluster + try: + self.runcmd("pg_lsclusters | grep -q '^9.6 '") + except Exception as e: + logger.warning("It looks like there's not active 9.6 cluster, so probably don't need to run this migration") + return + if not space_used_by_directory("/var/lib/postgresql/9.6") > free_space_in_directory("/var/lib/postgresql"): raise YunohostError("migration_0017_not_enough_space", path="/var/lib/postgresql/") self.runcmd("systemctl stop postgresql") - self.runcmd("pg_dropcluster --stop 11 main") + self.runcmd("pg_dropcluster --stop 11 main || true") # We do not trigger an exception if the command fails because that probably means cluster 11 doesn't exists, which is fine because it's created during the pg_upgradecluster) self.runcmd("pg_upgradecluster -m upgrade 9.6 main") self.runcmd("pg_dropcluster --stop 9.6 main") self.runcmd("systemctl start postgresql") From 4cb6f7fddcda08c9fb5d32b01fbfce1b4980c377 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Aug 2020 17:21:32 +0200 Subject: [PATCH 116/138] Improve debugging for xtables -> nftables migration --- .../data_migrations/0018_xtable_to_nftable.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/yunohost/data_migrations/0018_xtable_to_nftable.py b/src/yunohost/data_migrations/0018_xtable_to_nftable.py index 7d3810518..ed9f7cf89 100644 --- a/src/yunohost/data_migrations/0018_xtable_to_nftable.py +++ b/src/yunohost/data_migrations/0018_xtable_to_nftable.py @@ -36,20 +36,20 @@ class MyMigration(Migration): # Backup existing legacy rules to be able to rollback if self.do_ipv4 and not os.path.exists(self.backup_rules_ipv4): - os.system("iptables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? - subprocess.check_call("iptables-legacy-save > %s" % self.backup_rules_ipv4, shell=True) - assert subprocess.check_output("cat %s" % self.backup_rules_ipv4, shell=True).strip(), "Uhoh backup of legacy ipv4 rules is empty !?" + self.runcmd("iptables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? + self.runcmd("iptables-legacy-save > %s" % self.backup_rules_ipv4) + assert open(self.backup_rules_ipv4).read().strip(), "Uhoh backup of legacy ipv4 rules is empty !?" if self.do_ipv6 and not os.path.exists(self.backup_rules_ipv6): - os.system("ip6tables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? - subprocess.check_call("ip6tables-legacy-save > %s" % self.backup_rules_ipv6, shell=True) - assert subprocess.check_output("cat %s" % self.backup_rules_ipv6, shell=True).strip(), "Uhoh backup of legacy ipv6 rules is empty !?" + self.runcmd("ip6tables-legacy -L >/dev/null") # For some reason if we don't do this, iptables-legacy-save is empty ? + self.runcmd("ip6tables-legacy-save > %s" % self.backup_rules_ipv6) + assert open(self.backup_rules_ipv6).read().strip(), "Uhoh backup of legacy ipv6 rules is empty !?" # We inject the legacy rules (iptables-legacy) into the new iptable (just "iptables") try: if self.do_ipv4: - subprocess.check_call("iptables-legacy-save | iptables-restore", shell=True) + self.runcmd("iptables-legacy-save | iptables-restore") if self.do_ipv6: - subprocess.check_call("ip6tables-legacy-save | ip6tables-restore", shell=True) + self.runcmd("ip6tables-legacy-save | ip6tables-restore") except Exception as e: self.rollback() raise YunohostError("migration_0018_failed_to_migrate_iptables_rules", error=e) @@ -58,19 +58,17 @@ class MyMigration(Migration): # Stolen from https://serverfault.com/a/200642 try: if self.do_ipv4: - subprocess.check_call( + self.runcmd( "iptables-legacy-save | awk '/^[*]/ { print $1 }" # Keep lines like *raw, *filter and *nat " /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }" # Turn all policies to accept " /COMMIT/ { print $0; }'" # Keep the line COMMIT - " | iptables-legacy-restore", - shell=True) + " | iptables-legacy-restore") if self.do_ipv6: - subprocess.check_call( + self.runcmd( "ip6tables-legacy-save | awk '/^[*]/ { print $1 }" # Keep lines like *raw, *filter and *nat " /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }" # Turn all policies to accept " /COMMIT/ { print $0; }'" # Keep the line COMMIT - " | ip6tables-legacy-restore", - shell=True) + " | ip6tables-legacy-restore") except Exception as e: self.rollback() raise YunohostError("migration_0018_failed_to_reset_legacy_rules", error=e) @@ -87,6 +85,22 @@ class MyMigration(Migration): def rollback(self): if self.do_ipv4: - subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv4, shell=True) + self.runcmd("iptables-legacy-restore < %s" % self.backup_rules_ipv4) if self.do_ipv6: - subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv6, shell=True) + self.runcmd("iptables-legacy-restore < %s" % self.backup_rules_ipv6) + + def runcmd(self, cmd, raise_on_errors=True): + p = subprocess.Popen(cmd, + shell=True, + executable='/bin/bash', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + out, err = p.communicate() + returncode = p.returncode + if raise_on_errors and returncode != 0: + raise YunohostError("Failed to run command '{}'.\nreturncode: {}\nstdout:\n{}\nstderr:\n{}\n".format(cmd, returncode, out, err)) + + out = out.strip().split("\n") + return (returncode, out, err) + From 4b14402c5af83f2b342aafcd94d3c05a3bcfcf0d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Aug 2020 18:13:15 +0200 Subject: [PATCH 117/138] Moaaar debug --- src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py | 3 +++ src/yunohost/data_migrations/0018_xtable_to_nftable.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py index e39cd84ad..955393c5b 100644 --- a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py +++ b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py @@ -47,6 +47,9 @@ class MyMigration(Migration): return returncode == 0 def runcmd(self, cmd, raise_on_errors=True): + + logger.debug("Running command: " + cmd) + p = subprocess.Popen(cmd, shell=True, executable='/bin/bash', diff --git a/src/yunohost/data_migrations/0018_xtable_to_nftable.py b/src/yunohost/data_migrations/0018_xtable_to_nftable.py index ed9f7cf89..2f931a6ef 100644 --- a/src/yunohost/data_migrations/0018_xtable_to_nftable.py +++ b/src/yunohost/data_migrations/0018_xtable_to_nftable.py @@ -90,6 +90,9 @@ class MyMigration(Migration): self.runcmd("iptables-legacy-restore < %s" % self.backup_rules_ipv6) def runcmd(self, cmd, raise_on_errors=True): + + logger.debug("Running command: " + cmd) + p = subprocess.Popen(cmd, shell=True, executable='/bin/bash', From 1697943afcbe4586ec1e667ffe586752e59dbe96 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Aug 2020 18:15:32 +0200 Subject: [PATCH 118/138] Update changelog for 4.0.4 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 01db3b109..0fc8ef27b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (4.0.4) stable; urgency=low + + - Debugging and robustness improvements for postgresql 9.6 -> 11 and xtables->nftables migrations (accc2da4, 59bd7d66, 4cb6f7fd, 4b14402c) + + -- Alexandre Aubin Wed, 12 Aug 2020 18:14:00 +0200 + yunohost (4.0.3) stable; urgency=low - Bump version number for stable release From 7b1f02e01f42116a038f0bcbcb221b263fce97ee Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Aug 2020 13:27:07 +0200 Subject: [PATCH 119/138] Fix ref to variable in i18n string (c.f. issue 1647) --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index d40fcfb14..3240cffe7 100644 --- a/locales/en.json +++ b/locales/en.json @@ -122,7 +122,7 @@ "certmanager_cert_signing_failed": "Could not sign the new certificate", "certmanager_certificate_fetching_or_enabling_failed": "Trying to use the new certificate for {domain:s} did not work...", "certmanager_couldnt_fetch_intermediate_cert": "Timed out when trying to fetch intermediate certificate from Let's Encrypt. Certificate installation/renewal aborted—please try again later.", - "certmanager_domain_not_diagnosed_yet": "There is no diagnosis result for domain %s yet. Please re-run a diagnosis for categories 'DNS records' and 'Web' in the diagnosis section to check if the domain is ready for Let's Encrypt. (Or if you know what you are doing, use '--no-checks' to turn off those checks.)", + "certmanager_domain_not_diagnosed_yet": "There is no diagnosis result for domain {domain} yet. Please re-run a diagnosis for categories 'DNS records' and 'Web' in the diagnosis section to check if the domain is ready for Let's Encrypt. (Or if you know what you are doing, use '--no-checks' to turn off those checks.)", "certmanager_domain_cert_not_selfsigned": "The certificate for domain {domain:s} is not self-signed. Are you sure you want to replace it? (Use '--force' to do so.)", "certmanager_domain_dns_ip_differs_from_public_ip": "The DNS records for domain '{domain:s}' is different from this server's IP. Please check the 'DNS records' (basic) category in the diagnosis for more info. If you recently modified your A record, please wait for it to propagate (some DNS propagation checkers are available online). (If you know what you are doing, use '--no-checks' to turn off those checks.)", "certmanager_domain_http_not_working": "Domain {domain:s} does not seem to be accessible through HTTP. Please check the 'Web' category in the diagnosis for more info. (If you know what you are doing, use '--no-checks' to turn off those checks.)", From f3a4334a13aec8e73af7e00f85400eab1db9b3a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Aug 2020 14:50:54 +0200 Subject: [PATCH 120/138] Upgrade nginx configuration according to Mozilla guidelines now that we're on Buster --- data/templates/nginx/security.conf.inc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/data/templates/nginx/security.conf.inc b/data/templates/nginx/security.conf.inc index 4c6461506..4b4f3fe5b 100644 --- a/data/templates/nginx/security.conf.inc +++ b/data/templates/nginx/security.conf.inc @@ -2,15 +2,16 @@ ssl_session_timeout 1d; ssl_session_cache shared:SSL:50m; # about 200000 sessions ssl_session_tickets off; -# nginx 1.10 in stretch doesn't support TLS1.3 and Mozilla doesn't have any -# "modern" config recommendation with it. -# So until buster the modern conf is same as intermediate -{% if compatibility == "modern" %} {% else %} {% endif %} - +{% if compatibility == "modern" %} +# generated 2020-08-14, Mozilla Guideline v5.6, nginx 1.14.2, OpenSSL 1.1.1d, modern configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=modern&openssl=1.1.1d&guideline=5.6 +ssl_protocols TLSv1.3; +ssl_prefer_server_ciphers off; +{% else %} # Ciphers with intermediate compatibility -# generated 2020-04-03, Mozilla Guideline v5.4, nginx 1.10.3, OpenSSL 1.1.0l, intermediate configuration -# https://ssl-config.mozilla.org/#server=nginx&version=1.10.3&config=intermediate&openssl=1.1.0l&guideline=5.4 -ssl_protocols TLSv1.2; +# generated 2020-08-14, Mozilla Guideline v5.6, nginx 1.14.2, OpenSSL 1.1.1d, intermediate configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 +ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; ssl_prefer_server_ciphers off; @@ -18,6 +19,8 @@ ssl_prefer_server_ciphers off; # From https://ssl-config.mozilla.org/ffdhe2048.txt # https://security.stackexchange.com/a/149818 ssl_dhparam /usr/share/yunohost/other/ffdhe2048.pem; +{% endif %} + # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners # https://wiki.mozilla.org/Security/Guidelines/Web_Security From ca0a42f2b42628805aa41bca7095e234c6365a52 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 14 Aug 2020 18:08:51 +0200 Subject: [PATCH 121/138] git clone with depth during CI --- .gitlab/ci/build.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml index 67232ba1f..5e25bcd1f 100644 --- a/.gitlab/ci/build.gitlab-ci.yml +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -38,7 +38,7 @@ build-ssowat: variables: PACKAGE: "ssowat" script: - - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE + - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1 - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE - *build_script @@ -47,6 +47,6 @@ build-moulinette: variables: PACKAGE: "moulinette" script: - - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE + - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1 - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE - *build_script From 485c65a9a5bb59d5d4e657ec15c967324bf0dbe8 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Sat, 15 Aug 2020 11:05:59 +0200 Subject: [PATCH 122/138] apt-daily is now disabled by the CI --- .gitlab/ci/install.gitlab-ci.yml | 8 -------- .gitlab/ci/test.gitlab-ci.yml | 8 -------- 2 files changed, 16 deletions(-) diff --git a/.gitlab/ci/install.gitlab-ci.yml b/.gitlab/ci/install.gitlab-ci.yml index eb5187e11..1df4fc4b9 100644 --- a/.gitlab/ci/install.gitlab-ci.yml +++ b/.gitlab/ci/install.gitlab-ci.yml @@ -26,12 +26,4 @@ install-postinstall: script: - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb - - systemctl -q stop apt-daily.timer - - systemctl -q stop apt-daily-upgrade.timer - - systemctl -q stop apt-daily.service - - systemctl -q stop apt-daily-upgrade.service - - systemctl -q disable apt-daily.timer - - systemctl -q disable apt-daily-upgrade.timer - - systemctl -q disable apt-daily.service - - systemctl -q disable apt-daily-upgrade.service - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 12379342a..3d88354b6 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,12 +1,4 @@ .install_debs: &install_debs - - systemctl -q stop apt-daily.timer - - systemctl -q stop apt-daily-upgrade.timer - - systemctl -q stop apt-daily.service - - systemctl -q stop apt-daily-upgrade.service - - systemctl -q disable apt-daily.timer - - systemctl -q disable apt-daily-upgrade.timer - - systemctl -q disable apt-daily.service - - systemctl -q disable apt-daily-upgrade.service - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb From 0fb5b139242f482467950fe1bb1ea7cdd74d5b94 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 17 Aug 2020 19:51:32 +0200 Subject: [PATCH 123/138] trying to fix the ci --- .gitlab/ci/test.gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 3d88354b6..ef21731f3 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -36,7 +36,7 @@ full-tests: - *install_debs - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns script: - - pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml + - python -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml needs: - job: build-yunohost artifacts: true @@ -51,70 +51,70 @@ full-tests: root-tests: extends: .test-stage script: - - py.test tests + - python -m pytest tests test-apps: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_apps.py + - python -m pytest tests/test_apps.py test-appscatalog: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_appscatalog.py + - python -m pytest tests/test_appscatalog.py test-appurl: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_appurl.py + - python -m pytest tests/test_appurl.py test-apps-arguments-parsing: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_apps_arguments_parsing.py + - python -m pytest tests/test_apps_arguments_parsing.py test-backuprestore: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_backuprestore.py + - python -m pytest tests/test_backuprestore.py test-changeurl: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_changeurl.py + - python -m pytest tests/test_changeurl.py test-permission: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_permission.py + - python -m pytest tests/test_permission.py test-settings: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_settings.py + - python -m pytest tests/test_settings.py test-user-group: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_user-group.py + - python -m pytest tests/test_user-group.py test-regenconf: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_regenconf.py + - python -m pytest tests/test_regenconf.py test-service: extends: .test-stage script: - cd src/yunohost - - py.test tests/test_service.py + - python -m pytest tests/test_service.py From 93bff46bc3a68a30af1e71ab6c75de02dd620062 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 17 Aug 2020 23:54:20 +0200 Subject: [PATCH 124/138] imove tox and add python3 linter --- .gitlab/ci/lint.gitlab-ci.yml | 26 +++++++++++++++++++++----- tox.ini | 25 ++++++------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 9a2d1d95f..8db1ee756 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -3,20 +3,36 @@ ######################################## # later we must fix lint and format-check jobs and remove "allow_failure" -lint: +lint27: stage: lint image: "before-install" needs: [] allow_failure: true script: - - tox -e lint + - tox -e py27-lint -invalidcode: +lint37: + stage: lint + image: "before-install" + needs: [] + allow_failure: true + script: + - tox -e py37-lint + +invalidcode27: stage: lint image: "before-install" needs: [] script: - - tox -e invalidcode + - tox -e py27-invalidcode + +invalidcode37: + stage: lint + image: "before-install" + allow_failure: true + needs: [] + script: + - tox -e py37-invalidcode format-check: stage: lint @@ -24,4 +40,4 @@ format-check: needs: [] allow_failure: true script: - - black --check --diff src doc data tests + - tox -e py37-black diff --git a/tox.ini b/tox.ini index 4598ad3d3..0c0b01a5e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,25 +1,12 @@ [tox] -envlist = - py27 - lint -skipdist = True +envlist = py{27,37}-{lint,invalidcode},py37-black [testenv] skip_install=True deps = - pytest >= 4.6.3, < 5.0 - pyyaml >= 5.1.2, < 6.0 - flake8 >= 3.7.9, < 3.8 - urllib3 + py{27,37}-{lint,invalidcode}: flake8 + py37-black: black commands = - pytest {posargs} - -[testenv:lint] -skip_install=True -commands = flake8 src doc data tests -deps = flake8 - -[testenv:invalidcode] -skip_install=True -commands = flake8 src data --exclude src/yunohost/tests --select F --ignore F401,F841 -deps = flake8 + py{27,37}-lint: flake8 src doc data tests + py{27,37}-invalidcode: flake8 src data --exclude src/yunohost/tests --select F --ignore F401,F841 + py37-black: black --check --diff src doc data tests \ No newline at end of file From 0c9b6eec98ad223a98a1662b6d234c63a89f14e5 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 18 Aug 2020 00:02:08 +0200 Subject: [PATCH 125/138] fix travis --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8674d4d03..9a0f40674 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,18 @@ language: python matrix: allow_failures: - - env: TOXENV=lint + - env: TOXENV=py27-lint + - env: TOXENV=py37-lint + - env: TOXENV=py37-invalidcode include: - python: 2.7 - env: TOXENV=py27 + env: TOXENV=py27-lint - python: 2.7 - env: TOXENV=lint + env: TOXENV=py27-invalidcode + - python: 3.7 + env: TOXENV=py37-lint + - python: 3.7 + env: TOXENV=py37-invalidcode install: - pip install tox From 89bcf1ba6d84d64267c3c37df69e518991c29420 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Aug 2020 13:25:38 +0200 Subject: [PATCH 126/138] Update postfix configuration now that we're on Buster + we have DH param --- data/templates/postfix/main.cf | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index 18e457a76..02b119cdc 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -19,34 +19,35 @@ readme_directory = no # -- TLS for incoming connections ############################################################################### -# generated 2020-04-03, Mozilla Guideline v5.4, Postfix 3.1.14, OpenSSL 1.1.0l, intermediate configuration -# https://ssl-config.mozilla.org/#server=postfix&version=3.1.14&config=intermediate&openssl=1.1.0l&guideline=5.4 - -# (No modern conf support until we're on buster...) -# {% if compatibility == "intermediate" %} {% else %} {% endif %} - smtpd_use_tls = yes smtpd_tls_security_level = may smtpd_tls_auth_only = yes smtpd_tls_cert_file = /etc/yunohost/certs/{{ main_domain }}/crt.pem smtpd_tls_key_file = /etc/yunohost/certs/{{ main_domain }}/key.pem + +{% if compatibility == "intermediate" %} +# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, intermediate configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=intermediate&openssl=1.1.1d&guideline=5.6 + smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 -# smtpd_tls_mandatory_ciphers = medium # (c.f. below) +smtpd_tls_mandatory_ciphers = medium # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam.pem # not actually 1024 bits, this applies to all DHE >= 1024 bits -# smtpd_tls_dh1024_param_file = /path/to/dhparam.pem +smtpd_tls_dh1024_param_file = /usr/share/yunohost/other/ffdhe2048.pem; + +tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +{% else %} +# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, modern configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=modern&openssl=1.1.1d&guideline=5.6 + +smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 +smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 +{% else %} -# This custom medium cipherlist recommendation only works if we have a DH ... which we don't, c.f. https://github.com/YunoHost/issues/issues/93 -# On the other hand, the postfix doc strongly discourage tweaking this list ... So whatever, let's keep the mandatory_ciphers to high like we did before applying the Mozilla recommendation ... -#tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 tls_preempt_cipherlist = no - -# Custom Yunohost stuff ... because we can't use the recommendation about medium cipher list ... -smtpd_tls_mandatory_ciphers=high -smtpd_tls_eecdh_grade = ultra ############################################################################### smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtpd_tls_loglevel=1 From 2d661737b33ae2471ffaf31ebb8d1aeb64698e3e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Aug 2020 13:28:46 +0200 Subject: [PATCH 127/138] Update dovecot configuration now that we have a DH param --- data/templates/dovecot/dovecot.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/data/templates/dovecot/dovecot.conf b/data/templates/dovecot/dovecot.conf index d64b15356..2e17ff163 100644 --- a/data/templates/dovecot/dovecot.conf +++ b/data/templates/dovecot/dovecot.conf @@ -14,14 +14,17 @@ mail_plugins = $mail_plugins quota ############################################################################### -# generated 2020-05-02, Mozilla Guideline v5.4, Dovecot 2.3.4.1, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4.1&config=intermediate&openssl=1.1.1d&guideline=5.4 +# generated 2020-08-18, Mozilla Guideline v5.6, Dovecot 2.3.4, OpenSSL 1.1.1d, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4&config=intermediate&openssl=1.1.1d&guideline=5.6 ssl = required ssl_cert = /path/to/dhparam +ssl_dh = /usr/share/yunohost/other/ffdhe2048.pem; + # intermediate configuration ssl_min_protocol = TLSv1.2 ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 From 8454f2ec12a96bb019d855c6f4256cd38d46baff Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Aug 2020 20:40:55 +0200 Subject: [PATCH 128/138] Recursively enforce ownership for rspamd --- data/hooks/conf_regen/31-rspamd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/31-rspamd b/data/hooks/conf_regen/31-rspamd index 6da4af643..87ed722a7 100755 --- a/data/hooks/conf_regen/31-rspamd +++ b/data/hooks/conf_regen/31-rspamd @@ -42,7 +42,7 @@ do_post_regen() { chown _rspamd /etc/dkim/*.mail.key chmod 400 /etc/dkim/*.mail.key - [ ! -e /var/log/rspamd ] || chown _rspamd:_rspamd /var/log/rspamd + [ ! -e /var/log/rspamd ] || chown -R _rspamd:_rspamd /var/log/rspamd regen_conf_files=$1 [ -z "$regen_conf_files" ] && exit 0 From 6ec0e7b6af442e4c166e6e90060f234f9a7b9e03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Aug 2020 15:41:41 +0200 Subject: [PATCH 129/138] Fix stupid encoding issue when fetching service description --- src/yunohost/service.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 2eeb078bc..522395718 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -346,16 +346,20 @@ def _get_and_format_service_status(service, infos): 'configuration': "unknown", } - translation_key = "service_description_%s" % service + # Try to get description directly from services.yml description = infos.get("description") + + # If no description was there, try to get it from the .json locales if not description: + translation_key = "service_description_%s" % service description = m18n.n(translation_key) - # that mean that we don't have a translation for this string - # that's the only way to test for that for now - # if we don't have it, uses the one provided by systemd - if description.decode('utf-8') == translation_key: - description = str(raw_status.get("Description", "")) + # If descrption is still equal to the translation key, + # that mean that we don't have a translation for this string + # that's the only way to test for that for now + # if we don't have it, uses the one provided by systemd + if description.decode('utf-8') == translation_key: + description = str(raw_status.get("Description", "")) output = { 'status': str(raw_status.get("SubState", "unknown")), From d491b3208b6b60fe90a0e8118e037b567c6df5e8 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 24 Aug 2020 23:25:40 +0200 Subject: [PATCH 130/138] Update acme_tiny to 4.1.0 (#1037) --- src/yunohost/vendor/acme_tiny/acme_tiny.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/yunohost/vendor/acme_tiny/acme_tiny.py b/src/yunohost/vendor/acme_tiny/acme_tiny.py index ba04e37ad..6d1d085c6 100644 --- a/src/yunohost/vendor/acme_tiny/acme_tiny.py +++ b/src/yunohost/vendor/acme_tiny/acme_tiny.py @@ -48,7 +48,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check # helper function - make signed requests def _send_signed_request(url, payload, err_msg, depth=0): - payload64 = _b64(json.dumps(payload).encode('utf8')) + payload64 = "" if payload is None else _b64(json.dumps(payload).encode('utf8')) new_nonce = _do_request(directory['newNonce'])[2]['Replay-Nonce'] protected = {"url": url, "alg": alg, "nonce": new_nonce} protected.update({"jwk": jwk} if acct_headers is None else {"kid": acct_headers['Location']}) @@ -63,12 +63,12 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check # helper function - poll until complete def _poll_until_not(url, pending_statuses, err_msg): - while True: - result, _, _ = _do_request(url, err_msg=err_msg) - if result['status'] in pending_statuses: - time.sleep(2) - continue - return result + result, t0 = None, time.time() + while result is None or result['status'] in pending_statuses: + assert (time.time() - t0 < 3600), "Polling timeout" # 1 hour timeout + time.sleep(0 if result is None else 2) + result, _, _ = _send_signed_request(url, None, err_msg) + return result # parse account key to get public key log.info("Parsing account key...") @@ -93,7 +93,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8')) if common_name is not None: domains.add(common_name.group(1)) - subject_alt_names = re.search(r"X509v3 Subject Alternative Name: \n +([^\n]+)\n", out.decode('utf8'), re.MULTILINE|re.DOTALL) + subject_alt_names = re.search(r"X509v3 Subject Alternative Name: (?:critical)?\n +([^\n]+)\n", out.decode('utf8'), re.MULTILINE|re.DOTALL) if subject_alt_names is not None: for san in subject_alt_names.group(1).split(", "): if san.startswith("DNS:"): @@ -123,7 +123,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check # get the authorizations that need to be completed for auth_url in order['authorizations']: - authorization, _, _ = _do_request(auth_url, err_msg="Error getting challenges") + authorization, _, _ = _send_signed_request(auth_url, None, "Error getting challenges") domain = authorization['identifier']['value'] log.info("Verifying {0}...".format(domain)) @@ -138,9 +138,8 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check # check that the file is in place try: wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token) - assert(disable_check or _do_request(wellknown_url)[0] == keyauthorization) + assert (disable_check or _do_request(wellknown_url)[0] == keyauthorization) except (AssertionError, ValueError) as e: - os.remove(wellknown_path) raise ValueError("Wrote file to {0}, but couldn't download {1}: {2}".format(wellknown_path, wellknown_url, e)) # say the challenge is done @@ -148,6 +147,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check authorization = _poll_until_not(auth_url, ["pending"], "Error checking challenge status for {0}".format(domain)) if authorization['status'] != "valid": raise ValueError("Challenge did not pass for {0}: {1}".format(domain, authorization)) + os.remove(wellknown_path) log.info("{0} verified!".format(domain)) # finalize the order with the csr @@ -161,7 +161,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check raise ValueError("Order failed: {0}".format(order)) # download the certificate - certificate_pem, _, _ = _do_request(order['certificate'], err_msg="Certificate download failed") + certificate_pem, _, _ = _send_signed_request(order['certificate'], None, "Certificate download failed") log.info("Certificate signed!") return certificate_pem From e5e26c2b81a324cdc2bbb65c5c95124a4d7b33b7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Aug 2020 16:46:02 +0200 Subject: [PATCH 131/138] Typo in postfix's conf template --- data/templates/postfix/main.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index 02b119cdc..b4b645138 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -45,7 +45,7 @@ tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA25 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 -{% else %} +{% endif %} tls_preempt_cipherlist = no ############################################################################### From f9c075097c021efeda1f28127d8ff175dc76f7cc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Aug 2020 20:09:38 +0200 Subject: [PATCH 132/138] Update changelog for 4.0.5 --- debian/changelog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0fc8ef27b..92d5b6410 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,16 @@ +yunohost (4.0.5) testing; urgency=low + + - [enh] Update postfix, dovecot, nginx configuration according to Mozilla guidelines (Buster + DH params) (f3a4334a, 89bcf1ba, 2d661737) + - [enh] Update acme_tiny to 4.1.0 (#1037) + - [fix] ref to variable in i18n string (c.f. issue 1647) (7b1f02e0) + - [fix] Recursively enforce ownership for rspamd (8454f2ec) + - [fix] Stupid encoding issue when fetching service description (6ec0e7b6) + - [fix] Misc fixes for CI (ca0a42f2, 485c65a9, #1038, a891d20a) + + Thanks to all contributors <3 ! (Eric G., Kay0u) + + -- Alexandre Aubin Tue, 25 Aug 2020 19:32:27 +0200 + yunohost (4.0.4) stable; urgency=low - Debugging and robustness improvements for postgresql 9.6 -> 11 and xtables->nftables migrations (accc2da4, 59bd7d66, 4cb6f7fd, 4b14402c) From 5f242aea18227004cd7500e1e21562048938e7a1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Aug 2020 17:54:44 +0200 Subject: [PATCH 133/138] Fix code that was half broken because of undefined vars :| --- src/yunohost/app.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index eb30f796b..b7ae4f6ba 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2477,7 +2477,7 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions): or config_panel.json/toml """ from yunohost.domain import domain_list, _get_maindomain - from yunohost.user import user_list + from yunohost.user import user_list, user_info parsed_answers_dict = OrderedDict() @@ -2503,27 +2503,9 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions): question_value = user_answers[question_name] else: if 'ask' in question: - # Retrieve proper ask string - text_for_user_input_in_cli = _value_for_locale(question['ask']) - - # Append extra strings - if question_type == 'boolean': - text_for_user_input_in_cli += ' [yes | no]' - elif question_choices: - text_for_user_input_in_cli += ' [{0}]'.format(' | '.join(question_choices)) - - if question_default is not None: - if question_type == 'boolean': - text_for_user_input_in_cli += ' (default: {0})'.format("yes" if question_default == 1 else "no") - else: - text_for_user_input_in_cli += ' (default: {0})'.format(question_default) - - # Check for a password argument - is_password = True if question_type == 'password' else False if question_type == 'domain': question_default = _get_maindomain() - text_for_user_input_in_cli += ' (default: {0})'.format(question_default) msignals.display(m18n.n('domains_available')) for domain in domain_list()['domains']: msignals.display("- {}".format(domain)) @@ -2537,12 +2519,30 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions): root_mail = "root@%s" % _get_maindomain() for user in users.keys(): if root_mail in user_info(user)["mail-aliases"]: - arg_default = user - ask_string += ' (default: {0})'.format(arg_default) + question_default = user + break elif question_type == 'password': msignals.display(m18n.n('good_practices_about_user_password')) + # Retrieve proper ask string + text_for_user_input_in_cli = _value_for_locale(question['ask']) + + # Append extra strings + if question_type == 'boolean': + text_for_user_input_in_cli += ' [yes | no]' + elif question_choices: + text_for_user_input_in_cli += ' [{0}]'.format(' | '.join(question_choices)) + + + if question_default is not None: + if question_type == 'boolean': + text_for_user_input_in_cli += ' (default: {0})'.format("yes" if question_default == 1 else "no") + else: + text_for_user_input_in_cli += ' (default: {0})'.format(question_default) + + is_password = True if question_type == 'password' else False + try: input_string = msignals.prompt(text_for_user_input_in_cli, is_password) except NotImplementedError: From 77ce657ae1809ef0d62bfedc44799144e141e3ff Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Aug 2020 19:22:09 +0200 Subject: [PATCH 134/138] Ugly hack to have the name of the main logger, otherwise error/exception messages ain't displayed --- src/yunohost/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 810d6127a..ce505ffd9 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -99,6 +99,7 @@ def init_logging(interface="cli", if interface != "api": configure_logging({ 'version': 1, + 'main_logger': "yunohost", 'disable_existing_loggers': True, 'formatters': { 'tty-debug': { From 2ee08189bad784953486a88e9db3b66b912c0b7e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Aug 2020 15:06:28 +0200 Subject: [PATCH 135/138] Don't log helper loading because hmpf --- data/helpers | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/helpers b/data/helpers index a56a6a57a..5ec562f61 100644 --- a/data/helpers +++ b/data/helpers @@ -1,7 +1,8 @@ # -*- shell-script -*- -# TODO : use --regex to validate against a namespace +set +x for helper in $(run-parts --list /usr/share/yunohost/helpers.d 2>/dev/null) ; do [ -r $helper ] && . $helper || true done +set -x From f5acbffb5cb62de21095656b65ba18572e52831c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Aug 2020 17:12:07 +0200 Subject: [PATCH 136/138] Add a diagnosis of processes rencently killed by oom_reaper --- data/hooks/diagnosis/50-systemresources.py | 46 ++++++++++++++++++++++ locales/en.json | 1 + 2 files changed, 47 insertions(+) diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index 682fb897f..6960b9bf0 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -1,6 +1,9 @@ #!/usr/bin/env python import os import psutil +import subprocess +import datetime +import re from yunohost.diagnosis import Diagnoser @@ -96,6 +99,49 @@ class SystemResourcesDiagnoser(Diagnoser): yield item + # + # Recent kills by oom_reaper + # + + kills_count = self.recent_kills_by_oom_reaper() + if kills_count: + kills_summary = "\n".join(["%s (x%s)" % (proc, count) for proc, count in kills_count]) + + yield dict(meta={"test": "oom_reaper"}, + status="WARNING", + summary="diagnosis_processes_killed_by_oom_reaper", + data={"kills_summary": kills_summary}) + + + def recent_kills_by_oom_reaper(self): + if not os.path.exists("/var/log/kern.log"): + return [] + + def analyzed_kern_log(): + + cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process"' + out = subprocess.check_output(cmd, shell=True) + lines = out.strip().split("\n") + + now = datetime.datetime.now() + + for line in reversed(lines): + # Lines look like : + # Aug 25 18:48:21 yolo kernel: [ 9623.613667] oom_reaper: reaped process 11509 (uwsgi), now anon-rss:0kB, file-rss:0kB, shmem-rss:328kB + date_str = str(now.year) + " " + " ".join(line.split()[:3]) + date = datetime.datetime.strptime(date_str, '%Y %b %d %H:%M:%S') + diff = now - date + if diff.days >= 1: + break + process_killed = re.search(r"\(.*\)", line).group().strip("()") + yield process_killed + + processes = list(analyzed_kern_log()) + kills_count = [(p, len([p_ for p_ in processes if p_ == p])) for p in set(processes)] + kills_count = sorted(kills_count, key=lambda p: p[1], reverse=True) + + return kills_count + def human_size(bytes_): # Adapted from https://stackoverflow.com/a/1094933 diff --git a/locales/en.json b/locales/en.json index e180bf828..83cc84442 100644 --- a/locales/en.json +++ b/locales/en.json @@ -262,6 +262,7 @@ "diagnosis_http_nginx_conf_not_up_to_date_details": "To fix the situation, inspect the difference with the command line using yunohost tools regen-conf nginx --dry-run --with-diff and if you're ok, apply the changes with yunohost tools regen-conf nginx --force.", "diagnosis_unknown_categories": "The following categories are unknown: {categories}", "diagnosis_never_ran_yet": "It looks like this server was setup recently and there's no diagnosis report to show yet. You should start by running a full diagnosis, either from the webadmin or using 'yunohost diagnosis run' from the command line.", + "diagnosis_processes_killed_by_oom_reaper": "Some processes were recently killed by the system because it ran out of memory. This is typically symptomatic of a lack of memory on the system or of a process that ate up to much memory. Summary of the processes killed:\n{kills_summary}", "domain_cannot_remove_main": "You cannot remove '{domain:s}' since it's the main domain, you first need to set another domain as the main domain using 'yunohost domain main-domain -n '; here is the list of candidate domains: {other_domains:s}", "domain_cannot_add_xmpp_upload": "You cannot add domains starting with 'xmpp-upload.'. This kind of name is reserved for the XMPP upload feature integrated in YunoHost.", "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain:s}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add ', then set is as the main domain using 'yunohost domain main-domain -n ' and then you can remove the domain '{domain:s}' using 'yunohost domain remove {domain:s}'.'", From f6a25dcbb3ae3d6f179844c08b5a98c79e3d7fd3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Aug 2020 17:27:56 +0200 Subject: [PATCH 137/138] Bump version number to fix the CI --- debian/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian/changelog b/debian/changelog index 92d5b6410..5adf01293 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,7 @@ +yunohost (4.1.0) testing; urgency=low + + - Tmp bump of the version number to fix CI (c.f. Breaks: yunohost(<<4.1) in moulinette) + yunohost (4.0.5) testing; urgency=low - [enh] Update postfix, dovecot, nginx configuration according to Mozilla guidelines (Buster + DH params) (f3a4334a, 89bcf1ba, 2d661737) From 5a4c09d2c7f9167015f5175a2ab8f744b96efaf3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Aug 2020 17:58:16 +0200 Subject: [PATCH 138/138] Yunohost 4.1 requires moulinette 4.1 --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 1f2c24678..cf0138d55 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python:Depends}, ${misc:Depends} - , moulinette (>= 4.0.0~alpha), ssowat (>= 4.0.0~alpha) + , moulinette (>= 4.1), ssowat (>= 4.0) , python-psutil, python-requests, python-dnspython, python-openssl , python-miniupnpc, python-dbus, python-jinja2 , python-toml, python-packaging, python-publicsuffix