From ee67ebfd86bd4dca98e23ae037a71f3b92dba8ff Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 18 Aug 2019 01:40:48 +0200 Subject: [PATCH 1/3] Remove deprecated helpers while still supporting them through hacky patching... --- data/actionsmap/yunohost.yml | 55 +--------- data/helpers.d/network | 25 +++++ locales/en.json | 7 -- src/yunohost/app.py | 191 ++++++++++++++++------------------- src/yunohost/backup.py | 5 +- src/yunohost/tools.py | 19 ---- 6 files changed, 116 insertions(+), 186 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 2b87b6daa..778029775 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -738,30 +738,6 @@ app: help: Delete the key action: store_true - ### app_checkport() - checkport: - action_help: Check availability of a local port - api: GET /tools/checkport - deprecated: true - arguments: - port: - help: Port to check - extra: - pattern: &pattern_port - - !!str ^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$ - - "pattern_port" - - ### app_checkurl() - checkurl: - action_help: Check availability of a web path - api: GET /tools/checkurl - deprecated: True - arguments: - url: - help: Url to check - -a: - full: --app - help: Write domain & path to app settings for further checks ### app_register_url() register-url: @@ -776,24 +752,6 @@ app: help: The path to be registered (e.g. /coffee) - ### app_initdb() - initdb: - action_help: Create database and initialize it with optionnal attached script - api: POST /tools/initdb - deprecated: true - arguments: - user: - help: Name of the DB user - -p: - full: --password - help: Password of the DB (generated unless set) - -d: - full: --db - help: DB name (user unless set) - -s: - full: --sql - help: Initial SQL file - ### app_debug() debug: action_help: Display all debug informations for an application @@ -802,6 +760,7 @@ app: app: help: App name + ### app_makedefault() makedefault: action_help: Redirect domain root to an app @@ -1645,16 +1604,6 @@ tools: help: Show private data (domain, IP) action: store_true - ### tools_port_available() - port-available: - action_help: Check availability of a local port - api: GET /tools/portavailable - arguments: - port: - help: Port to check - extra: - pattern: *pattern_port - ### tools_shell() shell: action_help: Launch a development shell @@ -1863,7 +1812,7 @@ log: api: GET /logs arguments: category: - help: Log category to display (default operations), could be operation, history, package, system, access, service or app + help: Log category to display (default operations), could be operation, history, package, system, access, service or app nargs: "*" -l: full: --limit diff --git a/data/helpers.d/network b/data/helpers.d/network index 0f75cb165..d43c4ec13 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -24,6 +24,31 @@ ynh_find_port () { echo $port } +# Test if a port is available +# +# example: ynh_port_available --port=1234 || ynh_die "Port 1234 is needs to be available for this app" +# +# usage: ynh_find_port --port=XYZ +# | arg: -p, --port - port to check +# +# Requires YunoHost version 3.7.x or higher. +ynh_port_available () { + # Declare an array to define the options of this helper. + local legacy_args=p + declare -Ar args_array=( [p]=port= ) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if netcat -z 127.0.0.1 $port; + then + return 1 + else + return 0 + fi +} + + # Validate an IP address # # usage: ynh_validate_ip --family=family --ip_address=ip_address diff --git a/locales/en.json b/locales/en.json index b45739149..d0071edb9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -22,9 +22,7 @@ "app_id_invalid": "Invalid app id", "app_incompatible": "The app {app} is incompatible with your YunoHost version", "app_install_files_invalid": "Invalid installation files", - "app_location_already_used": "The app '{app}' is already installed on that location ({path})", "app_make_default_location_already_used": "Can't make the app '{app}' the default on the domain {domain} is already used by the other app '{other_app}'", - "app_location_install_failed": "Unable to install the app in this location because it conflit with the app '{other_app}' already installed on '{other_path}'", "app_location_unavailable": "This url is not available or conflicts with the already installed app(s):\n{apps:s}", "app_manifest_invalid": "Invalid app manifest: {error}", "app_no_upgrade": "No apps to upgrade", @@ -393,9 +391,6 @@ "monitor_stats_no_update": "No monitoring statistics to update", "monitor_stats_period_unavailable": "No available statistics for the period", "mountpoint_unknown": "Unknown mountpoint", - "mysql_db_creation_failed": "MySQL database creation failed", - "mysql_db_init_failed": "MySQL database init failed", - "mysql_db_initialized": "The MySQL database has been initialized", "need_define_permission_before": "You need to redefine the permission using 'yunohost user permission add -u USER' before removing an allowed group", "network_check_mx_ko": "DNS MX record is not set", "network_check_smtp_ko": "Outbound mail (SMTP port 25) seems to be blocked by your network", @@ -444,8 +439,6 @@ "permission_update_nothing_to_do": "No permissions to update", "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections", "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", - "port_available": "Port {port:d} is available", - "port_unavailable": "Port {port:d} is not available", "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create $username' or the admin interface.", "remove_main_permission_not_allowed": "Removing the main permission is not allowed", "remove_user_of_group_not_allowed": "You are not allowed to remove the user {user:s} in the group {group:s}", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4831f050c..e403592a2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -41,7 +41,7 @@ from datetime import datetime from moulinette import msignals, m18n, msettings from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json, read_toml +from moulinette.utils.filesystem import read_file, read_json, read_toml, write_to_file from yunohost.service import service_log, service_status, _run_service_command from yunohost.utils import packages @@ -315,6 +315,7 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False): # dirty: we used to have manifest containing multi_instance value in form of a string # but we've switched to bool, this line ensure retrocompatibility + app_info_dict["manifest"]["multi_instance"] = is_true(app_info_dict["manifest"].get("multi_instance", False)) list_dict[app_id] = app_info_dict @@ -667,6 +668,9 @@ def app_upgrade(app=[], url=None, file=None): operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict) operation_logger.start() + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(extracted_app_folder) @@ -854,6 +858,9 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu app_settings['install_time'] = status['installed_at'] _set_app_settings(app_instance_name, app_settings) + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(extracted_app_folder) @@ -996,6 +1003,9 @@ def app_remove(operation_logger, app): except: pass + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(app_setting_path) + # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove # script might date back from jessie install) _patch_php5(app_setting_path) @@ -1197,24 +1207,6 @@ def app_setting(app, key, value=None, delete=False): _set_app_settings(app, app_settings) -def app_checkport(port): - """ - Check availability of a local port - - Keyword argument: - port -- Port to check - - """ - - # This import cannot be moved on top of file because it create a recursive - # import... - from yunohost.tools import tools_port_available - if tools_port_available(port): - logger.success(m18n.n('port_available', port=int(port))) - else: - raise YunohostError('port_unavailable', port=int(port)) - - def app_register_url(app, domain, path): """ Book/register a web path for a given app @@ -1258,93 +1250,6 @@ def app_register_url(app, domain, path): app_setting(app, 'path', value=path) -def app_checkurl(url, app=None): - """ - Check availability of a web path - - Keyword argument: - url -- Url to check - app -- Write domain & path to app settings for further checks - - """ - - logger.error("Packagers /!\\ : 'app checkurl' is deprecated ! Please use the helper 'ynh_webpath_register' instead !") - - from yunohost.domain import domain_list, _normalize_domain_path - - if "https://" == url[:8]: - url = url[8:] - elif "http://" == url[:7]: - url = url[7:] - - if url[-1:] != '/': - url = url + '/' - - domain = url[:url.index('/')] - path = url[url.index('/'):] - installed = False - - domain, path = _normalize_domain_path(domain, path) - - apps_map = app_map(raw=True) - - if domain not in domain_list()['domains']: - raise YunohostError('domain_unknown') - - if domain in apps_map: - # Loop through apps - for p, a in apps_map[domain].items(): - # Skip requested app checking - if app is not None and a['id'] == app: - installed = True - continue - if path == p: - raise YunohostError('app_location_already_used', app=a["id"], path=path) - # can't install "/a/b/" if "/a/" exists - elif path.startswith(p) or p.startswith(path): - raise YunohostError('app_location_install_failed', other_path=p, other_app=a['id']) - - if app is not None and not installed: - app_setting(app, 'domain', value=domain) - app_setting(app, 'path', value=path) - - -def app_initdb(user, password=None, db=None, sql=None): - """ - Create database and initialize it with optionnal attached script - - Keyword argument: - db -- DB name (user unless set) - user -- Name of the DB user - password -- Password of the DB (generated unless set) - sql -- Initial SQL file - - """ - - logger.error("Packagers /!\\ : 'app initdb' is deprecated ! Please use the helper 'ynh_mysql_setup_db' instead !") - - if db is None: - db = user - - return_pwd = False - if password is None: - password = random_password(12) - return_pwd = True - - mysql_root_pwd = open('/etc/yunohost/mysql').read().rstrip() - mysql_command = 'mysql -u root -p%s -e "CREATE DATABASE %s ; GRANT ALL PRIVILEGES ON %s.* TO \'%s\'@localhost IDENTIFIED BY \'%s\';"' % (mysql_root_pwd, db, db, user, password) - if os.system(mysql_command) != 0: - raise YunohostError('mysql_db_creation_failed') - if sql is not None: - if os.system('mysql -u %s -p%s %s < %s' % (user, password, db, sql)) != 0: - raise YunohostError('mysql_db_init_failed') - - if return_pwd: - return password - - logger.success(m18n.n('mysql_db_initialized')) - - def app_ssowatconf(): """ Regenerate SSOwat configuration file @@ -2951,3 +2856,77 @@ def _patch_php5(app_folder): "-e 's@php5@php7.0@g' " \ "%s" % filename os.system(c) + +def _patch_legacy_helpers(app_folder): + + files_to_patch = [] + files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) + + stuff_to_replace = { + # Replace + # sudo yunohost app initdb $db_user -p $db_pwd + # by + # ynh_mysql_setup_db --db_user=$db_user --db_name=$db_user --db_pwd=$db_pwd + "yunohost app initdb": ( + r"(sudo )?yunohost app initdb \"?(\$\{?\w+\}?)\"?\s+-p\s\"?(\$\{?\w+\}?)\"?", + r"ynh_mysql_setup_db --db_user=\2 --db_name=\2 --db_pwd=\3"), + # Replace + # sudo yunohost app checkport whaterver + # by + # ynh_port_available whatever + "yunohost app checkport": ( + r"(sudo )?yunohost app checkport", + r"ynh_port_available"), + # We can't migrate easily port-available + # .. but at the time of writing this code, only two non-working apps are using it. + "yunohost tools port-available": (None, None), + # Replace + # yunohost app checkurl "${domain}${path_url}" -a "${app}" + # by + # ynh_webpath_register --app=${app} --domain=${domain} --path_url=${path_url} + "yunohost app checkurl": ( + r"(sudo )?yunohost app checkurl \"?(\$\{?\w+\}?)\/?(\$\{?\w+\}?)\"?\s+-a\s\"?(\$\{?\w+\}?)\"?", + r"ynh_webpath_register --app=\4 --domain=\2 --path_url=\3"), + } + + stuff_to_replace_compiled = {h: (re.compile(r[0]), r[1]) if r[0] else (None,None) for h, r in stuff_to_replace.items()} + + for filename in files_to_patch: + + # Ignore non-regular files + if not os.path.isfile(filename): + continue + + content = read_file(filename) + replaced_stuff = False + + for helper, regexes in stuff_to_replace_compiled.items(): + pattern, replace = regexes + # If helper is used, attempt to patch the file + if helper in content and pattern != "": + try: + content = pattern.sub(replace, content) + replaced_stuff = True + except Exception as e: + import pdb; pdb.set_trace() + + # If we couldn't patch the deprecated helper, abort the install or whichever step is performed + if helper in content: + raise YunohostError("This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.") + + if replaced_stuff: + + # Check the app do load the helper + # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there... + if filename.split("/")[-1] in ["install", "remove", "upgrade", "backup", "restore"]: + source_helpers = "source /usr/share/yunohost/helpers" + if source_helpers not in content: + content.replace("#!/bin/bash", "#!/bin/bash\n"+source_helpers) + if source_helpers not in content: + content = source_helpers + "\n" + content + + # Actually write the new content in the file + write_to_file(filename, content) + # And complain about those damn deprecated helpers + logger.error("/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ...") diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index bd5d5750d..9c585275f 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -43,7 +43,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir from yunohost.app import ( - app_info, _is_installed, _parse_app_instance_name, _patch_php5 + app_info, _is_installed, _parse_app_instance_name, _patch_php5, _patch_legacy_helpers ) from yunohost.hook import ( hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER @@ -1335,6 +1335,9 @@ class RestoreManager(): app_settings_in_archive = os.path.join(app_dir_in_archive, 'settings') app_scripts_in_archive = os.path.join(app_settings_in_archive, 'scripts') + # Attempt to patch legacy helpers... + _patch_legacy_helpers(app_settings_in_archive) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(app_settings_in_archive) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index c63f1ed33..6eb7e4d62 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -907,25 +907,6 @@ def _check_if_vulnerable_to_meltdown(): return CVEs[0]["VULNERABLE"] -def tools_port_available(port): - """ - Check availability of a local port - - Keyword argument: - port -- Port to check - - """ - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(1) - s.connect(("localhost", int(port))) - s.close() - except socket.error: - return True - else: - return False - - @is_unit_operation() def tools_shutdown(operation_logger, force=False): shutdown = force From 8671a4c23018cb59dce8ae9111967ebc32a57743 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Nov 2019 22:50:47 +0100 Subject: [PATCH 2/3] Remove pdb used for debug, and improve comment --- src/yunohost/app.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e403592a2..cb6ef251f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2905,13 +2905,12 @@ def _patch_legacy_helpers(app_folder): pattern, replace = regexes # If helper is used, attempt to patch the file if helper in content and pattern != "": - try: - content = pattern.sub(replace, content) - replaced_stuff = True - except Exception as e: - import pdb; pdb.set_trace() + content = pattern.sub(replace, content) + replaced_stuff = True - # If we couldn't patch the deprecated helper, abort the install or whichever step is performed + # If the helpert is *still* in the content, it means that we + # couldn't patch the deprecated helper in the previous lines. In + # that case, abort the install or whichever step is performed if helper in content: raise YunohostError("This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.") From a89fd44ab6b4759cc1f0d9900ef678d2f2d4eb38 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Nov 2019 23:04:32 +0100 Subject: [PATCH 3/3] Use ss instead of netcat to check if a port is already used ... c.f. PR #827 --- data/helpers.d/network | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index d43c4ec13..e0f815fc4 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -40,7 +40,7 @@ ynh_port_available () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if netcat -z 127.0.0.1 $port; + if ss -nltu | grep -q -w :$port then return 1 else