Merge pull request #780 from YunoHost/remove-deprecated-helpers

[mod] Remove deprecated helpers while still supporting them through hacky patching

data/actionsmap/yunohost.yml

@@ -681,30 +681,6 @@ app:
                     help: Delete the key
                     action: store_true
 
-        ### app_checkport()
-        checkport:
-            action_help: Check availability of a local port
-            api: GET /tools/checkport
-            deprecated: true
-            arguments:
-                port:
-                    help: Port to check
-                    extra:
-                        pattern: &pattern_port
-                            - !!str ^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$
-                            - "pattern_port"
-
-        ### app_checkurl()
-        checkurl:
-            action_help: Check availability of a web path
-            api: GET /tools/checkurl
-            deprecated: True
-            arguments:
-                url:
-                    help: Url to check
-                -a:
-                    full: --app
-                    help: Write domain & path to app settings for further checks
 
         ### app_register_url()
         register-url:
@@ -719,24 +695,6 @@ app:
                     help: The path to be registered (e.g. /coffee)
 
 
-        ### app_initdb()
-        initdb:
-            action_help: Create database and initialize it with optionnal attached script
-            api: POST /tools/initdb
-            deprecated: true
-            arguments:
-                user:
-                    help: Name of the DB user
-                -p:
-                    full: --password
-                    help: Password of the DB (generated unless set)
-                -d:
-                    full: --db
-                    help: DB name (user unless set)
-                -s:
-                    full: --sql
-                    help: Initial SQL file
-
         ### app_makedefault()
         makedefault:
             action_help: Redirect domain root to an app
@@ -1437,16 +1395,6 @@ tools:
                     help: Upgrade only the system packages
                     action: store_true
 
-        ### tools_port_available()
-        port-available:
-            action_help: Check availability of a local port
-            api: GET /tools/portavailable
-            arguments:
-                port:
-                    help: Port to check
-                    extra:
-                        pattern: *pattern_port
-
         ### tools_shell()
         shell:
             action_help: Launch a development shell
@@ -1656,7 +1604,7 @@ log:
             api: GET /logs
             arguments:
                 category:
-                    help: Log category to display (default operations), could be operation, history, package, system, access, service or app 
+                    help: Log category to display (default operations), could be operation, history, package, system, access, service or app
                     nargs: "*"
                 -l:
                     full: --limit

data/helpers.d/network

@@ -24,6 +24,31 @@ ynh_find_port () {
 	echo $port
 }
 
+# Test if a port is available
+#
+# example: ynh_port_available --port=1234 || ynh_die "Port 1234 is needs to be available for this app"
+#
+# usage: ynh_find_port --port=XYZ
+# | arg: -p, --port - port to check
+#
+# Requires YunoHost version 3.7.x or higher.
+ynh_port_available () {
+	# Declare an array to define the options of this helper.
+	local legacy_args=p
+	declare -Ar args_array=( [p]=port= )
+	local port
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+    if ss -nltu | grep -q -w :$port
+    then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
 # Validate an IP address
 #
 # usage: ynh_validate_ip --family=family --ip_address=ip_address

locales/en.json

@@ -24,9 +24,7 @@
     "app_install_files_invalid": "These files cannot be installed",
     "app_install_failed": "Could not install {app}: {error}",
     "app_install_script_failed": "An error occurred inside the app installation script",
-    "app_location_already_used": "The app '{app}' is already installed in ({path})",
     "app_make_default_location_already_used": "Can't make the app '{app}' the default on the domain, '{domain}' is already in use by the other app '{other_app}'",
-    "app_location_install_failed": "Cannot install the app there because it conflicts with the app '{other_app}' already installed in '{other_path}'",
     "app_location_unavailable": "This URL is either unavailable, or conflicts with the already installed app(s):\n{apps:s}",
     "app_manifest_invalid": "Something is wrong with the app manifest: {error}",
     "app_not_upgraded": "The app '{failed_app}' failed to upgrade, and as a consequence the following apps upgrades have been cancelled: {apps}",
@@ -432,9 +430,6 @@
     "migrations_skip_migration": "Skipping migration {id}…",
     "migrations_success_forward": "Migration {id} completed",
     "migrations_to_be_ran_manually":  "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.",
-    "mysql_db_creation_failed": "Could not create MySQL database",
-    "mysql_db_init_failed": "Could not initialize MySQL database",
-    "mysql_db_initialized": "The MySQL database is now initialized",
     "no_internet_connection": "The server is not connected to the Internet",
     "not_enough_disk_space": "Not enough free space on '{path:s}'",
     "operation_interrupted": "The operation was manually interrupted?",
@@ -476,8 +471,6 @@
     "permission_require_account": "Permission {permission} only makes sense for users having an account, and therefore cannot be enabled for visitors.",
     "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections",
     "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections",
-    "port_available": "Port {port:d} is available",
-    "port_unavailable": "Port {port:d} is not available",
     "recommend_to_add_first_user": "The post-install is finished, but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create <username>' or do it from the admin interface.",
     "regenconf_file_backed_up": "Configuration file '{conf}' backed up to '{backup}'",
     "regenconf_file_copy_failed": "Could not copy the new configuration file '{new}' to '{conf}'",

src/yunohost/app.py

@@ -40,7 +40,7 @@ from datetime import datetime
 from moulinette import msignals, m18n, msettings
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.network import download_json
-from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir
+from moulinette.utils.filesystem import read_file, read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir
 
 from yunohost.service import service_log, service_status, _run_service_command
 from yunohost.utils import packages
@@ -140,6 +140,7 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False):
 
             # dirty: we used to have manifest containing multi_instance value in form of a string
             # but we've switched to bool, this line ensure retrocompatibility
+
             app_info_dict["manifest"]["multi_instance"] = is_true(app_info_dict["manifest"].get("multi_instance", False))
 
             list_dict[app_id] = app_info_dict
@@ -547,6 +548,9 @@ def app_upgrade(app=[], url=None, file=None):
         operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict)
         operation_logger.start()
 
+        # Attempt to patch legacy helpers ...
+        _patch_legacy_helpers(extracted_app_folder)
+
         # Apply dirty patch to make php5 apps compatible with php7
         _patch_php5(extracted_app_folder)
 
@@ -787,6 +791,9 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
     app_settings['install_time'] = status['installed_at']
     _set_app_settings(app_instance_name, app_settings)
 
+    # Attempt to patch legacy helpers ...
+    _patch_legacy_helpers(extracted_app_folder)
+
     # Apply dirty patch to make php5 apps compatible with php7
     _patch_php5(extracted_app_folder)
 
@@ -992,6 +999,9 @@ def app_remove(operation_logger, app):
     except Exception:
         pass
 
+    # Attempt to patch legacy helpers ...
+    _patch_legacy_helpers(app_setting_path)
+
     # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove
     # script might date back from jessie install)
     _patch_php5(app_setting_path)
@@ -1193,24 +1203,6 @@ def app_setting(app, key, value=None, delete=False):
         user_permission_update(app + ".main", remove="all_users", add="visitors")
 
 
-def app_checkport(port):
-    """
-    Check availability of a local port
-
-    Keyword argument:
-        port -- Port to check
-
-    """
-
-    # This import cannot be moved on top of file because it create a recursive
-    # import...
-    from yunohost.tools import tools_port_available
-    if tools_port_available(port):
-        logger.success(m18n.n('port_available', port=int(port)))
-    else:
-        raise YunohostError('port_unavailable', port=int(port))
-
-
 def app_register_url(app, domain, path):
     """
     Book/register a web path for a given app
@@ -1254,93 +1246,6 @@ def app_register_url(app, domain, path):
     app_setting(app, 'path', value=path)
 
 
-def app_checkurl(url, app=None):
-    """
-    Check availability of a web path
-
-    Keyword argument:
-        url -- Url to check
-        app -- Write domain & path to app settings for further checks
-
-    """
-
-    logger.error("Packagers /!\\ : 'app checkurl' is deprecated ! Please use the helper 'ynh_webpath_register' instead !")
-
-    from yunohost.domain import domain_list, _normalize_domain_path
-
-    if "https://" == url[:8]:
-        url = url[8:]
-    elif "http://" == url[:7]:
-        url = url[7:]
-
-    if url[-1:] != '/':
-        url = url + '/'
-
-    domain = url[:url.index('/')]
-    path = url[url.index('/'):]
-    installed = False
-
-    domain, path = _normalize_domain_path(domain, path)
-
-    apps_map = app_map(raw=True)
-
-    if domain not in domain_list()['domains']:
-        raise YunohostError('domain_unknown')
-
-    if domain in apps_map:
-        # Loop through apps
-        for p, a in apps_map[domain].items():
-            # Skip requested app checking
-            if app is not None and a['id'] == app:
-                installed = True
-                continue
-            if path == p:
-                raise YunohostError('app_location_already_used', app=a["id"], path=path)
-            # can't install "/a/b/" if "/a/" exists
-            elif path.startswith(p) or p.startswith(path):
-                raise YunohostError('app_location_install_failed', other_path=p, other_app=a['id'])
-
-    if app is not None and not installed:
-        app_setting(app, 'domain', value=domain)
-        app_setting(app, 'path', value=path)
-
-
-def app_initdb(user, password=None, db=None, sql=None):
-    """
-    Create database and initialize it with optionnal attached script
-
-    Keyword argument:
-        db -- DB name (user unless set)
-        user -- Name of the DB user
-        password -- Password of the DB (generated unless set)
-        sql -- Initial SQL file
-
-    """
-
-    logger.error("Packagers /!\\ : 'app initdb' is deprecated ! Please use the helper 'ynh_mysql_setup_db' instead !")
-
-    if db is None:
-        db = user
-
-    return_pwd = False
-    if password is None:
-        password = random_password(12)
-        return_pwd = True
-
-    mysql_root_pwd = open('/etc/yunohost/mysql').read().rstrip()
-    mysql_command = 'mysql -u root -p%s -e "CREATE DATABASE %s ; GRANT ALL PRIVILEGES ON %s.* TO \'%s\'@localhost IDENTIFIED BY \'%s\';"' % (mysql_root_pwd, db, db, user, password)
-    if os.system(mysql_command) != 0:
-        raise YunohostError('mysql_db_creation_failed')
-    if sql is not None:
-        if os.system('mysql -u %s -p%s %s < %s' % (user, password, db, sql)) != 0:
-            raise YunohostError('mysql_db_init_failed')
-
-    if return_pwd:
-        return password
-
-    logger.success(m18n.n('mysql_db_initialized'))
-
-
 def app_ssowatconf():
     """
     Regenerate SSOwat configuration file
@@ -3006,3 +2911,76 @@ def _patch_php5(app_folder):
             "-e 's@php5@php7.0@g' " \
             "%s" % filename
         os.system(c)
+
+def _patch_legacy_helpers(app_folder):
+
+    files_to_patch = []
+    files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
+
+    stuff_to_replace = {
+        # Replace
+        #    sudo yunohost app initdb $db_user -p $db_pwd
+        # by
+        #    ynh_mysql_setup_db --db_user=$db_user --db_name=$db_user --db_pwd=$db_pwd
+        "yunohost app initdb": (
+            r"(sudo )?yunohost app initdb \"?(\$\{?\w+\}?)\"?\s+-p\s\"?(\$\{?\w+\}?)\"?",
+            r"ynh_mysql_setup_db --db_user=\2 --db_name=\2 --db_pwd=\3"),
+        # Replace
+        #    sudo yunohost app checkport whaterver
+        # by
+        #    ynh_port_available whatever
+        "yunohost app checkport": (
+            r"(sudo )?yunohost app checkport",
+            r"ynh_port_available"),
+        # We can't migrate easily port-available
+        # .. but at the time of writing this code, only two non-working apps are using it.
+        "yunohost tools port-available": (None, None),
+        # Replace
+        #    yunohost app checkurl "${domain}${path_url}" -a "${app}"
+        # by
+        #    ynh_webpath_register --app=${app} --domain=${domain} --path_url=${path_url}
+        "yunohost app checkurl": (
+            r"(sudo )?yunohost app checkurl \"?(\$\{?\w+\}?)\/?(\$\{?\w+\}?)\"?\s+-a\s\"?(\$\{?\w+\}?)\"?",
+            r"ynh_webpath_register --app=\4 --domain=\2 --path_url=\3"),
+    }
+
+    stuff_to_replace_compiled = {h: (re.compile(r[0]), r[1]) if r[0] else (None,None) for h, r in stuff_to_replace.items()}
+
+    for filename in files_to_patch:
+
+        # Ignore non-regular files
+        if not os.path.isfile(filename):
+            continue
+
+        content = read_file(filename)
+        replaced_stuff = False
+
+        for helper, regexes in stuff_to_replace_compiled.items():
+            pattern, replace = regexes
+            # If helper is used, attempt to patch the file
+            if helper in content and pattern != "":
+                content = pattern.sub(replace, content)
+                replaced_stuff = True
+
+            # If the helpert is *still* in the content, it means that we
+            # couldn't patch the deprecated helper in the previous lines.  In
+            # that case, abort the install or whichever step is performed
+            if helper in content:
+                raise YunohostError("This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.")
+
+        if replaced_stuff:
+
+            # Check the app do load the helper
+            # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there...
+            if filename.split("/")[-1] in ["install", "remove", "upgrade", "backup", "restore"]:
+                source_helpers = "source /usr/share/yunohost/helpers"
+                if source_helpers not in content:
+                    content.replace("#!/bin/bash", "#!/bin/bash\n"+source_helpers)
+                if source_helpers not in content:
+                    content = source_helpers + "\n" + content
+
+            # Actually write the new content in the file
+            write_to_file(filename, content)
+            # And complain about those damn deprecated helpers
+            logger.error("/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ...")

src/yunohost/backup.py

@@ -43,7 +43,7 @@ from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
 
 from yunohost.app import (
-    app_info, _is_installed, _parse_app_instance_name, _patch_php5
+    app_info, _is_installed, _parse_app_instance_name, _patch_php5, _patch_legacy_helpers
 )
 from yunohost.hook import (
     hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER
@@ -1321,6 +1321,9 @@ class RestoreManager():
         app_settings_in_archive = os.path.join(app_dir_in_archive, 'settings')
         app_scripts_in_archive = os.path.join(app_settings_in_archive, 'scripts')
 
+        # Attempt to patch legacy helpers...
+        _patch_legacy_helpers(app_settings_in_archive)
+
         # Apply dirty patch to make php5 apps compatible with php7
         _patch_php5(app_settings_in_archive)
 

src/yunohost/tools.py

@@ -670,25 +670,6 @@ def tools_upgrade(operation_logger, apps=None, system=False):
             operation_logger.success()
 
 
-def tools_port_available(port):
-    """
-    Check availability of a local port
-
-    Keyword argument:
-        port -- Port to check
-
-    """
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        s.settimeout(1)
-        s.connect(("localhost", int(port)))
-        s.close()
-    except socket.error:
-        return True
-    else:
-        return False
-
-
 @is_unit_operation()
 def tools_shutdown(operation_logger, force=False):
     shutdown = force