diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 85bc82cf6..3dfc4d878 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -681,30 +681,6 @@ app: help: Delete the key action: store_true - ### app_checkport() - checkport: - action_help: Check availability of a local port - api: GET /tools/checkport - deprecated: true - arguments: - port: - help: Port to check - extra: - pattern: &pattern_port - - !!str ^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$ - - "pattern_port" - - ### app_checkurl() - checkurl: - action_help: Check availability of a web path - api: GET /tools/checkurl - deprecated: True - arguments: - url: - help: Url to check - -a: - full: --app - help: Write domain & path to app settings for further checks ### app_register_url() register-url: @@ -719,24 +695,6 @@ app: help: The path to be registered (e.g. /coffee) - ### app_initdb() - initdb: - action_help: Create database and initialize it with optionnal attached script - api: POST /tools/initdb - deprecated: true - arguments: - user: - help: Name of the DB user - -p: - full: --password - help: Password of the DB (generated unless set) - -d: - full: --db - help: DB name (user unless set) - -s: - full: --sql - help: Initial SQL file - ### app_makedefault() makedefault: action_help: Redirect domain root to an app @@ -1437,16 +1395,6 @@ tools: help: Upgrade only the system packages action: store_true - ### tools_port_available() - port-available: - action_help: Check availability of a local port - api: GET /tools/portavailable - arguments: - port: - help: Port to check - extra: - pattern: *pattern_port - ### tools_shell() shell: action_help: Launch a development shell @@ -1656,7 +1604,7 @@ log: api: GET /logs arguments: category: - help: Log category to display (default operations), could be operation, history, package, system, access, service or app + help: Log category to display (default operations), could be operation, history, package, system, access, service or app nargs: "*" -l: full: --limit diff --git a/data/helpers.d/network b/data/helpers.d/network index 948a327b0..5d20b22ac 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -24,6 +24,31 @@ ynh_find_port () { echo $port } +# Test if a port is available +# +# example: ynh_port_available --port=1234 || ynh_die "Port 1234 is needs to be available for this app" +# +# usage: ynh_find_port --port=XYZ +# | arg: -p, --port - port to check +# +# Requires YunoHost version 3.7.x or higher. +ynh_port_available () { + # Declare an array to define the options of this helper. + local legacy_args=p + declare -Ar args_array=( [p]=port= ) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ss -nltu | grep -q -w :$port + then + return 1 + else + return 0 + fi +} + + # Validate an IP address # # usage: ynh_validate_ip --family=family --ip_address=ip_address diff --git a/locales/en.json b/locales/en.json index 027d9fc0b..fd7f1ac0d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -24,9 +24,7 @@ "app_install_files_invalid": "These files cannot be installed", "app_install_failed": "Could not install {app}: {error}", "app_install_script_failed": "An error occurred inside the app installation script", - "app_location_already_used": "The app '{app}' is already installed in ({path})", "app_make_default_location_already_used": "Can't make the app '{app}' the default on the domain, '{domain}' is already in use by the other app '{other_app}'", - "app_location_install_failed": "Cannot install the app there because it conflicts with the app '{other_app}' already installed in '{other_path}'", "app_location_unavailable": "This URL is either unavailable, or conflicts with the already installed app(s):\n{apps:s}", "app_manifest_invalid": "Something is wrong with the app manifest: {error}", "app_not_upgraded": "The app '{failed_app}' failed to upgrade, and as a consequence the following apps upgrades have been cancelled: {apps}", @@ -432,9 +430,6 @@ "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", - "mysql_db_creation_failed": "Could not create MySQL database", - "mysql_db_init_failed": "Could not initialize MySQL database", - "mysql_db_initialized": "The MySQL database is now initialized", "no_internet_connection": "The server is not connected to the Internet", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", @@ -476,8 +471,6 @@ "permission_require_account": "Permission {permission} only makes sense for users having an account, and therefore cannot be enabled for visitors.", "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections", "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", - "port_available": "Port {port:d} is available", - "port_unavailable": "Port {port:d} is not available", "recommend_to_add_first_user": "The post-install is finished, but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create ' or do it from the admin interface.", "regenconf_file_backed_up": "Configuration file '{conf}' backed up to '{backup}'", "regenconf_file_copy_failed": "Could not copy the new configuration file '{new}' to '{conf}'", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d17ea8424..3a722d39a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -40,7 +40,7 @@ from datetime import datetime from moulinette import msignals, m18n, msettings from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json -from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir +from moulinette.utils.filesystem import read_file, read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir from yunohost.service import service_log, service_status, _run_service_command from yunohost.utils import packages @@ -140,6 +140,7 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False): # dirty: we used to have manifest containing multi_instance value in form of a string # but we've switched to bool, this line ensure retrocompatibility + app_info_dict["manifest"]["multi_instance"] = is_true(app_info_dict["manifest"].get("multi_instance", False)) list_dict[app_id] = app_info_dict @@ -547,6 +548,9 @@ def app_upgrade(app=[], url=None, file=None): operation_logger = OperationLogger('app_upgrade', related_to, env=env_dict) operation_logger.start() + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(extracted_app_folder) @@ -787,6 +791,9 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu app_settings['install_time'] = status['installed_at'] _set_app_settings(app_instance_name, app_settings) + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(extracted_app_folder) @@ -992,6 +999,9 @@ def app_remove(operation_logger, app): except Exception: pass + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(app_setting_path) + # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove # script might date back from jessie install) _patch_php5(app_setting_path) @@ -1193,24 +1203,6 @@ def app_setting(app, key, value=None, delete=False): user_permission_update(app + ".main", remove="all_users", add="visitors") -def app_checkport(port): - """ - Check availability of a local port - - Keyword argument: - port -- Port to check - - """ - - # This import cannot be moved on top of file because it create a recursive - # import... - from yunohost.tools import tools_port_available - if tools_port_available(port): - logger.success(m18n.n('port_available', port=int(port))) - else: - raise YunohostError('port_unavailable', port=int(port)) - - def app_register_url(app, domain, path): """ Book/register a web path for a given app @@ -1254,93 +1246,6 @@ def app_register_url(app, domain, path): app_setting(app, 'path', value=path) -def app_checkurl(url, app=None): - """ - Check availability of a web path - - Keyword argument: - url -- Url to check - app -- Write domain & path to app settings for further checks - - """ - - logger.error("Packagers /!\\ : 'app checkurl' is deprecated ! Please use the helper 'ynh_webpath_register' instead !") - - from yunohost.domain import domain_list, _normalize_domain_path - - if "https://" == url[:8]: - url = url[8:] - elif "http://" == url[:7]: - url = url[7:] - - if url[-1:] != '/': - url = url + '/' - - domain = url[:url.index('/')] - path = url[url.index('/'):] - installed = False - - domain, path = _normalize_domain_path(domain, path) - - apps_map = app_map(raw=True) - - if domain not in domain_list()['domains']: - raise YunohostError('domain_unknown') - - if domain in apps_map: - # Loop through apps - for p, a in apps_map[domain].items(): - # Skip requested app checking - if app is not None and a['id'] == app: - installed = True - continue - if path == p: - raise YunohostError('app_location_already_used', app=a["id"], path=path) - # can't install "/a/b/" if "/a/" exists - elif path.startswith(p) or p.startswith(path): - raise YunohostError('app_location_install_failed', other_path=p, other_app=a['id']) - - if app is not None and not installed: - app_setting(app, 'domain', value=domain) - app_setting(app, 'path', value=path) - - -def app_initdb(user, password=None, db=None, sql=None): - """ - Create database and initialize it with optionnal attached script - - Keyword argument: - db -- DB name (user unless set) - user -- Name of the DB user - password -- Password of the DB (generated unless set) - sql -- Initial SQL file - - """ - - logger.error("Packagers /!\\ : 'app initdb' is deprecated ! Please use the helper 'ynh_mysql_setup_db' instead !") - - if db is None: - db = user - - return_pwd = False - if password is None: - password = random_password(12) - return_pwd = True - - mysql_root_pwd = open('/etc/yunohost/mysql').read().rstrip() - mysql_command = 'mysql -u root -p%s -e "CREATE DATABASE %s ; GRANT ALL PRIVILEGES ON %s.* TO \'%s\'@localhost IDENTIFIED BY \'%s\';"' % (mysql_root_pwd, db, db, user, password) - if os.system(mysql_command) != 0: - raise YunohostError('mysql_db_creation_failed') - if sql is not None: - if os.system('mysql -u %s -p%s %s < %s' % (user, password, db, sql)) != 0: - raise YunohostError('mysql_db_init_failed') - - if return_pwd: - return password - - logger.success(m18n.n('mysql_db_initialized')) - - def app_ssowatconf(): """ Regenerate SSOwat configuration file @@ -3006,3 +2911,76 @@ def _patch_php5(app_folder): "-e 's@php5@php7.0@g' " \ "%s" % filename os.system(c) + +def _patch_legacy_helpers(app_folder): + + files_to_patch = [] + files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) + + stuff_to_replace = { + # Replace + # sudo yunohost app initdb $db_user -p $db_pwd + # by + # ynh_mysql_setup_db --db_user=$db_user --db_name=$db_user --db_pwd=$db_pwd + "yunohost app initdb": ( + r"(sudo )?yunohost app initdb \"?(\$\{?\w+\}?)\"?\s+-p\s\"?(\$\{?\w+\}?)\"?", + r"ynh_mysql_setup_db --db_user=\2 --db_name=\2 --db_pwd=\3"), + # Replace + # sudo yunohost app checkport whaterver + # by + # ynh_port_available whatever + "yunohost app checkport": ( + r"(sudo )?yunohost app checkport", + r"ynh_port_available"), + # We can't migrate easily port-available + # .. but at the time of writing this code, only two non-working apps are using it. + "yunohost tools port-available": (None, None), + # Replace + # yunohost app checkurl "${domain}${path_url}" -a "${app}" + # by + # ynh_webpath_register --app=${app} --domain=${domain} --path_url=${path_url} + "yunohost app checkurl": ( + r"(sudo )?yunohost app checkurl \"?(\$\{?\w+\}?)\/?(\$\{?\w+\}?)\"?\s+-a\s\"?(\$\{?\w+\}?)\"?", + r"ynh_webpath_register --app=\4 --domain=\2 --path_url=\3"), + } + + stuff_to_replace_compiled = {h: (re.compile(r[0]), r[1]) if r[0] else (None,None) for h, r in stuff_to_replace.items()} + + for filename in files_to_patch: + + # Ignore non-regular files + if not os.path.isfile(filename): + continue + + content = read_file(filename) + replaced_stuff = False + + for helper, regexes in stuff_to_replace_compiled.items(): + pattern, replace = regexes + # If helper is used, attempt to patch the file + if helper in content and pattern != "": + content = pattern.sub(replace, content) + replaced_stuff = True + + # If the helpert is *still* in the content, it means that we + # couldn't patch the deprecated helper in the previous lines. In + # that case, abort the install or whichever step is performed + if helper in content: + raise YunohostError("This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.") + + if replaced_stuff: + + # Check the app do load the helper + # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there... + if filename.split("/")[-1] in ["install", "remove", "upgrade", "backup", "restore"]: + source_helpers = "source /usr/share/yunohost/helpers" + if source_helpers not in content: + content.replace("#!/bin/bash", "#!/bin/bash\n"+source_helpers) + if source_helpers not in content: + content = source_helpers + "\n" + content + + # Actually write the new content in the file + write_to_file(filename, content) + # And complain about those damn deprecated helpers + logger.error("/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ...") diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index ddf64774e..885ecea71 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -43,7 +43,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml from yunohost.app import ( - app_info, _is_installed, _parse_app_instance_name, _patch_php5 + app_info, _is_installed, _parse_app_instance_name, _patch_php5, _patch_legacy_helpers ) from yunohost.hook import ( hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER @@ -1321,6 +1321,9 @@ class RestoreManager(): app_settings_in_archive = os.path.join(app_dir_in_archive, 'settings') app_scripts_in_archive = os.path.join(app_settings_in_archive, 'scripts') + # Attempt to patch legacy helpers... + _patch_legacy_helpers(app_settings_in_archive) + # Apply dirty patch to make php5 apps compatible with php7 _patch_php5(app_settings_in_archive) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index bcd39b0e2..2c368bebc 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -670,25 +670,6 @@ def tools_upgrade(operation_logger, apps=None, system=False): operation_logger.success() -def tools_port_available(port): - """ - Check availability of a local port - - Keyword argument: - port -- Port to check - - """ - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(1) - s.connect(("localhost", int(port))) - s.close() - except socket.error: - return True - else: - return False - - @is_unit_operation() def tools_shutdown(operation_logger, force=False): shutdown = force