diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..ed13dfa68 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[report] +omit=src/yunohost/tests/*,src/yunohost/vendor/*,/usr/lib/moulinette/yunohost/* diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index b3aea606f..1aad46fbe 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -36,7 +36,7 @@ full-tests: - *install_debs - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns --force-diskspace script: - - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml + - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ data/hooks/diagnosis/ --junitxml=report.xml - cd tests - bash test_helpers.sh needs: @@ -113,10 +113,10 @@ test-apps: test-appscatalog: extends: .test-stage script: - - python3 -m pytest src/yunohost/tests/test_appscatalog.py + - python3 -m pytest src/yunohost/tests/test_app_catalog.py only: changes: - - src/yunohost/app.py + - src/yunohost/app_calalog.py test-appurl: extends: .test-stage diff --git a/README.md b/README.md index 9fc93740d..df3a4bb9f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@
+![Version](https://img.shields.io/github/v/tag/yunohost/yunohost?label=version&sort=semver) [![Build status](https://shields.io/gitlab/pipeline/yunohost/yunohost/dev)](https://gitlab.com/yunohost/yunohost/-/pipelines) +![Test coverage](https://img.shields.io/gitlab/coverage/yunohost/yunohost/dev) [![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE) [![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost) diff --git a/bin/yunomdns b/bin/yunomdns index 862a1f477..0aee28195 100755 --- a/bin/yunomdns +++ b/bin/yunomdns @@ -4,160 +4,152 @@ Pythonic declaration of mDNS .local domains for YunoHost """ -import subprocess -import re import sys import yaml - -import socket from time import sleep from typing import List, Dict -from zeroconf import Zeroconf, ServiceInfo +import ifaddr +from ipaddress import ip_address +from zeroconf import Zeroconf, ServiceInfo, ServiceBrowser -# Helper command taken from Moulinette -def check_output(args, stderr=subprocess.STDOUT, shell=True, **kwargs): - """Run command with arguments and return its output as a byte string - Overwrite some of the arguments to capture standard error in the result - and use shell by default before calling subprocess.check_output. + +def get_network_local_interfaces() -> Dict[str, Dict[str, List[str]]]: """ - return ( - subprocess.check_output(args, stderr=stderr, shell=shell, **kwargs) - .decode("utf-8") - .strip() - ) - -# Helper command taken from Moulinette -def _extract_inet(string, skip_netmask=False, skip_loopback=True): + Returns interfaces with their associated local IPs """ - Extract IP addresses (v4 and/or v6) from a string limited to one - address by protocol - Keyword argument: - string -- String to search in - skip_netmask -- True to skip subnet mask extraction - skip_loopback -- False to include addresses reserved for the - loopback interface - - Returns: - A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6' - - """ - ip4_pattern = ( - r"((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}" - ) - ip6_pattern = r"(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::?((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)" - ip4_pattern += r"/[0-9]{1,2})" if not skip_netmask else ")" - ip6_pattern += r"/[0-9]{1,3})" if not skip_netmask else ")" - result = {} - - for m in re.finditer(ip4_pattern, string): - addr = m.group(1) - if skip_loopback and addr.startswith("127."): - continue - - # Limit to only one result - result["ipv4"] = addr - break - - for m in re.finditer(ip6_pattern, string): - addr = m.group(1) - if skip_loopback and addr == "::1": - continue - - # Limit to only one result - result["ipv6"] = addr - break - - return result - -# Helper command taken from Moulinette -def get_network_interfaces(): - - # Get network devices and their addresses (raw infos from 'ip addr') - devices_raw = {} - output = check_output("ip --brief a").split("\n") - for line in output: - line = line.split() - iname = line[0] - ips = ' '.join(line[2:]) - - devices_raw[iname] = ips - - # Parse relevant informations for each of them - devices = { - name: _extract_inet(addrs) - for name, addrs in devices_raw.items() - if name != "lo" + interfaces = { + adapter.name: { + "ipv4": [ip.ip for ip in adapter.ips if ip.is_IPv4 and ip_address(ip.ip).is_private], + "ipv6": [ip.ip[0] for ip in adapter.ips if ip.is_IPv6 and ip_address(ip.ip[0]).is_private and not ip_address(ip.ip[0]).is_link_local], + } + for adapter in ifaddr.get_adapters() + if adapter.name != "lo" } + return interfaces - return devices -if __name__ == '__main__': +# Listener class, to detect duplicates on the network +# Stores the list of servers in its list property +class Listener: + def __init__(self): + self.list = [] + + def remove_service(self, zeroconf, type, name): + info = zeroconf.get_service_info(type, name) + self.list.remove(info.server) + + def update_service(self, zeroconf, type, name): + pass + + def add_service(self, zeroconf, type, name): + info = zeroconf.get_service_info(type, name) + self.list.append(info.server[:-1]) + + +def main() -> bool: ### # CONFIG ### - with open('/etc/yunohost/mdns.yml', 'r') as f: + with open("/etc/yunohost/mdns.yml", "r") as f: config = yaml.safe_load(f) or {} - updated = False - required_fields = ["interfaces", "domains"] + required_fields = ["domains"] missing_fields = [field for field in required_fields if field not in config] + interfaces = get_network_local_interfaces() if missing_fields: - print("The fields %s are required" % ', '.join(missing_fields)) + print(f"The fields {missing_fields} are required in mdns.yml") + return False - if config['interfaces'] is None: - print('No interface listed for broadcast.') - sys.exit(0) + if "interfaces" not in config: + config["interfaces"] = [interface + for interface, local_ips in interfaces.items() + if local_ips["ipv4"]] - if 'yunohost.local' not in config['domains']: - config['domains'].append('yunohost.local') + if "ban_interfaces" in config: + config["interfaces"] = [interface + for interface in config["interfaces"] + if interface not in config["ban_interfaces"]] - zcs = {} - interfaces = get_network_interfaces() - for interface in config['interfaces']: - infos = [] # List of ServiceInfo objects, to feed Zeroconf - ips = [] # Human-readable IPs - b_ips = [] # Binary-convered IPs + # Let's discover currently published .local domains accross the network + zc = Zeroconf() + listener = Listener() + browser = ServiceBrowser(zc, "_device-info._tcp.local.", listener) + sleep(2) + browser.cancel() + zc.close() - ipv4 = interfaces[interface]['ipv4'].split('/')[0] - if ipv4: - ips.append(ipv4) - b_ips.append(socket.inet_pton(socket.AF_INET, ipv4)) + # Always attempt to publish yunohost.local + if "yunohost.local" not in config["domains"]: + config["domains"].append("yunohost.local") - ipv6 = interfaces[interface]['ipv6'].split('/')[0] - if ipv6: - ips.append(ipv6) - b_ips.append(socket.inet_pton(socket.AF_INET6, ipv6)) + def find_domain_not_already_published(domain): + + # Try domain.local ... but if it's already published by another entity, + # try domain-2.local, domain-3.local, ... + + i = 1 + domain_i = domain + + while domain_i in listener.list: + print(f"Uh oh, {domain_i} already exists on the network...") + + i += 1 + domain_i = domain.replace(".local", f"-{i}.local") + + return domain_i + + config['domains'] = [find_domain_not_already_published(domain) for domain in config['domains']] + + zcs: Dict[Zeroconf, List[ServiceInfo]] = {} + + for interface in config["interfaces"]: + + if interface not in interfaces: + print(f"Interface {interface} listed in config file is not present on system.") + continue + + # Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7 + # Buster only ships 0.1.6 + # Bullseye ships 0.1.7 + # To be re-enabled once we're on bullseye... + # ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] + ips: List[str] = interfaces[interface]["ipv4"] # If at least one IP is listed - if ips: - # Create a Zeroconf object, and store the ServiceInfos - zc = Zeroconf(interfaces=ips) - zcs[zc]=[] - for d in config['domains']: - d_domain=d.replace('.local','') - if '.' in d_domain: - print(d_domain+'.local: subdomains are not supported.') - else: - # Create a ServiceInfo object for each .local domain - zcs[zc].append(ServiceInfo( - type_='_device-info._tcp.local.', - name=interface+': '+d_domain+'._device-info._tcp.local.', - addresses=b_ips, - port=80, - server=d+'.', - )) - print('Adding '+d+' with addresses '+str(ips)+' on interface '+interface) + if not ips: + continue + + # Create a Zeroconf object, and store the ServiceInfos + zc = Zeroconf(interfaces=ips) # type: ignore + zcs[zc] = [] + + for d in config["domains"]: + d_domain = d.replace(".local", "") + if "." in d_domain: + print(f"{d_domain}.local: subdomains are not supported.") + continue + # Create a ServiceInfo object for each .local domain + zcs[zc].append( + ServiceInfo( + type_="_device-info._tcp.local.", + name=f"{interface}: {d_domain}._device-info._tcp.local.", + parsed_addresses=ips, + port=80, + server=f"{d}.", + ) + ) + print(f"Adding {d} with addresses {ips} on interface {interface}") # Run registration print("Registering...") for zc, infos in zcs.items(): for info in infos: - zc.register_service(info) + zc.register_service(info, allow_name_change=True, cooperating_responders=True) try: print("Registered. Press Ctrl+C or stop service to stop.") @@ -168,6 +160,11 @@ if __name__ == '__main__': finally: print("Unregistering...") for zc, infos in zcs.items(): - for info in infos: - zc.unregister_service(info) + zc.unregister_all_services() zc.close() + + return True + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/data/helpers.d/config b/data/helpers.d/config index 7a2ccde46..d12316996 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -1,6 +1,154 @@ #!/bin/bash +_ynh_app_config_get_one() { + local short_setting="$1" + local type="$2" + local bind="$3" + local getter="get__${short_setting}" + # Get value from getter if exists + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; + then + old[$short_setting]="$($getter)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; + then + old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == "null" ]] + then + old[$short_setting]="YNH_NULL" + + # Get value from app settings or from another file + elif [[ "$type" == "file" ]] + then + if [[ "$bind" == "settings" ]] + then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)" + file_hash[$short_setting]="true" + + # Get multiline text from settings or from a full file + elif [[ "$type" == "text" ]] + then + if [[ "$bind" == "settings" ]] + then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + elif [[ "$bind" == *":"* ]] + then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + else + old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + fi + + # Get value from a kind of key/value file + else + local bind_after="" + if [[ "$bind" == "settings" ]] + then + bind=":/etc/yunohost/apps/$app/settings.yml" + fi + local bind_key="$(echo "$bind" | cut -d: -f1)" + bind_key=${bind_key:-$short_setting} + if [[ "$bind_key" == *">"* ]]; + then + bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" + bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" + fi + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")" + + fi +} +_ynh_app_config_apply_one() { + local short_setting="$1" + local setter="set__${short_setting}" + local bind="${binds[$short_setting]}" + local type="${types[$short_setting]}" + if [ "${changed[$short_setting]}" == "true" ] + then + # Apply setter if exists + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; + then + $setter + + elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; + then + "set__${bind%%(*}" $short_setting $type $bind + + elif [[ "$bind" == "null" ]] + then + continue + + # Save in a file + elif [[ "$type" == "file" ]] + then + if [[ "$bind" == "settings" ]] + then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + if [[ "${!short_setting}" == "" ]] + then + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_secure_remove --file="$bind_file" + ynh_delete_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' removed" + else + ynh_backup_if_checksum_is_different --file="$bind_file" + if [[ "${!short_setting}" != "$bind_file" ]] + then + cp "${!short_setting}" "$bind_file" + fi + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}" + fi + + # Save value in app settings + elif [[ "$bind" == "settings" ]] + then + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$short_setting' edited in app settings" + + # Save multiline text in a file + elif [[ "$type" == "text" ]] + then + if [[ "$bind" == *":"* ]] + then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + fi + local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + ynh_backup_if_checksum_is_different --file="$bind_file" + echo "${!short_setting}" > "$bind_file" + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwrited with the content you provieded in '${short_setting}' question" + + # Set value into a kind of key/value file + else + local bind_after="" + local bind_key="$(echo "$bind" | cut -d: -f1)" + bind_key=${bind_key:-$short_setting} + if [[ "$bind_key" == *">"* ]]; + then + bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" + bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" + fi + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" --after="${bind_after}" + ynh_store_file_checksum --file="$bind_file" --update_only + + # We stored the info in settings in order to be able to upgrade the app + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$bind_key' edited into $bind_file" + + fi + fi +} _ynh_app_config_get() { # From settings local lines @@ -29,62 +177,11 @@ EOL do # Split line into short_setting, type and bind IFS=';' read short_setting type bind <<< "$line" - local getter="get__${short_setting}" binds[${short_setting}]="$bind" types[${short_setting}]="$type" file_hash[${short_setting}]="" formats[${short_setting}]="" - # Get value from getter if exists - if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; - then - old[$short_setting]="$($getter)" - formats[${short_setting}]="yaml" - - elif [[ "$bind" == "null" ]] - then - old[$short_setting]="YNH_NULL" - - # Get value from app settings or from another file - elif [[ "$type" == "file" ]] - then - if [[ "$bind" == "settings" ]] - then - ynh_die --message="File '${short_setting}' can't be stored in settings" - fi - old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)" - file_hash[$short_setting]="true" - - # Get multiline text from settings or from a full file - elif [[ "$type" == "text" ]] - then - if [[ "$bind" == "settings" ]] - then - old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" - elif [[ "$bind" == *":"* ]] - then - ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" - else - old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" - fi - - # Get value from a kind of key/value file - else - local bind_after="" - if [[ "$bind" == "settings" ]] - then - bind=":/etc/yunohost/apps/$app/settings.yml" - fi - local bind_key="$(echo "$bind" | cut -d: -f1)" - bind_key=${bind_key:-$short_setting} - if [[ "$bind_key" == *">"* ]]; - then - bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" - bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" - fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")" - - fi + ynh_app_config_get_one $short_setting $type $bind done @@ -93,85 +190,7 @@ EOL _ynh_app_config_apply() { for short_setting in "${!old[@]}" do - local setter="set__${short_setting}" - local bind="${binds[$short_setting]}" - local type="${types[$short_setting]}" - if [ "${changed[$short_setting]}" == "true" ] - then - # Apply setter if exists - if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; - then - $setter - - elif [[ "$bind" == "null" ]] - then - continue - - # Save in a file - elif [[ "$type" == "file" ]] - then - if [[ "$bind" == "settings" ]] - then - ynh_die --message="File '${short_setting}' can't be stored in settings" - fi - local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - if [[ "${!short_setting}" == "" ]] - then - ynh_backup_if_checksum_is_different --file="$bind_file" - ynh_secure_remove --file="$bind_file" - ynh_delete_file_checksum --file="$bind_file" --update_only - ynh_print_info --message="File '$bind_file' removed" - else - ynh_backup_if_checksum_is_different --file="$bind_file" - if [[ "${!short_setting}" != "$bind_file" ]] - then - cp "${!short_setting}" "$bind_file" - fi - ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}" - fi - - # Save value in app settings - elif [[ "$bind" == "settings" ]] - then - ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" - ynh_print_info --message="Configuration key '$short_setting' edited in app settings" - - # Save multiline text in a file - elif [[ "$type" == "text" ]] - then - if [[ "$bind" == *":"* ]] - then - ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" - fi - local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - ynh_backup_if_checksum_is_different --file="$bind_file" - echo "${!short_setting}" > "$bind_file" - ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info --message="File '$bind_file' overwrited with the content you provieded in '${short_setting}' question" - - # Set value into a kind of key/value file - else - local bind_after="" - local bind_key="$(echo "$bind" | cut -d: -f1)" - bind_key=${bind_key:-$short_setting} - if [[ "$bind_key" == *">"* ]]; - then - bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" - bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" - fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - - ynh_backup_if_checksum_is_different --file="$bind_file" - ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" --after="${bind_after}" - ynh_store_file_checksum --file="$bind_file" --update_only - - # We stored the info in settings in order to be able to upgrade the app - ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" - ynh_print_info --message="Configuration key '$bind_key' edited into $bind_file" - - fi - fi + ynh_app_config_apply_one $short_setting done } @@ -253,6 +272,9 @@ _ynh_app_config_validate() { if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then result="$(validate__$short_setting)" + elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; + then + "validate__${bind%%(*}" $short_setting fi if [ -n "$result" ] then @@ -283,6 +305,10 @@ _ynh_app_config_validate() { } +ynh_app_config_get_one() { + _ynh_app_config_get_one $1 $2 $3 +} + ynh_app_config_get() { _ynh_app_config_get } @@ -295,6 +321,9 @@ ynh_app_config_validate() { _ynh_app_config_validate } +ynh_app_config_apply_one() { + _ynh_app_config_apply_one $1 +} ynh_app_config_apply() { _ynh_app_config_apply } diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index a796b68fd..06b948d00 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -1,6 +1,6 @@ #!/bin/bash -n_version=7.3.0 +n_version=7.5.0 n_install_dir="/opt/node_n" node_version_path="$n_install_dir/n/versions/node" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. @@ -17,7 +17,7 @@ ynh_install_n () { ynh_print_info --message="Installation of N - Node.js version management" # Build an app.src for n echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz -SOURCE_SUM=b908b0fc86922ede37e89d1030191285209d7d521507bf136e62895e5797847f" > "$YNH_APP_BASEDIR/conf/n.src" +SOURCE_SUM=d4da7ea91f680de0c9b5876e097e2a793e8234fcd0f7ca87a0599b925be087a3" > "$YNH_APP_BASEDIR/conf/n.src" # Download and extract n ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n # Install n diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index 6301708d5..c4d05f2fd 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -142,6 +142,7 @@ EOF touch ${pending_dir}/etc/systemd/system/proc-hidepid.service fi + mkdir -p ${pending_dir}/etc/dpkg/origins/ cp dpkg-origins ${pending_dir}/etc/dpkg/origins/yunohost } diff --git a/data/hooks/conf_regen/37-mdns b/data/hooks/conf_regen/37-mdns index a9ae402fb..03a353ae0 100755 --- a/data/hooks/conf_regen/37-mdns +++ b/data/hooks/conf_regen/37-mdns @@ -12,13 +12,6 @@ _generate_config() { [[ "$domain" =~ ^[^.]+\.local$ ]] || continue echo " - $domain" done - - echo "interfaces:" - local_network_interfaces="$(ip --brief a | grep ' 10\.\| 192\.168\.' | awk '{print $1}')" - for interface in $local_network_interfaces - do - echo " - $interface" - done } do_init_regen() { diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 935d4f42d..674337800 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -8,13 +8,16 @@ from publicsuffix2 import PublicSuffixList from moulinette.utils.process import check_output -from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS +from yunohost.utils.dns import ( + dig, + YNH_DYNDNS_DOMAINS, + is_yunohost_dyndns_domain, + is_special_use_tld, +) from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _get_maindomain from yunohost.dns import _build_dns_conf, _get_dns_zone_for_domain -SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] - class DNSRecordsDiagnoser(Diagnoser): @@ -26,23 +29,20 @@ class DNSRecordsDiagnoser(Diagnoser): main_domain = _get_maindomain() - all_domains = domain_list(exclude_subdomains=True)["domains"] - for domain in all_domains: + major_domains = domain_list(exclude_subdomains=True)["domains"] + for domain in major_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) - is_specialusedomain = any( - domain.endswith("." + tld) for tld in SPECIAL_USE_TLDS - ) + for report in self.check_domain( domain, domain == main_domain, - is_specialusedomain=is_specialusedomain, ): yield report # Check if a domain buy by the user will expire soon psl = PublicSuffixList() domains_from_registrar = [ - psl.get_public_suffix(domain) for domain in all_domains + psl.get_public_suffix(domain) for domain in major_domains ] domains_from_registrar = [ domain for domain in domains_from_registrar if "." in domain @@ -53,7 +53,16 @@ class DNSRecordsDiagnoser(Diagnoser): for report in self.check_expiration_date(domains_from_registrar): yield report - def check_domain(self, domain, is_main_domain, is_specialusedomain): + def check_domain(self, domain, is_main_domain): + + if is_special_use_tld(domain): + categories = [] + yield dict( + meta={"domain": domain}, + data={}, + status="INFO", + summary="diagnosis_dns_specialusedomain", + ) base_dns_zone = _get_dns_zone_for_domain(domain) basename = domain.replace(base_dns_zone, "").rstrip(".") or "@" @@ -64,15 +73,6 @@ class DNSRecordsDiagnoser(Diagnoser): categories = ["basic", "mail", "xmpp", "extra"] - if is_specialusedomain: - categories = [] - yield dict( - meta={"domain": domain}, - data={}, - status="INFO", - summary="diagnosis_dns_specialusedomain", - ) - for category in categories: records = expected_configuration[category] @@ -84,7 +84,8 @@ class DNSRecordsDiagnoser(Diagnoser): id_ = r["type"] + ":" + r["name"] fqdn = r["name"] + "." + base_dns_zone if r["name"] != "@" else domain - # Ugly hack to not check mail records for subdomains stuff, otherwise will end up in a shitstorm of errors for people with many subdomains... + # Ugly hack to not check mail records for subdomains stuff, + # otherwise will end up in a shitstorm of errors for people with many subdomains... # Should find a cleaner solution in the suggested conf... if r["type"] in ["MX", "TXT"] and fqdn not in [ domain, @@ -131,6 +132,12 @@ class DNSRecordsDiagnoser(Diagnoser): status = "SUCCESS" summary = "diagnosis_dns_good_conf" + # If status is okay and there's actually no expected records + # (e.g. XMPP disabled) + # then let's not yield any diagnosis line + if not records and "status" == "SUCCESS": + continue + output = dict( meta={"domain": domain, "category": category}, data=results, @@ -140,10 +147,7 @@ class DNSRecordsDiagnoser(Diagnoser): if discrepancies: # For ynh-managed domains (nohost.me etc...), tell people to try to "yunohost dyndns update --force" - if any( - domain.endswith(ynh_dyndns_domain) - for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS - ): + if is_yunohost_dyndns_domain(domain): output["details"] = ["diagnosis_dns_try_dyndns_update_force"] # Otherwise point to the documentation else: diff --git a/data/hooks/diagnosis/21-web.py b/data/hooks/diagnosis/21-web.py index 2072937e5..450296e7e 100644 --- a/data/hooks/diagnosis/21-web.py +++ b/data/hooks/diagnosis/21-web.py @@ -8,6 +8,7 @@ from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list +from yunohost.utils.dns import is_special_use_tld DIAGNOSIS_SERVER = "diagnosis.yunohost.org" @@ -34,11 +35,11 @@ class WebDiagnoser(Diagnoser): summary="diagnosis_http_nginx_conf_not_up_to_date", details=["diagnosis_http_nginx_conf_not_up_to_date_details"], ) - elif domain.endswith(".local"): + elif is_special_use_tld(domain): yield dict( meta={"domain": domain}, status="INFO", - summary="diagnosis_http_localdomain", + summary="diagnosis_http_special_use_tld", ) else: domains_to_check.append(domain) diff --git a/data/templates/mdns/yunomdns.service b/data/templates/mdns/yunomdns.service index ce2641b5d..c1f1b7b06 100644 --- a/data/templates/mdns/yunomdns.service +++ b/data/templates/mdns/yunomdns.service @@ -6,6 +6,7 @@ After=network.target User=mdns Group=mdns Type=simple +Environment=PYTHONUNBUFFERED=1 ExecStart=/usr/bin/yunomdns StandardOutput=syslog diff --git a/debian/changelog b/debian/changelog index 003a9de21..24221543d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,21 @@ yunohost (11.0.0~alpha) unstable; urgency=low -- Alexandre Aubin Fri, 05 Feb 2021 00:02:38 +0100 +yunohost (4.3.1) testing; urgency=low + + - [fix] diagnosis: new app diagnosis grep reporing comments as issues ([#1333](https://github.com/YunoHost/yunohost/pull/1333)) + - [enh] configpanel: Bind function for hotspot (79126809) + - [enh] cli: Rework/improve prompt mecanic ([#1338](https://github.com/YunoHost/yunohost/pull/1338)) + - [fix] dyndns update broke because of buggy dns record names (da1b9089) + - [enh] dns: general improvement for special-use TLD / ynh dyndns domains (17aafe6f) + - [fix] yunomdns: various fixes/improvements ([#1335](https://github.com/YunoHost/yunohost/pull/1335)) + - [fix] certs: Adapt ready_for_ACME check to the new dnsrecord result format... (d75c1a61) + - [i18n] Translations updated for French + + Thanks to all contributors <3 ! (Éric Gaspar, Félix Piédallu, Kayou, ljf, tituspijean) + + -- Alexandre Aubin Wed, 29 Sep 2021 22:22:42 +0200 + yunohost (4.3.0) testing; urgency=low - [users] Import/export users from/to CSV ([#1089](https://github.com/YunoHost/yunohost/pull/1089)) diff --git a/debian/control b/debian/control index a896bb1eb..a06823a16 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python3:Depends}, ${misc:Depends} - , moulinette (>= 4.2), ssowat (>= 4.0) + , moulinette (>= 4.3), ssowat (>= 4.3) , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 diff --git a/locales/en.json b/locales/en.json index 4ed4decca..caf19b44b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -13,10 +13,8 @@ "app_already_installed": "{app} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_up_to_date": "{app} is already up-to-date", - "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'", + "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})", "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", - "app_argument_password_help_keep": "Press Enter to keep the current value", - "app_argument_password_help_optional": "Type one space to empty the password", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_required": "Argument '{name}' is required", "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.", @@ -38,7 +36,6 @@ "app_manifest_install_ask_is_public": "Should this app be exposed to anonymous visitors?", "app_manifest_install_ask_password": "Choose an administration password for this app", "app_manifest_install_ask_path": "Choose the URL path (after the domain) where this app should be installed", - "app_manifest_invalid": "Something is wrong with the app manifest: {error}", "app_not_correctly_installed": "{app} seems to be incorrectly installed", "app_not_installed": "Could not find {app} in the list of installed apps: {all_apps}", "app_not_properly_removed": "{app} has not been properly removed", @@ -194,7 +191,7 @@ "diagnosis_dns_good_conf": "DNS records are correctly configured for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with the following info.
Type: {type}
Name: {name}
Value: {value}", "diagnosis_dns_point_to_doc": "Please check the documentation at https://yunohost.org/dns_config if you need help about configuring DNS records.", - "diagnosis_dns_specialusedomain": "Domain {domain} is based on a special-use top-level domain (TLD) and is therefore not expected to have actual DNS records.", + "diagnosis_dns_specialusedomain": "Domain {domain} is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to have actual DNS records.", "diagnosis_dns_try_dyndns_update_force": "This domain's DNS configuration should automatically be managed by YunoHost. If that's not the case, you can try to force an update using yunohost dyndns update --force.", "diagnosis_domain_expiration_error": "Some domains will expire VERY SOON!", "diagnosis_domain_expiration_not_found": "Unable to check the expiration date for some domains", @@ -203,7 +200,7 @@ "diagnosis_domain_expiration_warning": "Some domains will expire soon!", "diagnosis_domain_expires_in": "{domain} expires in {days} days.", "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired!", - "diagnosis_everything_ok": "Everything looks good for {category}!", + "diagnosis_everything_ok": "Everything looks OK for {category}!", "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}': {error}", "diagnosis_failed_for_category": "Diagnosis failed for category '{category}': {error}", "diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!", @@ -216,7 +213,7 @@ "diagnosis_http_could_not_diagnose_details": "Error: {error}", "diagnosis_http_hairpinning_issue": "Your local network does not seem to have hairpinning enabled.", "diagnosis_http_hairpinning_issue_details": "This is probably because of your ISP box / router. As a result, people from outside your local network will be able to access your server as expected, but not people from inside the local network (like you, probably?) when using the domain name or global IP. You may be able to improve the situation by having a look at https://yunohost.org/dns_local_network", - "diagnosis_http_localdomain": "Domain {domain}, with a .local TLD, is not expected to be exposed outside the local network.", + "diagnosis_http_special_use_tld": "Domain {domain} is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to be exposed outside the local network.", "diagnosis_http_nginx_conf_not_up_to_date": "This domain's nginx configuration appears to have been modified manually, and prevents YunoHost from diagnosing if it's reachable on HTTP.", "diagnosis_http_nginx_conf_not_up_to_date_details": "To fix the situation, inspect the difference with the command line using yunohost tools regen-conf nginx --dry-run --with-diff and if you're ok, apply the changes with yunohost tools regen-conf nginx --force.", "diagnosis_http_ok": "Domain {domain} is reachable through HTTP from outside the local network.", @@ -310,6 +307,7 @@ "domain_deleted": "Domain deleted", "domain_deletion_failed": "Unable to delete domain {domain}: {error}", "domain_dns_conf_is_just_a_recommendation": "This command shows you the *recommended* configuration. It does not actually set up the DNS configuration for you. It is your responsability to configure your DNS zone in your registrar according to this recommendation.", + "domain_dns_conf_special_use_tld": "This domain is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to have actual DNS records.", "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain", "domain_exists": "The domain already exists", @@ -366,7 +364,6 @@ "extracting": "Extracting...", "field_invalid": "Invalid field '{}'", "file_does_not_exist": "The file {path} does not exist.", - "file_extension_not_accepted": "Refusing file '{path}' because its extension is not among the accepted extensions: {accept}", "firewall_reload_failed": "Could not reload the firewall", "firewall_reloaded": "Firewall reloaded", "firewall_rules_cmd_failed": "Some firewall rule commands have failed. More info in log.", @@ -560,6 +557,7 @@ "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations run`.", "not_enough_disk_space": "Not enough free space on '{path}'", "operation_interrupted": "The operation was manually interrupted?", + "other_available_options": "... and {n} other available options not shown", "packages_upgrade_failed": "Could not upgrade all the packages", "password_listed": "This password is among the most used passwords in the world. Please choose something more unique.", "password_too_simple_1": "The password needs to be at least 8 characters long", @@ -675,7 +673,7 @@ "service_stop_failed": "Unable to stop the service '{service}'\n\nRecent service logs:{logs}", "service_stopped": "Service '{service}' stopped", "service_unknown": "Unknown service '{service}'", - "show_tile_cant_be_enabled_for_regex": "You cannot enable 'show_tile' right no, because the URL for the permission '{permission}' is a regex", + "show_tile_cant_be_enabled_for_regex": "You cannot enable 'show_tile' right now, because the URL for the permission '{permission}' is a regex", "show_tile_cant_be_enabled_for_url_not_defined": "You cannot enable 'show_tile' right now, because you must first define an URL for the permission '{permission}'", "ssowat_conf_generated": "SSOwat configuration regenerated", "ssowat_conf_updated": "SSOwat configuration updated", diff --git a/locales/fr.json b/locales/fr.json index 6de6beed6..9d35c7827 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -431,7 +431,7 @@ "diagnosis_cache_still_valid": "(Le cache est encore valide pour le diagnostic {category}. Il ne sera pas re-diagnostiqué pour le moment !)", "diagnosis_ignored_issues": "(+ {nb_ignored} problème(s) ignoré(s))", "diagnosis_found_warnings": "Trouvé {warnings} objet(s) pouvant être amélioré(s) pour {category}.", - "diagnosis_everything_ok": "Tout semble bien pour {category} !", + "diagnosis_everything_ok": "Tout semble OK pour {category} !", "diagnosis_failed": "Échec de la récupération du résultat du diagnostic pour la catégorie '{category}' : {error}", "diagnosis_ip_connected_ipv4": "Le serveur est connecté à Internet en IPv4 !", "diagnosis_ip_no_ipv4": "Le serveur ne dispose pas d'une adresse IPv4.", @@ -675,5 +675,35 @@ "log_app_config_set": "Appliquer la configuration à l'application '{}'", "service_not_reloading_because_conf_broken": "Le service '{name}' n'a pas été rechargé/redémarré car sa configuration est cassée : {errors}", "app_argument_password_help_keep": "Tapez sur Entrée pour conserver la valeur actuelle", - "app_argument_password_help_optional": "Tapez un espace pour vider le mot de passe" -} \ No newline at end of file + "app_argument_password_help_optional": "Tapez un espace pour vider le mot de passe", + "domain_registrar_is_not_configured": "Le registrar n'est pas encore configuré pour le domaine {domain}.", + "domain_dns_push_not_applicable": "La fonction de configuration DNS automatique n'est pas applicable au domaine {domain}. Vous devez configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns_config.", + "domain_dns_registrar_yunohost": "Ce domaine est nohost.me / nohost.st / ynh.fr et sa configuration DNS est donc automatiquement gérée par YunoHost sans autre configuration. (voir la commande 'yunohost dyndns update')", + "domain_dns_registrar_supported": "YunoHost a détecté automatiquement que ce domaine est géré par le registrar **{registrar}**. Si vous le souhaitez, YunoHost configurera automatiquement cette zone DNS, si vous lui fournissez les identifiants API appropriés. Vous pouvez trouver de la documentation sur la façon d'obtenir vos identifiants API sur cette page : https://yunohost.org/registar_api_{registrar}. (Vous pouvez également configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns )", + "domain_config_features_disclaimer": "Jusqu'à présent, l'activation/désactivation des fonctionnalités de messagerie ou XMPP n'a d'impact que sur la configuration DNS recommandée et automatique, et non sur les configurations système !", + "domain_dns_push_managed_in_parent_domain": "La fonctionnalité de configuration DNS automatique est gérée dans le domaine parent {parent_domain}.", + "domain_dns_registrar_managed_in_parent_domain": "Ce domaine est un sous-domaine de {parent_domain_link}. La configuration du registrar DNS doit être gérée dans le panneau de configuration de {parent_domain}.", + "domain_dns_registrar_not_supported": "YunoHost n'a pas pu détecter automatiquement le bureau d'enregistrement gérant ce domaine. Vous devez configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns.", + "domain_dns_registrar_experimental": "Jusqu'à présent, l'interface avec l'API de **{registrar}** n'a pas été correctement testée et revue par la communauté YunoHost. L'assistance est **très expérimentale** - soyez prudent !", + "domain_dns_push_failed_to_authenticate": "Échec de l'authentification sur l'API du bureau d'enregistrement pour le domaine « {domain} ». Très probablement les informations d'identification sont incorrectes ? (Error: {error})", + "domain_dns_push_failed_to_list": "Échec de la liste des enregistrements actuels à l'aide de l'API du registraire : {error}", + "domain_dns_push_already_up_to_date": "Dossiers déjà à jour.", + "domain_dns_pushing": "Transmission des enregistrements DNS...", + "domain_dns_push_record_failed": "Échec de l'enregistrement {action} {type}/{name} : {error}", + "domain_dns_push_success": "Enregistrements DNS mis à jour !", + "domain_dns_push_failed": "La mise à jour des enregistrements DNS a échoué.", + "domain_dns_push_partial_failure": "Enregistrements DNS partiellement mis à jour : certains avertissements/erreurs ont été signalés.", + "domain_config_mail_in": "Emails entrants", + "domain_config_mail_out": "Emails sortants", + "domain_config_xmpp": "Messagerie instantanée (XMPP)", + "domain_config_auth_token": "Jeton d'authentification", + "domain_config_auth_key": "Clé d'authentification", + "domain_config_auth_secret": "Secret d'authentification", + "domain_config_api_protocol": "Protocole API", + "domain_config_auth_entrypoint": "Point d'entrée API", + "domain_config_auth_application_key": "Clé d'application", + "domain_config_auth_application_secret": "Clé secrète de l'application", + "ldap_attribute_already_exists": "L'attribut LDAP '{attribute}' existe déjà avec la valeur '{value}'", + "log_domain_config_set": "Mettre à jour la configuration du domaine '{}'", + "log_domain_dns_push": "Pousser les enregistrements DNS pour le domaine '{}'" +} diff --git a/src/yunohost/.coveragerc b/src/yunohost/.coveragerc deleted file mode 100644 index 43e152271..000000000 --- a/src/yunohost/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[report] -omit=tests/*,vendor/*,/usr/lib/moulinette/yunohost/ diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 926de6b1f..6b83f95c1 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -31,15 +31,12 @@ import yaml import time import re import subprocess -import glob -import urllib.parse import tempfile from collections import OrderedDict +from typing import List, Tuple, Dict, Any from moulinette import Moulinette, m18n -from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.network import download_json from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import ( read_file, @@ -48,98 +45,54 @@ from moulinette.utils.filesystem import ( read_yaml, write_to_file, write_to_json, - write_to_yaml, - mkdir, + cp, + rm, + chown, + chmod, ) from yunohost.utils import packages from yunohost.utils.config import ( ConfigPanel, - parse_args_in_yunohost_format, + ask_questions_and_parse_answers, + DomainQuestion, + PathQuestion, ) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory from yunohost.log import is_unit_operation, OperationLogger +from yunohost.app_catalog import ( + app_catalog, + app_search, + _load_apps_catalog, + app_fetchlist, +) # noqa logger = getActionLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs" -APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" -APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" -APPS_CATALOG_API_VERSION = 2 -APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" - re_app_instance_name = re.compile( r"^(?P[\w-]+?)(__(?P[1-9][0-9]*))?$" ) +APP_REPO_URL = re.compile( + r"^https://[a-zA-Z0-9-_.]+/[a-zA-Z0-9-_./]+/[a-zA-Z0-9-_.]+_ynh(/?(-/)?tree/[a-zA-Z0-9-_]+)?(\.git)?/?$" +) -def app_catalog(full=False, with_categories=False): - """ - Return a dict of apps available to installation from Yunohost's app catalog - """ - - # Get app list from catalog cache - catalog = _load_apps_catalog() - installed_apps = set(_installed_apps()) - - # Trim info for apps if not using --full - for app, infos in catalog["apps"].items(): - infos["installed"] = app in installed_apps - - infos["manifest"]["description"] = _value_for_locale( - infos["manifest"]["description"] - ) - - if not full: - catalog["apps"][app] = { - "description": infos["manifest"]["description"], - "level": infos["level"], - } - else: - infos["manifest"]["arguments"] = _set_default_ask_questions( - infos["manifest"].get("arguments", {}) - ) - - # Trim info for categories if not using --full - for category in catalog["categories"]: - category["title"] = _value_for_locale(category["title"]) - category["description"] = _value_for_locale(category["description"]) - for subtags in category.get("subtags", []): - subtags["title"] = _value_for_locale(subtags["title"]) - - if not full: - catalog["categories"] = [ - {"id": c["id"], "description": c["description"]} - for c in catalog["categories"] - ] - - if not with_categories: - return {"apps": catalog["apps"]} - else: - return {"apps": catalog["apps"], "categories": catalog["categories"]} - - -def app_search(string): - """ - Return a dict of apps whose description or name match the search string - """ - - # Retrieve a simple dict listing all apps - catalog_of_apps = app_catalog() - - # Selecting apps according to a match in app name or description - matching_apps = {"apps": {}} - for app in catalog_of_apps["apps"].items(): - if re.search(string, app[0], flags=re.IGNORECASE) or re.search( - string, app[1]["description"], flags=re.IGNORECASE - ): - matching_apps["apps"][app[0]] = app[1] - - return matching_apps +APP_FILES_TO_COPY = [ + "manifest.json", + "manifest.toml", + "actions.json", + "actions.toml", + "config_panel.toml", + "scripts", + "conf", + "hooks", + "doc", +] def app_list(full=False): @@ -415,32 +368,31 @@ def app_change_url(operation_logger, app, domain, path): old_path = app_setting(app, "path") # Normalize path and domain format - old_domain, old_path = _normalize_domain_path(old_domain, old_path) - domain, path = _normalize_domain_path(domain, path) + + domain = DomainQuestion.normalize(domain) + old_domain = DomainQuestion.normalize(old_domain) + path = PathQuestion.normalize(path) + old_path = PathQuestion.normalize(old_path) if (domain, path) == (old_domain, old_path): raise YunohostValidationError( "app_change_url_identical_domains", domain=domain, path=path ) - # Check the url is available - _assert_no_conflicting_apps(domain, path, ignore_app=app) - - manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) - - # Retrieve arguments list for change_url script - # TODO: Allow to specify arguments - args_odict = _parse_args_from_manifest(manifest, "change_url") + app_setting_path = os.path.join(APPS_SETTING_PATH, app) + path_requirement = _guess_webapp_path_requirement(app_setting_path) + _validate_webpath_requirement( + {"domain": domain, "path": path}, path_requirement, ignore_app=app + ) tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) # Prepare env. var. to pass to script - env_dict = _make_environment_for_app_script(app, args=args_odict) + env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app) env_dict["YNH_APP_OLD_DOMAIN"] = old_domain env_dict["YNH_APP_OLD_PATH"] = old_path env_dict["YNH_APP_NEW_DOMAIN"] = domain env_dict["YNH_APP_NEW_PATH"] = path - env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app if domain != old_domain: operation_logger.related_to.append(("domain", old_domain)) @@ -496,6 +448,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False ) from yunohost.permission import permission_sync_to_user from yunohost.regenconf import manually_modified_files + from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers apps = app # Check if disk space available @@ -528,22 +481,22 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if file and isinstance(file, dict): # We use this dirty hack to test chained upgrades in unit/functional tests - manifest, extracted_app_folder = _extract_app_from_file( - file[app_instance_name] - ) + new_app_src = file[app_instance_name] elif file: - manifest, extracted_app_folder = _extract_app_from_file(file) + new_app_src = file elif url: - manifest, extracted_app_folder = _fetch_app_from_git(url) + new_app_src = url elif app_dict["upgradable"] == "url_required": logger.warning(m18n.n("custom_app_url_required", app=app_instance_name)) continue elif app_dict["upgradable"] == "yes" or force: - manifest, extracted_app_folder = _fetch_app_from_git(app_instance_name) + new_app_src = app_dict["id"] else: logger.success(m18n.n("app_already_up_to_date", app=app_instance_name)) continue + manifest, extracted_app_folder = _extract_app(new_app_src) + # Manage upgrade type and avoid any upgrade if there is nothing to do upgrade_type = "UNKNOWN" # Get current_version and new version @@ -582,22 +535,19 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False upgrade_type = "UPGRADE_FULL" # Check requirements - _check_manifest_requirements(manifest, app_instance_name=app_instance_name) + _check_manifest_requirements(manifest) _assert_system_is_sane_for_app(manifest, "pre") app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) - # Retrieve arguments list for upgrade script - # TODO: Allow to specify arguments - args_odict = _parse_args_from_manifest(manifest, "upgrade") - # Prepare env. var. to pass to script - env_dict = _make_environment_for_app_script(app_instance_name, args=args_odict) + env_dict = _make_environment_for_app_script( + app_instance_name, workdir=extracted_app_folder + ) env_dict["YNH_APP_UPGRADE_TYPE"] = upgrade_type env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version) env_dict["YNH_APP_CURRENT_VERSION"] = str(app_current_version) env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0" - env_dict["YNH_APP_BASEDIR"] = extracted_app_folder # We'll check that the app didn't brutally edit some system configuration manually_modified_files_before_install = manually_modified_files() @@ -687,44 +637,21 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False hook_add(app_instance_name, extracted_app_folder + "/hooks/" + hook) # Replace scripts and manifest and conf (if exists) - os.system( - 'rm -rf "%s/scripts" "%s/manifest.toml %s/manifest.json %s/conf"' - % ( - app_setting_path, - app_setting_path, - app_setting_path, - app_setting_path, - ) - ) - - if os.path.exists(os.path.join(extracted_app_folder, "manifest.json")): - os.system( - 'mv "%s/manifest.json" "%s/scripts" %s' - % (extracted_app_folder, extracted_app_folder, app_setting_path) - ) - if os.path.exists(os.path.join(extracted_app_folder, "manifest.toml")): - os.system( - 'mv "%s/manifest.toml" "%s/scripts" %s' - % (extracted_app_folder, extracted_app_folder, app_setting_path) - ) - - for file_to_copy in [ - "actions.json", - "actions.toml", - "config_panel.toml", - "conf", - ]: + # Move scripts and manifest to the right place + for file_to_copy in APP_FILES_TO_COPY: + rm(f"{app_setting_path}/{file_to_copy}", recursive=True, force=True) if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): - os.system( - "cp -R %s/%s %s" - % (extracted_app_folder, file_to_copy, app_setting_path) + cp( + f"{extracted_app_folder}/{file_to_copy}", + f"{app_setting_path}/{file_to_copy}", + recursive=True, ) # Clean and set permissions shutil.rmtree(extracted_app_folder) - os.system("chmod 600 %s" % app_setting_path) - os.system("chmod 400 %s/settings.yml" % app_setting_path) - os.system("chown -R root: %s" % app_setting_path) + chmod(app_setting_path, 0o600) + chmod(f"{app_setting_path}/settings.yml", 0o400) + chown(app_setting_path, "root", recursive=True) # So much win logger.success(m18n.n("app_upgraded", app=app_instance_name)) @@ -739,14 +666,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False def app_manifest(app): - raw_app_list = _load_apps_catalog()["apps"] - - if app in raw_app_list or ("@" in app) or ("http://" in app) or ("https://" in app): - manifest, extracted_app_folder = _fetch_app_from_git(app) - elif os.path.exists(app): - manifest, extracted_app_folder = _extract_app_from_file(app) - else: - raise YunohostValidationError("app_unknown") + manifest, extracted_app_folder = _extract_app(app) shutil.rmtree(extracted_app_folder) @@ -788,20 +708,30 @@ def app_install( permission_sync_to_user, ) from yunohost.regenconf import manually_modified_files + from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + + # Check if disk space available + if free_space_in_directory("/") <= 512 * 1000 * 1000: + raise YunohostValidationError("disk_space_not_sufficient_install") + + def confirm_install(app): - def confirm_install(confirm): # Ignore if there's nothing for confirm (good quality app), if --force is used # or if request on the API (confirm already implemented on the API side) - if confirm is None or force or Moulinette.interface.type == "api": + if force or Moulinette.interface.type == "api": + return + + quality = _app_quality(app) + if quality == "success": return # i18n: confirm_app_install_warning # i18n: confirm_app_install_danger # i18n: confirm_app_install_thirdparty - if confirm in ["danger", "thirdparty"]: + if quality in ["danger", "thirdparty"]: answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + confirm, answers="Yes, I understand"), + m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"), color="red", ) if answer != "Yes, I understand": @@ -809,51 +739,13 @@ def app_install( else: answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + confirm, answers="Y/N"), color="yellow" + m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow" ) if answer.upper() != "Y": raise YunohostError("aborting") - raw_app_list = _load_apps_catalog()["apps"] - - if app in raw_app_list or ("@" in app) or ("http://" in app) or ("https://" in app): - - # If we got an app name directly (e.g. just "wordpress"), we gonna test this name - if app in raw_app_list: - app_name_to_test = app - # If we got an url like "https://github.com/foo/bar_ynh, we want to - # extract "bar" and test if we know this app - elif ("http://" in app) or ("https://" in app): - app_name_to_test = app.strip("/").split("/")[-1].replace("_ynh", "") - else: - # FIXME : watdo if '@' in app ? - app_name_to_test = None - - if app_name_to_test in raw_app_list: - - state = raw_app_list[app_name_to_test].get("state", "notworking") - level = raw_app_list[app_name_to_test].get("level", None) - confirm = "danger" - if state in ["working", "validated"]: - if isinstance(level, int) and level >= 5: - confirm = None - elif isinstance(level, int) and level > 0: - confirm = "warning" - else: - confirm = "thirdparty" - - confirm_install(confirm) - - manifest, extracted_app_folder = _fetch_app_from_git(app) - elif os.path.exists(app): - confirm_install("thirdparty") - manifest, extracted_app_folder = _extract_app_from_file(app) - else: - raise YunohostValidationError("app_unknown") - - # Check if disk space available - if free_space_in_directory("/") <= 512 * 1000 * 1000: - raise YunohostValidationError("disk_space_not_sufficient_install") + confirm_install(app) + manifest, extracted_app_folder = _extract_app(app) # Check ID if "id" not in manifest or "__" in manifest["id"] or "." in manifest["id"]: @@ -863,11 +755,11 @@ def app_install( label = label if label else manifest["name"] # Check requirements - _check_manifest_requirements(manifest, app_id) + _check_manifest_requirements(manifest) _assert_system_is_sane_for_app(manifest, "pre") # Check if app can be forked - instance_number = _installed_instance_number(app_id, last=True) + 1 + instance_number = _next_instance_number_for_app(app_id) if instance_number > 1: if "multi_instance" not in manifest or not is_true(manifest["multi_instance"]): raise YunohostValidationError("app_already_installed", app=app_id) @@ -878,13 +770,17 @@ def app_install( app_instance_name = app_id # Retrieve arguments list for install script - args_dict = ( - {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) - ) - args_odict = _parse_args_from_manifest(manifest, "install", args=args_dict) + raw_questions = manifest.get("arguments", {}).get("install", {}) + questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) + args = { + question.name: question.value + for question in questions + if question.value is not None + } # Validate domain / path availability for webapps - _validate_and_normalize_webpath(args_odict, extracted_app_folder) + path_requirement = _guess_webapp_path_requirement(extracted_app_folder) + _validate_webpath_requirement(args, path_requirement) # Attempt to patch legacy helpers ... _patch_legacy_helpers(extracted_app_folder) @@ -918,22 +814,12 @@ def app_install( _set_app_settings(app_instance_name, app_settings) # Move scripts and manifest to the right place - if os.path.exists(os.path.join(extracted_app_folder, "manifest.json")): - os.system("cp %s/manifest.json %s" % (extracted_app_folder, app_setting_path)) - if os.path.exists(os.path.join(extracted_app_folder, "manifest.toml")): - os.system("cp %s/manifest.toml %s" % (extracted_app_folder, app_setting_path)) - os.system("cp -R %s/scripts %s" % (extracted_app_folder, app_setting_path)) - - for file_to_copy in [ - "actions.json", - "actions.toml", - "config_panel.toml", - "conf", - ]: + for file_to_copy in APP_FILES_TO_COPY: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): - os.system( - "cp -R %s/%s %s" - % (extracted_app_folder, file_to_copy, app_setting_path) + cp( + f"{extracted_app_folder}/{file_to_copy}", + f"{app_setting_path}/{file_to_copy}", + recursive=True, ) # Initialize the main permission for the app @@ -949,13 +835,15 @@ def app_install( ) # Prepare env. var. to pass to script - env_dict = _make_environment_for_app_script(app_instance_name, args=args_odict) - env_dict["YNH_APP_BASEDIR"] = extracted_app_folder + env_dict = _make_environment_for_app_script( + app_instance_name, args=args, workdir=extracted_app_folder + ) env_dict_for_logging = env_dict.copy() - for arg_name, arg_value_and_type in args_odict.items(): - if arg_value_and_type[1] == "password": - del env_dict_for_logging["YNH_APP_ARG_%s" % arg_name.upper()] + for question in questions: + # Or should it be more generally question.redact ? + if question.type == "password": + del env_dict_for_logging["YNH_APP_ARG_%s" % question.name.upper()] operation_logger.extra.update({"env": env_dict_for_logging}) @@ -1010,12 +898,9 @@ def app_install( logger.warning(m18n.n("app_remove_after_failed_install")) # Setup environment for remove script - env_dict_remove = {} - env_dict_remove["YNH_APP_ID"] = app_id - env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name - env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number) - env_dict_remove["YNH_APP_MANIFEST_VERSION"] = manifest.get("version", "?") - env_dict_remove["YNH_APP_BASEDIR"] = extracted_app_folder + env_dict_remove = _make_environment_for_app_script( + app_instance_name, workdir=extracted_app_folder + ) # Execute remove script operation_logger_remove = OperationLogger( @@ -1078,9 +963,9 @@ def app_install( # Clean and set permissions shutil.rmtree(extracted_app_folder) - os.system("chmod 600 %s" % app_setting_path) - os.system("chmod 400 %s/settings.yml" % app_setting_path) - os.system("chown -R root: %s" % app_setting_path) + chmod(app_setting_path, 0o600) + chmod(f"{app_setting_path}/settings.yml", 0o400) + chown(app_setting_path, "root", recursive=True) logger.success(m18n.n("installation_complete")) @@ -1097,6 +982,7 @@ def app_remove(operation_logger, app, purge=False): purge -- Remove with all app data """ + from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers from yunohost.hook import hook_exec, hook_remove, hook_callback from yunohost.permission import ( user_permission_list, @@ -1128,12 +1014,8 @@ def app_remove(operation_logger, app, purge=False): env_dict = {} app_id, app_instance_nb = _parse_app_instance_name(app) - env_dict["YNH_APP_ID"] = app_id - env_dict["YNH_APP_INSTANCE_NAME"] = app - env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) - env_dict["YNH_APP_MANIFEST_VERSION"] = manifest.get("version", "?") - env_dict["YNH_APP_PURGE"] = str(purge) - env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app + env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app) + env_dict["YNH_APP_PURGE"] = str(1 if purge else 0) operation_logger.extra.update({"env": env_dict}) operation_logger.flush() @@ -1221,7 +1103,7 @@ def app_makedefault(operation_logger, app, domain=None): write_to_json( "/etc/ssowat/conf.json.persistent", ssowat_conf, sort_keys=True, indent=4 ) - os.system("chmod 644 /etc/ssowat/conf.json.persistent") + chmod("/etc/ssowat/conf.json.persistent", 0o644) logger.success(m18n.n("ssowat_conf_updated")) @@ -1387,7 +1269,8 @@ def app_register_url(app, domain, path): permission_sync_to_user, ) - domain, path = _normalize_domain_path(domain, path) + domain = DomainQuestion.normalize(domain) + path = PathQuestion.normalize(path) # We cannot change the url of an app already installed simply by changing # the settings... @@ -1553,18 +1436,20 @@ def app_action_run(operation_logger, app, action, args=None): action_declaration = actions[action] # Retrieve arguments list for install script - args_dict = ( - dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} - ) - args_odict = _parse_args_for_action(actions[action], args=args_dict) + raw_questions = actions[action].get("arguments", {}) + questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) + args = { + question.name: question.value + for question in questions + if question.value is not None + } tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) env_dict = _make_environment_for_app_script( - app, args=args_odict, args_prefix="ACTION_" + app, args=args, args_prefix="ACTION_", workdir=tmp_workdir_for_app ) env_dict["YNH_ACTION"] = action - env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app _, action_script = tempfile.mkstemp(dir=tmp_workdir_for_app) @@ -1705,22 +1590,6 @@ ynh_app_config_run $1 return values -def _get_all_installed_apps_id(): - """ - Return something like: - ' * app1 - * app2 - * ...' - """ - - all_apps_ids = sorted(_installed_apps()) - - all_apps_ids_formatted = "\n * ".join(all_apps_ids) - all_apps_ids_formatted = "\n * " + all_apps_ids_formatted - - return all_apps_ids_formatted - - def _get_app_actions(app_id): "Get app config panel stored in json or in toml" actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, "actions.toml") @@ -1859,55 +1728,6 @@ def _set_app_settings(app_id, settings): yaml.safe_dump(settings, f, default_flow_style=False) -def _extract_app_from_file(path): - """ - Unzip / untar / copy application tarball or directory to a tmp work directory - - Keyword arguments: - path -- Path of the tarball or directory - """ - logger.debug(m18n.n("extracting")) - - path = os.path.abspath(path) - - extracted_app_folder = _make_tmp_workdir_for_app() - - if ".zip" in path: - extract_result = os.system( - f"unzip '{path}' -d {extracted_app_folder} > /dev/null 2>&1" - ) - elif ".tar" in path: - extract_result = os.system( - f"tar -xf '{path}' -C {extracted_app_folder} > /dev/null 2>&1" - ) - elif os.path.isdir(path): - shutil.rmtree(extracted_app_folder) - if path[-1] != "/": - path = path + "/" - extract_result = os.system(f"cp -a '{path}' {extracted_app_folder}") - else: - extract_result = 1 - - if extract_result != 0: - raise YunohostError("app_extraction_failed") - - try: - if len(os.listdir(extracted_app_folder)) == 1: - for folder in os.listdir(extracted_app_folder): - extracted_app_folder = extracted_app_folder + "/" + folder - manifest = _get_manifest_of_app(extracted_app_folder) - manifest["lastUpdate"] = int(time.time()) - except IOError: - raise YunohostError("app_install_files_invalid") - except ValueError as e: - raise YunohostError("app_manifest_invalid", error=e) - - logger.debug(m18n.n("done")) - - manifest["remote"] = {"type": "file", "path": path} - return manifest, extracted_app_folder - - def _get_manifest_of_app(path): "Get app manifest stored in json or in toml" @@ -2103,143 +1923,193 @@ def _set_default_ask_questions(arguments): return arguments -def _get_git_last_commit_hash(repository, reference="HEAD"): - """ - Attempt to retrieve the last commit hash of a git repository +def _is_app_repo_url(string: str) -> bool: - Keyword arguments: - repository -- The URL or path of the repository + string = string.strip() + # Dummy test for ssh-based stuff ... should probably be improved somehow + if "@" in string: + return True + + return bool(APP_REPO_URL.match(string)) + + +def _app_quality(src: str) -> str: """ - try: - cmd = "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'".format( - repository, reference - ) - commit = check_output(cmd) - except subprocess.CalledProcessError: - logger.error("unable to get last commit from %s", repository) - raise ValueError("Unable to get last commit with git") + app may in fact be an app name, an url, or a path + """ + + raw_app_catalog = _load_apps_catalog()["apps"] + if src in raw_app_catalog or _is_app_repo_url(src): + + # If we got an app name directly (e.g. just "wordpress"), we gonna test this name + if src in raw_app_catalog: + app_name_to_test = src + # If we got an url like "https://github.com/foo/bar_ynh, we want to + # extract "bar" and test if we know this app + elif ("http://" in src) or ("https://" in src): + app_name_to_test = src.strip("/").split("/")[-1].replace("_ynh", "") + else: + # FIXME : watdo if '@' in app ? + return "thirdparty" + + if app_name_to_test in raw_app_catalog: + + state = raw_app_catalog[app_name_to_test].get("state", "notworking") + level = raw_app_catalog[app_name_to_test].get("level", None) + if state in ["working", "validated"]: + if isinstance(level, int) and level >= 5: + return "success" + elif isinstance(level, int) and level > 0: + return "warning" + return "danger" + else: + return "thirdparty" + + elif os.path.exists(src): + return "thirdparty" else: - return commit.strip() + if "http://" in src or "https://" in src: + logger.error( + f"{src} is not a valid app url: app url are expected to look like https://domain.tld/path/to/repo_ynh" + ) + raise YunohostValidationError("app_unknown") -def _fetch_app_from_git(app): +def _extract_app(src: str) -> Tuple[Dict, str]: """ - Unzip or untar application tarball to a tmp directory - - Keyword arguments: - app -- App_id or git repo URL + src may be an app name, an url, or a path """ - # Extract URL, branch and revision to download - if ("@" in app) or ("http://" in app) or ("https://" in app): - url = app - branch = "master" - if "/tree/" in url: - url, branch = url.split("/tree/", 1) - revision = "HEAD" - else: - app_dict = _load_apps_catalog()["apps"] + raw_app_catalog = _load_apps_catalog()["apps"] - app_id, _ = _parse_app_instance_name(app) - - if app_id not in app_dict: - raise YunohostValidationError("app_unknown") - elif "git" not in app_dict[app_id]: + # App is an appname in the catalog + if src in raw_app_catalog: + if "git" not in raw_app_catalog[src]: raise YunohostValidationError("app_unsupported_remote_type") - app_info = app_dict[app_id] + app_info = raw_app_catalog[src] url = app_info["git"]["url"] branch = app_info["git"]["branch"] revision = str(app_info["git"]["revision"]) + return _extract_app_from_gitrepo(url, branch, revision, app_info) + # App is a git repo url + elif _is_app_repo_url(src): + url = src.strip().strip("/") + branch = "master" + revision = "HEAD" + # gitlab urls may look like 'https://domain/org/group/repo/-/tree/testing' + # compated to github urls looking like 'https://domain/org/repo/tree/testing' + if "/-/" in url: + url = url.replace("/-/", "/") + if "/tree/" in url: + url, branch = url.split("/tree/", 1) + return _extract_app_from_gitrepo(url, branch, revision, {}) + # App is a local folder + elif os.path.exists(src): + return _extract_app_from_folder(src) + else: + if "http://" in src or "https://" in src: + logger.error( + f"{src} is not a valid app url: app url are expected to look like https://domain.tld/path/to/repo_ynh" + ) + raise YunohostValidationError("app_unknown") + + +def _extract_app_from_folder(path: str) -> Tuple[Dict, str]: + """ + Unzip / untar / copy application tarball or directory to a tmp work directory + + Keyword arguments: + path -- Path of the tarball or directory + """ + logger.debug(m18n.n("extracting")) + + path = os.path.abspath(path) extracted_app_folder = _make_tmp_workdir_for_app() + if os.path.isdir(path): + shutil.rmtree(extracted_app_folder) + if path[-1] != "/": + path = path + "/" + cp(path, extracted_app_folder, recursive=True) + else: + try: + shutil.unpack_archive(path, extracted_app_folder) + except Exception: + raise YunohostError("app_extraction_failed") + + try: + if len(os.listdir(extracted_app_folder)) == 1: + for folder in os.listdir(extracted_app_folder): + extracted_app_folder = extracted_app_folder + "/" + folder + except IOError: + raise YunohostError("app_install_files_invalid") + + manifest = _get_manifest_of_app(extracted_app_folder) + manifest["lastUpdate"] = int(time.time()) + + logger.debug(m18n.n("done")) + + manifest["remote"] = {"type": "file", "path": path} + return manifest, extracted_app_folder + + +def _extract_app_from_gitrepo( + url: str, branch: str, revision: str, app_info: Dict = {} +) -> Tuple[Dict, str]: + logger.debug(m18n.n("downloading")) + extracted_app_folder = _make_tmp_workdir_for_app() + # Download only this commit try: # We don't use git clone because, git clone can't download # a specific revision only + ref = branch if revision == "HEAD" else revision run_commands([["git", "init", extracted_app_folder]], shell=False) run_commands( [ ["git", "remote", "add", "origin", url], - [ - "git", - "fetch", - "--depth=1", - "origin", - branch if revision == "HEAD" else revision, - ], + ["git", "fetch", "--depth=1", "origin", ref], ["git", "reset", "--hard", "FETCH_HEAD"], ], cwd=extracted_app_folder, shell=False, ) - manifest = _get_manifest_of_app(extracted_app_folder) except subprocess.CalledProcessError: raise YunohostError("app_sources_fetch_failed") - except ValueError as e: - raise YunohostError("app_manifest_invalid", error=e) else: logger.debug(m18n.n("done")) + manifest = _get_manifest_of_app(extracted_app_folder) + # Store remote repository info into the returned manifest manifest["remote"] = {"type": "git", "url": url, "branch": branch} if revision == "HEAD": try: - manifest["remote"]["revision"] = _get_git_last_commit_hash(url, branch) + # Get git last commit hash + cmd = f"git ls-remote --exit-code {url} {branch} | awk '{{print $1}}'" + manifest["remote"]["revision"] = check_output(cmd) except Exception as e: - logger.debug("cannot get last commit hash because: %s ", e) + logger.warning("cannot get last commit hash because: %s ", e) else: manifest["remote"]["revision"] = revision - manifest["lastUpdate"] = app_info["lastUpdate"] + manifest["lastUpdate"] = app_info.get("lastUpdate") return manifest, extracted_app_folder -def _installed_instance_number(app, last=False): - """ - Check if application is installed and return instance number - - Keyword arguments: - app -- id of App to check - last -- Return only last instance number - - Returns: - Number of last installed instance | List or instances - - """ - if last: - number = 0 - try: - installed_apps = os.listdir(APPS_SETTING_PATH) - except OSError: - os.makedirs(APPS_SETTING_PATH) - return 0 - - for installed_app in installed_apps: - if number == 0 and app == installed_app: - number = 1 - elif "__" in installed_app: - if app == installed_app[: installed_app.index("__")]: - if int(installed_app[installed_app.index("__") + 2 :]) > number: - number = int(installed_app[installed_app.index("__") + 2 :]) - - return number - - else: - instance_number_list = [] - instances_dict = app_map(app=app, raw=True) - for key, domain in instances_dict.items(): - for key, path in domain.items(): - instance_number_list.append(path["instance"]) - - return sorted(instance_number_list) +# +# ############################### # +# Small utilities # +# ############################### # +# -def _is_installed(app): +def _is_installed(app: str) -> bool: """ Check if application is installed @@ -2253,18 +2123,34 @@ def _is_installed(app): return os.path.isdir(APPS_SETTING_PATH + app) -def _assert_is_installed(app): +def _assert_is_installed(app: str) -> None: if not _is_installed(app): raise YunohostValidationError( "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() ) -def _installed_apps(): +def _installed_apps() -> List[str]: return os.listdir(APPS_SETTING_PATH) -def _check_manifest_requirements(manifest, app_instance_name): +def _get_all_installed_apps_id(): + """ + Return something like: + ' * app1 + * app2 + * ...' + """ + + all_apps_ids = sorted(_installed_apps()) + + all_apps_ids_formatted = "\n * ".join(all_apps_ids) + all_apps_ids_formatted = "\n * " + all_apps_ids_formatted + + return all_apps_ids_formatted + + +def _check_manifest_requirements(manifest: Dict): """Check if required packages are met from the manifest""" packaging_format = int(manifest.get("packaging_format", 0)) @@ -2276,7 +2162,9 @@ def _check_manifest_requirements(manifest, app_instance_name): if not requirements: return - logger.debug(m18n.n("app_requirements_checking", app=app_instance_name)) + app = manifest.get("id", "?") + + logger.debug(m18n.n("app_requirements_checking", app=app)) # Iterate over requirements for pkgname, spec in requirements.items(): @@ -2287,83 +2175,31 @@ def _check_manifest_requirements(manifest, app_instance_name): pkgname=pkgname, version=version, spec=spec, - app=app_instance_name, + app=app, ) -def _parse_args_from_manifest(manifest, action, args={}): - """Parse arguments needed for an action from the manifest - - Retrieve specified arguments for the action from the manifest, and parse - given args according to that. If some required arguments are not provided, - its values will be asked if interaction is possible. - Parsed arguments will be returned as an OrderedDict - - Keyword arguments: - manifest -- The app manifest to use - action -- The action to retrieve arguments for - args -- A dictionnary of arguments to parse - - """ - if action not in manifest["arguments"]: - logger.debug("no arguments found for '%s' in manifest", action) - return OrderedDict() - - action_args = manifest["arguments"][action] - return parse_args_in_yunohost_format(args, action_args) - - -def _parse_args_for_action(action, args={}): - """Parse arguments needed for an action from the actions list - - Retrieve specified arguments for the action from the manifest, and parse - given args according to that. If some required arguments are not provided, - its values will be asked if interaction is possible. - Parsed arguments will be returned as an OrderedDict - - Keyword arguments: - action -- The action - args -- A dictionnary of arguments to parse - - """ - args_dict = OrderedDict() - - if "arguments" not in action: - logger.debug("no arguments found for '%s' in manifest", action) - return args_dict - - action_args = action["arguments"] - - return parse_args_in_yunohost_format(args, action_args) - - -def _validate_and_normalize_webpath(args_dict, app_folder): +def _guess_webapp_path_requirement(app_folder: str) -> str: # If there's only one "domain" and "path", validate that domain/path # is an available url and normalize the path. - domain_args = [ - (name, value[0]) for name, value in args_dict.items() if value[1] == "domain" + manifest = _get_manifest_of_app(app_folder) + raw_questions = manifest.get("arguments", {}).get("install", {}) + + domain_questions = [ + question for question in raw_questions if question.get("type") == "domain" ] - path_args = [ - (name, value[0]) for name, value in args_dict.items() if value[1] == "path" + path_questions = [ + question for question in raw_questions if question.get("type") == "path" ] - if len(domain_args) == 1 and len(path_args) == 1: - - domain = domain_args[0][1] - path = path_args[0][1] - domain, path = _normalize_domain_path(domain, path) - - # Check the url is available - _assert_no_conflicting_apps(domain, path) - - # (We save this normalized path so that the install script have a - # standard path format to deal with no matter what the user inputted) - args_dict[path_args[0][0]] = (path, "path") - - # This is likely to be a full-domain app... - elif len(domain_args) == 1 and len(path_args) == 0: + if len(domain_questions) == 0 and len(path_questions) == 0: + return "" + if len(domain_questions) == 1 and len(path_questions) == 1: + return "domain_and_path" + if len(domain_questions) == 1 and len(path_questions) == 0: + # This is likely to be a full-domain app... # Confirm that this is a full-domain app This should cover most cases # ... though anyway the proper solution is to implement some mechanism @@ -2373,36 +2209,30 @@ def _validate_and_normalize_webpath(args_dict, app_folder): # Full-domain apps typically declare something like path_url="/" or path=/ # and use ynh_webpath_register or yunohost_app_checkurl inside the install script - install_script_content = open( - os.path.join(app_folder, "scripts/install") - ).read() + install_script_content = read_file(os.path.join(app_folder, "scripts/install")) if re.search( - r"\npath(_url)?=[\"']?/[\"']?\n", install_script_content - ) and re.search( - r"(ynh_webpath_register|yunohost app checkurl)", install_script_content - ): + r"\npath(_url)?=[\"']?/[\"']?", install_script_content + ) and re.search(r"ynh_webpath_register", install_script_content): + return "full_domain" - domain = domain_args[0][1] - _assert_no_conflicting_apps(domain, "/", full_domain=True) + return "?" -def _normalize_domain_path(domain, path): +def _validate_webpath_requirement( + args: Dict[str, Any], path_requirement: str, ignore_app=None +) -> None: - # We want url to be of the format : - # some.domain.tld/foo + domain = args.get("domain") + path = args.get("path") - # Remove http/https prefix if it's there - if domain.startswith("https://"): - domain = domain[len("https://") :] - elif domain.startswith("http://"): - domain = domain[len("http://") :] + if path_requirement == "domain_and_path": + _assert_no_conflicting_apps(domain, path, ignore_app=ignore_app) - # Remove trailing slashes - domain = domain.rstrip("/").lower() - path = "/" + path.strip("/") - - return domain, path + elif path_requirement == "full_domain": + _assert_no_conflicting_apps( + domain, "/", full_domain=True, ignore_app=ignore_app + ) def _get_conflicting_apps(domain, path, ignore_app=None): @@ -2417,7 +2247,8 @@ def _get_conflicting_apps(domain, path, ignore_app=None): from yunohost.domain import _assert_domain_exists - domain, path = _normalize_domain_path(domain, path) + domain = DomainQuestion.normalize(domain) + path = PathQuestion.normalize(path) # Abort if domain is unknown _assert_domain_exists(domain) @@ -2466,7 +2297,9 @@ def _assert_no_conflicting_apps(domain, path, ignore_app=None, full_domain=False ) -def _make_environment_for_app_script(app, args={}, args_prefix="APP_ARG_"): +def _make_environment_for_app_script( + app, args={}, args_prefix="APP_ARG_", workdir=None +): app_setting_path = os.path.join(APPS_SETTING_PATH, app) @@ -2480,10 +2313,11 @@ def _make_environment_for_app_script(app, args={}, args_prefix="APP_ARG_"): "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"), } - for arg_name, arg_value_and_type in args.items(): - env_dict["YNH_%s%s" % (args_prefix, arg_name.upper())] = str( - arg_value_and_type[0] - ) + if workdir: + env_dict["YNH_APP_BASEDIR"] = workdir + + for arg_name, arg_value in args.items(): + env_dict["YNH_%s%s" % (args_prefix, arg_name.upper())] = str(arg_value) return env_dict @@ -2521,173 +2355,22 @@ def _parse_app_instance_name(app_instance_name): return (appid, app_instance_nb) -# -# ############################### # -# Applications list management # -# ############################### # -# +def _next_instance_number_for_app(app): + # Get list of sibling apps, such as {app}, {app}__2, {app}__4 + apps = _installed_apps() + sibling_app_ids = [a for a in apps if a == app or a.startswith(f"{app}__")] -def _initialize_apps_catalog_system(): - """ - This function is meant to intialize the apps_catalog system with YunoHost's default app catalog. - """ + # Find the list of ids, such as [1, 2, 4] + sibling_ids = [_parse_app_instance_name(a)[1] for a in sibling_app_ids] - default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] - - try: - logger.debug( - "Initializing apps catalog system with YunoHost's default app list" - ) - write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list) - except Exception as e: - raise YunohostError( - "Could not initialize the apps catalog system... : %s" % str(e) - ) - - logger.success(m18n.n("apps_catalog_init_success")) - - -def _read_apps_catalog_list(): - """ - Read the json corresponding to the list of apps catalogs - """ - - try: - list_ = read_yaml(APPS_CATALOG_CONF) - # Support the case where file exists but is empty - # by returning [] if list_ is None - return list_ if list_ else [] - except Exception as e: - raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e)) - - -def _actual_apps_catalog_api_url(base_url): - - return "{base_url}/v{version}/apps.json".format( - base_url=base_url, version=APPS_CATALOG_API_VERSION - ) - - -def _update_apps_catalog(): - """ - Fetches the json for each apps_catalog and update the cache - - apps_catalog_list is for example : - [ {"id": "default", "url": "https://app.yunohost.org/default/"} ] - - Then for each apps_catalog, the actual json URL to be fetched is like : - https://app.yunohost.org/default/vX/apps.json - - And store it in : - /var/cache/yunohost/repo/default.json - """ - - apps_catalog_list = _read_apps_catalog_list() - - logger.info(m18n.n("apps_catalog_updating")) - - # Create cache folder if needed - if not os.path.exists(APPS_CATALOG_CACHE): - logger.debug("Initialize folder for apps catalog cache") - mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root") - - for apps_catalog in apps_catalog_list: - apps_catalog_id = apps_catalog["id"] - actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"]) - - # Fetch the json - try: - apps_catalog_content = download_json(actual_api_url) - except Exception as e: - raise YunohostError( - "apps_catalog_failed_to_download", - apps_catalog=apps_catalog_id, - error=str(e), - ) - - # Remember the apps_catalog api version for later - apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION - - # Save the apps_catalog data in the cache - cache_file = "{cache_folder}/{list}.json".format( - cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id - ) - try: - write_to_json(cache_file, apps_catalog_content) - except Exception as e: - raise YunohostError( - "Unable to write cache data for %s apps_catalog : %s" - % (apps_catalog_id, str(e)) - ) - - logger.success(m18n.n("apps_catalog_update_success")) - - -def _load_apps_catalog(): - """ - Read all the apps catalog cache files and build a single dict (merged_catalog) - corresponding to all known apps and categories - """ - - merged_catalog = {"apps": {}, "categories": []} - - for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: - - # Let's load the json from cache for this catalog - cache_file = "{cache_folder}/{list}.json".format( - cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id - ) - - try: - apps_catalog_content = ( - read_json(cache_file) if os.path.exists(cache_file) else None - ) - except Exception as e: - raise YunohostError( - "Unable to read cache for apps_catalog %s : %s" % (cache_file, e), - raw_msg=True, - ) - - # Check that the version of the data matches version .... - # ... otherwise it means we updated yunohost in the meantime - # and need to update the cache for everything to be consistent - if ( - not apps_catalog_content - or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION - ): - logger.info(m18n.n("apps_catalog_obsolete_cache")) - _update_apps_catalog() - apps_catalog_content = read_json(cache_file) - - del apps_catalog_content["from_api_version"] - - # Add apps from this catalog to the output - for app, info in apps_catalog_content["apps"].items(): - - # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ... - # in which case we keep only the first one found) - if app in merged_catalog["apps"]: - logger.warning( - "Duplicate app %s found between apps catalog %s and %s" - % (app, apps_catalog_id, merged_catalog["apps"][app]["repository"]) - ) - continue - - info["repository"] = apps_catalog_id - merged_catalog["apps"][app] = info - - # Annnnd categories - merged_catalog["categories"] += apps_catalog_content["categories"] - - return merged_catalog - - -# -# ############################### # -# Small utilities # -# ############################### # -# + # Find the first 'i' that's not in the sibling_ids list already + i = 1 + while True: + if i not in sibling_ids: + return i + else: + i += 1 def _make_tmp_workdir_for_app(app=None): @@ -2816,184 +2499,3 @@ def _assert_system_is_sane_for_app(manifest, when): raise YunohostValidationError("dpkg_is_broken") elif when == "post": raise YunohostError("this_action_broke_dpkg") - - -LEGACY_PHP_VERSION_REPLACEMENTS = [ - ("/etc/php5", "/etc/php/7.4"), - ("/etc/php/7.0", "/etc/php/7.4"), - ("/etc/php/7.3", "/etc/php/7.4"), - ("/var/run/php5-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.0-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.3-fpm", "/var/run/php/php7.4-fpm"), - ("php5", "php7.4"), - ("php7.0", "php7.4"), - ("php7.3", "php7.4"), - ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'), - ( - 'phpversion="${phpversion:-7.0}"', - 'phpversion="${phpversion:-7.4}"', - ), # Many helpers like the composer ones use 7.0 by default ... - ( - 'phpversion="${phpversion:-7.3}"', - 'phpversion="${phpversion:-7.4}"', - ), # Many helpers like the composer ones use 7.0 by default ... - ( - '"$phpversion" == "7.0"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', - ), # patch ynh_install_php to refuse installing/removing php <= 7.3 - ( - '"$phpversion" == "7.3"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', - ), # patch ynh_install_php to refuse installing/removing php <= 7.3 -] - - -def _patch_legacy_php_versions(app_folder): - - files_to_patch = [] - files_to_patch.extend(glob.glob("%s/conf/*" % app_folder)) - files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) - files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder)) - files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) - files_to_patch.append("%s/manifest.json" % app_folder) - files_to_patch.append("%s/manifest.toml" % app_folder) - - for filename in files_to_patch: - - # Ignore non-regular files - if not os.path.isfile(filename): - continue - - c = ( - "sed -i " - + "".join( - "-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r) - for p, r in LEGACY_PHP_VERSION_REPLACEMENTS - ) - + "%s" % filename - ) - os.system(c) - - -def _patch_legacy_php_versions_in_settings(app_folder): - - settings = read_yaml(os.path.join(app_folder, "settings.yml")) - - if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]: - settings["fpm_config_dir"] = "/etc/php/7.4/fpm" - if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]: - settings["fpm_service"] = "php7.4-fpm" - if settings.get("phpversion") in ["7.0", "7.3"]: - settings["phpversion"] = "7.4" - - # We delete these checksums otherwise the file will appear as manually modified - list_to_remove = ["checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"] - settings = { - k: v - for k, v in settings.items() - if not any(k.startswith(to_remove) for to_remove in list_to_remove) - } - - write_to_yaml(app_folder + "/settings.yml", settings) - - -def _patch_legacy_helpers(app_folder): - - files_to_patch = [] - files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) - files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) - - stuff_to_replace = { - "yunohost app initdb": {"important": True}, - "yunohost app checkport": {"important": True}, - "yunohost tools port-available": {"important": True}, - "yunohost app checkurl": {"important": True}, - # Remove - # Automatic diagnosis data from YunoHost - # __PRE_TAG1__$(yunohost tools diagnosis | ...)__PRE_TAG2__" - # - "yunohost tools diagnosis": { - "pattern": r"(Automatic diagnosis data from YunoHost( *\n)*)? *(__\w+__)? *\$\(yunohost tools diagnosis.*\)(__\w+__)?", - "replace": r"", - "important": False, - }, - # Old $1, $2 in backup/restore scripts... - "app=$2": {"only_for": ["scripts/backup", "scripts/restore"], "important": True}, - # Old $1, $2 in backup/restore scripts... - "backup_dir=$1": {"only_for": ["scripts/backup", "scripts/restore"], "important": True}, - # Old $1, $2 in backup/restore scripts... - "restore_dir=$1": {"only_for": ["scripts/restore"], "important": True}, - # Old $1, $2 in install scripts... - # We ain't patching that shit because it ain't trivial to patch all args... - "domain=$1": {"only_for": ["scripts/install"], "important": True}, - } - - for helper, infos in stuff_to_replace.items(): - infos["pattern"] = ( - re.compile(infos["pattern"]) if infos.get("pattern") else None - ) - infos["replace"] = infos.get("replace") - - for filename in files_to_patch: - - # Ignore non-regular files - if not os.path.isfile(filename): - continue - - try: - content = read_file(filename) - except MoulinetteError: - continue - - replaced_stuff = False - show_warning = False - - for helper, infos in stuff_to_replace.items(): - - # Ignore if not relevant for this file - if infos.get("only_for") and not any( - filename.endswith(f) for f in infos["only_for"] - ): - continue - - # If helper is used, attempt to patch the file - if helper in content and infos["pattern"]: - content = infos["pattern"].sub(infos["replace"], content) - replaced_stuff = True - if infos["important"]: - show_warning = True - - # If the helper is *still* in the content, it means that we - # couldn't patch the deprecated helper in the previous lines. In - # that case, abort the install or whichever step is performed - if helper in content and infos["important"]: - raise YunohostValidationError( - "This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.", - raw_msg=True, - ) - - if replaced_stuff: - - # Check the app do load the helper - # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there... - if filename.split("/")[-1] in [ - "install", - "remove", - "upgrade", - "backup", - "restore", - ]: - source_helpers = "source /usr/share/yunohost/helpers" - if source_helpers not in content: - content.replace("#!/bin/bash", "#!/bin/bash\n" + source_helpers) - if source_helpers not in content: - content = source_helpers + "\n" + content - - # Actually write the new content in the file - write_to_file(filename, content) - - if show_warning: - # And complain about those damn deprecated helpers - logger.error( - r"/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ..." - ) diff --git a/src/yunohost/app_catalog.py b/src/yunohost/app_catalog.py new file mode 100644 index 000000000..e4ffa1db6 --- /dev/null +++ b/src/yunohost/app_catalog.py @@ -0,0 +1,255 @@ +import os +import re + +from moulinette import m18n +from moulinette.utils.log import getActionLogger +from moulinette.utils.network import download_json +from moulinette.utils.filesystem import ( + read_json, + read_yaml, + write_to_json, + write_to_yaml, + mkdir, +) + +from yunohost.utils.i18n import _value_for_locale +from yunohost.utils.error import YunohostError + +logger = getActionLogger("yunohost.app_catalog") + +APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" +APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" +APPS_CATALOG_API_VERSION = 2 +APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" + + +# Old legacy function... +def app_fetchlist(): + logger.warning( + "'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead" + ) + from yunohost.tools import tools_update + + tools_update(target="apps") + + +def app_catalog(full=False, with_categories=False): + """ + Return a dict of apps available to installation from Yunohost's app catalog + """ + + from yunohost.app import _installed_apps, _set_default_ask_questions + + # Get app list from catalog cache + catalog = _load_apps_catalog() + installed_apps = set(_installed_apps()) + + # Trim info for apps if not using --full + for app, infos in catalog["apps"].items(): + infos["installed"] = app in installed_apps + + infos["manifest"]["description"] = _value_for_locale( + infos["manifest"]["description"] + ) + + if not full: + catalog["apps"][app] = { + "description": infos["manifest"]["description"], + "level": infos["level"], + } + else: + infos["manifest"]["arguments"] = _set_default_ask_questions( + infos["manifest"].get("arguments", {}) + ) + + # Trim info for categories if not using --full + for category in catalog["categories"]: + category["title"] = _value_for_locale(category["title"]) + category["description"] = _value_for_locale(category["description"]) + for subtags in category.get("subtags", []): + subtags["title"] = _value_for_locale(subtags["title"]) + + if not full: + catalog["categories"] = [ + {"id": c["id"], "description": c["description"]} + for c in catalog["categories"] + ] + + if not with_categories: + return {"apps": catalog["apps"]} + else: + return {"apps": catalog["apps"], "categories": catalog["categories"]} + + +def app_search(string): + """ + Return a dict of apps whose description or name match the search string + """ + + # Retrieve a simple dict listing all apps + catalog_of_apps = app_catalog() + + # Selecting apps according to a match in app name or description + matching_apps = {"apps": {}} + for app in catalog_of_apps["apps"].items(): + if re.search(string, app[0], flags=re.IGNORECASE) or re.search( + string, app[1]["description"], flags=re.IGNORECASE + ): + matching_apps["apps"][app[0]] = app[1] + + return matching_apps + + +def _initialize_apps_catalog_system(): + """ + This function is meant to intialize the apps_catalog system with YunoHost's default app catalog. + """ + + default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] + + try: + logger.debug( + "Initializing apps catalog system with YunoHost's default app list" + ) + write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list) + except Exception as e: + raise YunohostError( + "Could not initialize the apps catalog system... : %s" % str(e) + ) + + logger.success(m18n.n("apps_catalog_init_success")) + + +def _read_apps_catalog_list(): + """ + Read the json corresponding to the list of apps catalogs + """ + + try: + list_ = read_yaml(APPS_CATALOG_CONF) + # Support the case where file exists but is empty + # by returning [] if list_ is None + return list_ if list_ else [] + except Exception as e: + raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e)) + + +def _actual_apps_catalog_api_url(base_url): + + return "{base_url}/v{version}/apps.json".format( + base_url=base_url, version=APPS_CATALOG_API_VERSION + ) + + +def _update_apps_catalog(): + """ + Fetches the json for each apps_catalog and update the cache + + apps_catalog_list is for example : + [ {"id": "default", "url": "https://app.yunohost.org/default/"} ] + + Then for each apps_catalog, the actual json URL to be fetched is like : + https://app.yunohost.org/default/vX/apps.json + + And store it in : + /var/cache/yunohost/repo/default.json + """ + + apps_catalog_list = _read_apps_catalog_list() + + logger.info(m18n.n("apps_catalog_updating")) + + # Create cache folder if needed + if not os.path.exists(APPS_CATALOG_CACHE): + logger.debug("Initialize folder for apps catalog cache") + mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root") + + for apps_catalog in apps_catalog_list: + apps_catalog_id = apps_catalog["id"] + actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"]) + + # Fetch the json + try: + apps_catalog_content = download_json(actual_api_url) + except Exception as e: + raise YunohostError( + "apps_catalog_failed_to_download", + apps_catalog=apps_catalog_id, + error=str(e), + ) + + # Remember the apps_catalog api version for later + apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION + + # Save the apps_catalog data in the cache + cache_file = "{cache_folder}/{list}.json".format( + cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id + ) + try: + write_to_json(cache_file, apps_catalog_content) + except Exception as e: + raise YunohostError( + "Unable to write cache data for %s apps_catalog : %s" + % (apps_catalog_id, str(e)) + ) + + logger.success(m18n.n("apps_catalog_update_success")) + + +def _load_apps_catalog(): + """ + Read all the apps catalog cache files and build a single dict (merged_catalog) + corresponding to all known apps and categories + """ + + merged_catalog = {"apps": {}, "categories": []} + + for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: + + # Let's load the json from cache for this catalog + cache_file = "{cache_folder}/{list}.json".format( + cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id + ) + + try: + apps_catalog_content = ( + read_json(cache_file) if os.path.exists(cache_file) else None + ) + except Exception as e: + raise YunohostError( + "Unable to read cache for apps_catalog %s : %s" % (cache_file, e), + raw_msg=True, + ) + + # Check that the version of the data matches version .... + # ... otherwise it means we updated yunohost in the meantime + # and need to update the cache for everything to be consistent + if ( + not apps_catalog_content + or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION + ): + logger.info(m18n.n("apps_catalog_obsolete_cache")) + _update_apps_catalog() + apps_catalog_content = read_json(cache_file) + + del apps_catalog_content["from_api_version"] + + # Add apps from this catalog to the output + for app, info in apps_catalog_content["apps"].items(): + + # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ... + # in which case we keep only the first one found) + if app in merged_catalog["apps"]: + logger.warning( + "Duplicate app %s found between apps catalog %s and %s" + % (app, apps_catalog_id, merged_catalog["apps"][app]["repository"]) + ) + continue + + info["repository"] = apps_catalog_id + merged_catalog["apps"][app] = info + + # Annnnd categories + merged_catalog["categories"] += apps_catalog_content["categories"] + + return merged_catalog diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index dc5ddbc83..a5e63cd27 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -49,10 +49,6 @@ from yunohost.app import ( app_info, _is_installed, _make_environment_for_app_script, - _patch_legacy_helpers, - _patch_legacy_php_versions, - _patch_legacy_php_versions_in_settings, - LEGACY_PHP_VERSION_REPLACEMENTS, _make_tmp_workdir_for_app, ) from yunohost.hook import ( @@ -1194,6 +1190,7 @@ class RestoreManager: """ Apply dirty patch to redirect php5 and php7.0 files to php7.4 """ + from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS backup_csv = os.path.join(self.work_dir, "backup.csv") @@ -1355,6 +1352,11 @@ class RestoreManager: app_instance_name -- (string) The app name to restore (no app with this name should be already install) """ + from yunohost.utils.legacy import ( + _patch_legacy_php_versions, + _patch_legacy_php_versions_in_settings, + _patch_legacy_helpers, + ) from yunohost.user import user_group_list from yunohost.permission import ( permission_create, @@ -1489,7 +1491,11 @@ class RestoreManager: logger.debug(m18n.n("restore_running_app_script", app=app_instance_name)) # Prepare env. var. to pass to script - env_dict = _make_environment_for_app_script(app_instance_name) + # FIXME : workdir should be a tmp workdir + app_workdir = os.path.join(self.work_dir, "apps", app_instance_name, "settings") + env_dict = _make_environment_for_app_script( + app_instance_name, workdir=app_workdir + ) env_dict.update( { "YNH_BACKUP_DIR": self.work_dir, @@ -1497,9 +1503,6 @@ class RestoreManager: "YNH_APP_BACKUP_DIR": os.path.join( self.work_dir, "apps", app_instance_name, "backup" ), - "YNH_APP_BASEDIR": os.path.join( - self.work_dir, "apps", app_instance_name, "settings" - ), } ) @@ -1536,11 +1539,9 @@ class RestoreManager: remove_script = os.path.join(app_scripts_in_archive, "remove") # Setup environment for remove script - env_dict_remove = _make_environment_for_app_script(app_instance_name) - env_dict_remove["YNH_APP_BASEDIR"] = os.path.join( - self.work_dir, "apps", app_instance_name, "settings" + env_dict_remove = _make_environment_for_app_script( + app_instance_name, workdir=app_workdir ) - remove_operation_logger = OperationLogger( "remove_on_failed_restore", [("app", app_instance_name)], diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index eb9d19c0b..db611f1ce 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -845,14 +845,9 @@ def _backup_current_cert(domain): def _check_domain_is_ready_for_ACME(domain): - dnsrecords = ( - Diagnoser.get_cached_report( - "dnsrecords", - item={"domain": domain, "category": "basic"}, - warn_if_no_cache=False, - ) - or {} - ) + from yunohost.domain import _get_parent_domain_of + from yunohost.dns import _get_dns_zone_for_domain + httpreachable = ( Diagnoser.get_cached_report( "web", item={"domain": domain}, warn_if_no_cache=False @@ -860,16 +855,47 @@ def _check_domain_is_ready_for_ACME(domain): or {} ) - if not dnsrecords or not httpreachable: + parent_domain = _get_parent_domain_of(domain) + + dnsrecords = ( + Diagnoser.get_cached_report( + "dnsrecords", + item={"domain": parent_domain, "category": "basic"}, + warn_if_no_cache=False, + ) + or {} + ) + + base_dns_zone = _get_dns_zone_for_domain(domain) + record_name = ( + domain.replace(f".{base_dns_zone}", "") if domain != base_dns_zone else "@" + ) + A_record_status = dnsrecords.get("data").get(f"A:{record_name}") + AAAA_record_status = dnsrecords.get("data").get(f"AAAA:{record_name}") + + # Fallback to wildcard in case no result yet for the DNS name? + if not A_record_status: + A_record_status = dnsrecords.get("data").get("A:*") + if not AAAA_record_status: + AAAA_record_status = dnsrecords.get("data").get("AAAA:*") + + if ( + not httpreachable + or not dnsrecords.get("data") + or (A_record_status, AAAA_record_status) == (None, None) + ): raise YunohostValidationError( "certmanager_domain_not_diagnosed_yet", domain=domain ) # Check if IP from DNS matches public IP - if not dnsrecords.get("status") in [ - "SUCCESS", - "WARNING", - ]: # Warning is for missing IPv6 record which ain't critical for ACME + # - 'MISSING' for IPv6 ain't critical for ACME + # - IPv4 can be None assuming there's at least an IPv6, and viveversa + # - (the case where both are None is checked before) + if not ( + A_record_status in [None, "OK"] + and AAAA_record_status in [None, "OK", "MISSING"] + ): raise YunohostValidationError( "certmanager_domain_dns_ip_differs_from_public_ip", domain=domain ) diff --git a/src/yunohost/data_migrations/0022_php73_to_php74_pools.py b/src/yunohost/data_migrations/0022_php73_to_php74_pools.py index 0157fa275..fbb180be2 100644 --- a/src/yunohost/data_migrations/0022_php73_to_php74_pools.py +++ b/src/yunohost/data_migrations/0022_php73_to_php74_pools.py @@ -4,7 +4,8 @@ from shutil import copy2 from moulinette.utils.log import getActionLogger -from yunohost.app import _is_installed, _patch_legacy_php_versions_in_settings +from yunohost.app import _is_installed +from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings from yunohost.tools import Migration from yunohost.service import _run_service_command diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 0581fa82c..534ade918 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -32,7 +32,7 @@ from collections import OrderedDict from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_file, write_to_file, read_toml +from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir from yunohost.domain import ( domain_list, @@ -40,8 +40,9 @@ from yunohost.domain import ( domain_config_get, _get_domain_settings, _set_domain_settings, + _list_subdomains_of, ) -from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS +from yunohost.utils.dns import dig, is_yunohost_dyndns_domain, is_special_use_tld from yunohost.utils.error import YunohostValidationError, YunohostError from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation @@ -61,6 +62,9 @@ def domain_dns_suggest(domain): """ + if is_special_use_tld(domain): + return m18n.n("domain_dns_conf_special_use_tld") + _assert_domain_exists(domain) dns_conf = _build_dns_conf(domain) @@ -104,18 +108,6 @@ def domain_dns_suggest(domain): return result -def _list_subdomains_of(parent_domain): - - _assert_domain_exists(parent_domain) - - out = [] - for domain in domain_list()["domains"]: - if domain.endswith(f".{parent_domain}"): - out.append(domain) - - return out - - def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): """ Internal function that will returns a data structure containing the needed @@ -169,10 +161,7 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): # If this is a ynh_dyndns_domain, we're not gonna include all the subdomains in the conf # Because dynette only accept a specific list of name/type # And the wildcard */A already covers the bulk of use cases - if any( - base_domain.endswith("." + ynh_dyndns_domain) - for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS - ): + if is_yunohost_dyndns_domain(base_domain): subdomains = [] else: subdomains = _list_subdomains_of(base_domain) @@ -297,6 +286,12 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): # Defined by custom hooks ships in apps for example ... + # FIXME : this ain't practical for apps that may want to add + # custom dns records for a subdomain ... there's no easy way for + # an app to compare the base domain is the parent of the subdomain ? + # (On the other hand, in sep 2021, it looks like no app is using + # this mechanism...) + hook_results = hook_callback("custom_dns_rules", args=[base_domain]) for hook_name, results in hook_results.items(): # @@ -426,9 +421,14 @@ def _get_dns_zone_for_domain(domain): # First, check if domain is a nohost.me / noho.st / ynh.fr # This is mainly meant to speed up things for "dyndns update" # ... otherwise we end up constantly doing a bunch of dig requests - for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS: - if domain.endswith("." + ynh_dyndns_domain): - return ynh_dyndns_domain + if is_yunohost_dyndns_domain(domain): + # Keep only foo.nohost.me even if we have subsub.sub.foo.nohost.me + return ".".join(domain.rsplit(".", 3)[-3:]) + + # Same thing with .local, .test, ... domains + if is_special_use_tld(domain): + # Keep only foo.local even if we have subsub.sub.foo.local + return ".".join(domain.rsplit(".", 2)[-2:]) # Check cache cache_folder = "/var/cache/yunohost/dns_zones" @@ -471,7 +471,7 @@ def _get_dns_zone_for_domain(domain): # Check if there's a NS record for that domain answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external") if answer[0] == "ok": - os.system(f"mkdir -p {cache_folder}") + mkdir(cache_folder, parents=True, force=True) write_to_file(cache_file, parent) return parent @@ -520,7 +520,7 @@ def _get_registrar_config_section(domain): # TODO big project, integrate yunohost's dynette as a registrar-like provider # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... - if dns_zone in YNH_DYNDNS_DOMAINS: + if is_yunohost_dyndns_domain(dns_zone): registrar_infos["registrar"] = OrderedDict( { "type": "alert", @@ -530,6 +530,15 @@ def _get_registrar_config_section(domain): } ) return OrderedDict(registrar_infos) + elif is_special_use_tld(dns_zone): + registrar_infos["registrar"] = OrderedDict( + { + "type": "alert", + "style": "info", + "ask": m18n.n("domain_dns_conf_special_use_tld"), + "value": None, + } + ) try: registrar = _relevant_provider_for_domain(dns_zone)[0] @@ -603,6 +612,10 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= _assert_domain_exists(domain) + if is_special_use_tld(domain): + logger.info(m18n.n("domain_dns_conf_special_use_tld")) + return {} + if not registrar or registrar == "None": # yes it's None as a string raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 1f96ced8a..b40831d25 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,7 +29,7 @@ from typing import Dict, Any from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml +from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm from yunohost.app import ( app_ssowatconf, @@ -105,6 +105,33 @@ def _assert_domain_exists(domain): raise YunohostValidationError("domain_name_unknown", domain=domain) +def _list_subdomains_of(parent_domain): + + _assert_domain_exists(parent_domain) + + out = [] + for domain in domain_list()["domains"]: + if domain.endswith(f".{parent_domain}"): + out.append(domain) + + return out + + +def _get_parent_domain_of(domain): + + _assert_domain_exists(domain) + + if "." not in domain: + return domain + + parent_domain = domain.split(".", 1)[-1] + if parent_domain not in domain_list()["domains"]: + return domain # Domain is its own parent + + else: + return _get_parent_domain_of(parent_domain) + + @is_unit_operation() def domain_add(operation_logger, domain, dyndns=False): """ @@ -301,7 +328,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): ] for stuff in stuff_to_delete: - os.system("rm -rf {stuff}") + rm(stuff, force=True, recursive=True) # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index bd462c468..bc0745fa1 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -33,7 +33,7 @@ import subprocess from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file, read_file +from moulinette.utils.filesystem import write_to_file, read_file, rm, chown, chmod from moulinette.utils.network import download_json from yunohost.utils.error import YunohostError, YunohostValidationError @@ -152,13 +152,12 @@ def dyndns_subscribe( os.system( "cd /etc/yunohost/dyndns && " - "dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s" - % domain - ) - os.system( - "chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private" + f"dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER {domain}" ) + chmod("/etc/yunohost/dyndns", 0o600, recursive=True) + chown("/etc/yunohost/dyndns", "root", recursive=True) + private_file = glob.glob("/etc/yunohost/dyndns/*%s*.private" % domain)[0] key_file = glob.glob("/etc/yunohost/dyndns/*%s*.key" % domain)[0] with open(key_file) as f: @@ -175,12 +174,12 @@ def dyndns_subscribe( timeout=30, ) except Exception as e: - os.system("rm -f %s" % private_file) - os.system("rm -f %s" % key_file) + rm(private_file, force=True) + rm(key_file, force=True) raise YunohostError("dyndns_registration_failed", error=str(e)) if r.status_code != 201: - os.system("rm -f %s" % private_file) - os.system("rm -f %s" % key_file) + rm(private_file, force=True) + rm(key_file, force=True) try: error = json.loads(r.text)["error"] except Exception: diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index 4be6810ec..a1c0b187f 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -31,7 +31,6 @@ from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import process from moulinette.utils.log import getActionLogger -from moulinette.utils.text import prependlines FIREWALL_FILE = "/etc/yunohost/firewall.yml" UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp" @@ -240,7 +239,7 @@ def firewall_reload(skip_upnp=False): except process.CalledProcessError as e: logger.debug( "iptables seems to be not available, it outputs:\n%s", - prependlines(e.output.rstrip(), "> "), + e.output.decode().strip(), ) logger.warning(m18n.n("iptables_unavailable")) else: @@ -273,7 +272,7 @@ def firewall_reload(skip_upnp=False): except process.CalledProcessError as e: logger.debug( "ip6tables seems to be not available, it outputs:\n%s", - prependlines(e.output.rstrip(), "> "), + e.output.decode().strip(), ) logger.warning(m18n.n("ip6tables_unavailable")) else: @@ -526,6 +525,6 @@ def _on_rule_command_error(returncode, cmd, output): '"%s" returned non-zero exit status %d:\n%s', cmd, returncode, - prependlines(output.rstrip(), "> "), + output.decode().strip(), ) return True diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index c55809fce..20757bf3c 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -34,7 +34,7 @@ from importlib import import_module from moulinette import m18n, Moulinette from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import log -from moulinette.utils.filesystem import read_yaml +from moulinette.utils.filesystem import read_yaml, cp HOOK_FOLDER = "/usr/share/yunohost/hooks/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" @@ -60,8 +60,7 @@ def hook_add(app, file): os.makedirs(CUSTOM_HOOK_FOLDER + action) finalpath = CUSTOM_HOOK_FOLDER + action + "/" + priority + "-" + app - os.system("cp %s %s" % (file, finalpath)) - os.system("chown -hR admin: %s" % HOOK_FOLDER) + cp(file, finalpath) return {"hook": finalpath} diff --git a/src/yunohost/tests/test_appscatalog.py b/src/yunohost/tests/test_app_catalog.py similarity index 99% rename from src/yunohost/tests/test_appscatalog.py rename to src/yunohost/tests/test_app_catalog.py index a2619a660..8423b868e 100644 --- a/src/yunohost/tests/test_appscatalog.py +++ b/src/yunohost/tests/test_app_catalog.py @@ -9,7 +9,7 @@ from moulinette import m18n from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml from yunohost.utils.error import YunohostError -from yunohost.app import ( +from yunohost.app_catalog import ( _initialize_apps_catalog_system, _read_apps_catalog_list, _update_apps_catalog, diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py index d705076c4..0eb813672 100644 --- a/src/yunohost/tests/test_app_config.py +++ b/src/yunohost/tests/test_app_config.py @@ -2,9 +2,11 @@ import glob import os import shutil import pytest +from mock import patch from .conftest import get_test_apps_dir +from moulinette import Moulinette from moulinette.utils.filesystem import read_file from yunohost.domain import _get_maindomain @@ -146,7 +148,9 @@ def test_app_config_regular_setting(config_app): assert app_config_get(config_app, "main.components.boolean") == "1" assert app_setting(config_app, "boolean") == "1" - with pytest.raises(YunohostValidationError): + with pytest.raises(YunohostValidationError), patch.object( + os, "isatty", return_value=False + ), patch.object(Moulinette, "prompt", return_value="pwet"): app_config_set(config_app, "main.components.boolean", "pwet") diff --git a/src/yunohost/tests/test_appurl.py b/src/yunohost/tests/test_appurl.py index f15ed391f..186b76cdf 100644 --- a/src/yunohost/tests/test_appurl.py +++ b/src/yunohost/tests/test_appurl.py @@ -4,7 +4,7 @@ import os from .conftest import get_test_apps_dir from yunohost.utils.error import YunohostError -from yunohost.app import app_install, app_remove, _normalize_domain_path +from yunohost.app import app_install, app_remove, _is_app_repo_url from yunohost.domain import _get_maindomain, domain_url_available from yunohost.permission import _validate_and_sanitize_permission_url @@ -28,20 +28,42 @@ def teardown_function(function): pass -def test_normalize_domain_path(): +def test_repo_url_definition(): + assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh") + assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh/") + assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh.git") + assert _is_app_repo_url( + "https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing" + ) + assert _is_app_repo_url( + "https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing/" + ) + assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo-bar-123_ynh") + assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo_bar_123_ynh") + assert _is_app_repo_url("https://github.com/YunoHost-Apps/FooBar123_ynh") + assert _is_app_repo_url("https://github.com/labriqueinternet/vpnclient_ynh") + assert _is_app_repo_url("https://framagit.org/YunoHost/apps/nodebb_ynh") + assert _is_app_repo_url( + "https://framagit.org/YunoHost/apps/nodebb_ynh/-/tree/testing" + ) + assert _is_app_repo_url("https://gitlab.com/yunohost-apps/foobar_ynh") + assert _is_app_repo_url("https://code.antopie.org/miraty/qr_ynh") + assert _is_app_repo_url( + "https://gitlab.domainepublic.net/Neutrinet/neutrinet_ynh/-/tree/unstable" + ) + assert _is_app_repo_url("git@github.com:YunoHost-Apps/foobar_ynh.git") - assert _normalize_domain_path("https://yolo.swag/", "macnuggets") == ( - "yolo.swag", - "/macnuggets", - ) - assert _normalize_domain_path("http://yolo.swag", "/macnuggets/") == ( - "yolo.swag", - "/macnuggets", - ) - assert _normalize_domain_path("yolo.swag/", "macnuggets/") == ( - "yolo.swag", - "/macnuggets", + assert not _is_app_repo_url("github.com/YunoHost-Apps/foobar_ynh") + assert not _is_app_repo_url("http://github.com/YunoHost-Apps/foobar_ynh") + assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_wat") + assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_ynh_wat") + assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar/tree/testing") + assert not _is_app_repo_url( + "https://github.com/YunoHost-Apps/foobar_ynh_wat/tree/testing" ) + assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/") + assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet") + assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet_foo") def test_urlavailable(): diff --git a/src/yunohost/tests/test_dns.py b/src/yunohost/tests/test_dns.py index 497cab2fd..a23ac7982 100644 --- a/src/yunohost/tests/test_dns.py +++ b/src/yunohost/tests/test_dns.py @@ -34,8 +34,13 @@ def test_get_dns_zone_from_domain_existing(): assert ( _get_dns_zone_for_domain("non-existing-domain.yunohost.org") == "yunohost.org" ) - assert _get_dns_zone_for_domain("yolo.nohost.me") == "nohost.me" - assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "nohost.me" + assert _get_dns_zone_for_domain("yolo.nohost.me") == "yolo.nohost.me" + assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "yolo.nohost.me" + assert _get_dns_zone_for_domain("bar.foo.yolo.nohost.me") == "yolo.nohost.me" + + assert _get_dns_zone_for_domain("yolo.test") == "yolo.test" + assert _get_dns_zone_for_domain("foo.yolo.test") == "yolo.test" + assert _get_dns_zone_for_domain("yolo.tld") == "yolo.tld" assert _get_dns_zone_for_domain("foo.yolo.tld") == "yolo.tld" diff --git a/src/yunohost/tests/test_permission.py b/src/yunohost/tests/test_permission.py index b33c2f213..00799d0fd 100644 --- a/src/yunohost/tests/test_permission.py +++ b/src/yunohost/tests/test_permission.py @@ -1049,7 +1049,7 @@ def test_permission_app_remove(): def test_permission_app_change_url(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), - args="domain=%s&domain_2=%s&path=%s&admin=%s" + args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s" % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), force=True, ) @@ -1072,7 +1072,7 @@ def test_permission_app_change_url(): def test_permission_protection_management_by_helper(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), - args="domain=%s&domain_2=%s&path=%s&admin=%s" + args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s" % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), force=True, ) @@ -1135,7 +1135,7 @@ def test_permission_legacy_app_propagation_on_ssowat(): app_install( os.path.join(get_test_apps_dir(), "legacy_app_ynh"), - args="domain=%s&domain_2=%s&path=%s" + args="domain=%s&domain_2=%s&path=%s&is_public=1" % (maindomain, other_domains[0], "/legacy"), force=True, ) diff --git a/src/yunohost/tests/test_questions.py b/src/yunohost/tests/test_questions.py index 9753b08e4..cf4e67733 100644 --- a/src/yunohost/tests/test_questions.py +++ b/src/yunohost/tests/test_questions.py @@ -2,19 +2,21 @@ import sys import pytest import os -from mock import patch, MagicMock +from mock import patch from io import StringIO -from collections import OrderedDict from moulinette import Moulinette from yunohost import domain, user from yunohost.utils.config import ( - parse_args_in_yunohost_format, + ask_questions_and_parse_answers, PasswordQuestion, - Question, + DomainQuestion, + PathQuestion, + BooleanQuestion, + FileQuestion, ) -from yunohost.utils.error import YunohostError +from yunohost.utils.error import YunohostError, YunohostValidationError """ @@ -41,7 +43,7 @@ User answers: def test_question_empty(): - assert parse_args_in_yunohost_format({}, []) == {} + ask_questions_and_parse_answers([], {}) == [] def test_question_string(): @@ -52,8 +54,29 @@ def test_question_string(): } ] answers = {"some_string": "some_value"} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" + + +def test_question_string_from_query_string(): + + questions = [ + { + "name": "some_string", + "type": "string", + } + ] + answers = "foo=bar&some_string=some_value&lorem=ipsum" + + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_default_type(): @@ -63,8 +86,12 @@ def test_question_string_default_type(): } ] answers = {"some_string": "some_value"} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_no_input(): @@ -76,7 +103,7 @@ def test_question_string_no_input(): answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_string_input(): @@ -87,12 +114,15 @@ def test_question_string_input(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_input_no_ask(): @@ -102,12 +132,15 @@ def test_question_string_input_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_no_input_optional(): @@ -118,9 +151,12 @@ def test_question_string_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("", "string")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "" def test_question_string_optional_with_input(): @@ -132,12 +168,15 @@ def test_question_string_optional_with_input(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_optional_with_empty_input(): @@ -149,12 +188,15 @@ def test_question_string_optional_with_empty_input(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("", "string")}) with patch.object(Moulinette, "prompt", return_value=""), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "" def test_question_string_optional_with_input_without_ask(): @@ -165,12 +207,15 @@ def test_question_string_optional_with_input_without_ask(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_no_input_default(): @@ -182,9 +227,12 @@ def test_question_string_no_input_default(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "some_value" def test_question_string_input_test_ask(): @@ -200,13 +248,15 @@ def test_question_string_input_test_ask(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill="", is_multiline=False, + autocomplete=[], + help=None, ) @@ -225,13 +275,15 @@ def test_question_string_input_test_ask_with_default(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill=default_text, is_multiline=False, + autocomplete=[], + help=None, ) @@ -251,7 +303,7 @@ def test_question_string_input_test_ask_with_example(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert example_text in prompt.call_args[1]["message"] @@ -272,7 +324,7 @@ def test_question_string_input_test_ask_with_help(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert help_text in prompt.call_args[1]["message"] @@ -280,18 +332,24 @@ def test_question_string_input_test_ask_with_help(): def test_question_string_with_choice(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} - expected_result = OrderedDict({"some_string": ("fr", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "fr" def test_question_string_with_choice_prompt(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} - expected_result = OrderedDict({"some_string": ("fr", "string")}) with patch.object(Moulinette, "prompt", return_value="fr"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "fr" def test_question_string_with_choice_bad(): @@ -299,7 +357,7 @@ def test_question_string_with_choice_bad(): answers = {"some_string": "bad"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_string_with_choice_ask(): @@ -317,7 +375,7 @@ def test_question_string_with_choice_ask(): with patch.object(Moulinette, "prompt", return_value="ru") as prompt, patch.object( os, "isatty", return_value=True ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] for choice in choices: @@ -334,9 +392,12 @@ def test_question_string_with_choice_default(): } ] answers = {} - expected_result = OrderedDict({"some_string": ("en", "string")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_string" + assert out.type == "string" + assert out.value == "en" def test_question_password(): @@ -347,8 +408,11 @@ def test_question_password(): } ] answers = {"some_password": "some_value"} - expected_result = OrderedDict({"some_password": ("some_value", "password")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "some_value" def test_question_password_no_input(): @@ -361,7 +425,7 @@ def test_question_password_no_input(): answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_password_input(): @@ -373,12 +437,15 @@ def test_question_password_input(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "some_value" def test_question_password_input_no_ask(): @@ -389,12 +456,15 @@ def test_question_password_input_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "some_value" def test_question_password_no_input_optional(): @@ -406,17 +476,24 @@ def test_question_password_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("", "password")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "" questions = [ {"name": "some_password", "type": "password", "optional": True, "default": ""} ] with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "" def test_question_password_optional_with_input(): @@ -429,12 +506,15 @@ def test_question_password_optional_with_input(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "some_value" def test_question_password_optional_with_empty_input(): @@ -447,12 +527,15 @@ def test_question_password_optional_with_empty_input(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("", "password")}) with patch.object(Moulinette, "prompt", return_value=""), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "" def test_question_password_optional_with_input_without_ask(): @@ -464,12 +547,15 @@ def test_question_password_optional_with_input_without_ask(): } ] answers = {} - expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_password" + assert out.type == "password" + assert out.value == "some_value" def test_question_password_no_input_default(): @@ -485,7 +571,7 @@ def test_question_password_no_input_default(): # no default for password! with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) @pytest.mark.skip # this should raises @@ -502,7 +588,7 @@ def test_question_password_no_input_example(): # no example for password! with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_password_input_test_ask(): @@ -519,13 +605,15 @@ def test_question_password_input_test_ask(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=True, confirm=False, prefill="", is_multiline=False, + autocomplete=[], + help=None, ) @@ -546,7 +634,7 @@ def test_question_password_input_test_ask_with_example(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert example_text in prompt.call_args[1]["message"] @@ -568,7 +656,7 @@ def test_question_password_input_test_ask_with_help(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert help_text in prompt.call_args[1]["message"] @@ -587,7 +675,7 @@ def test_question_password_bad_chars(): with pytest.raises(YunohostError), patch.object( os, "isatty", return_value=False ): - parse_args_in_yunohost_format({"some_password": i * 8}, questions) + ask_questions_and_parse_answers(questions, {"some_password": i * 8}) def test_question_password_strong_enough(): @@ -602,10 +690,10 @@ def test_question_password_strong_enough(): with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): # too short - parse_args_in_yunohost_format({"some_password": "a"}, questions) + ask_questions_and_parse_answers(questions, {"some_password": "a"}) with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format({"some_password": "password"}, questions) + ask_questions_and_parse_answers(questions, {"some_password": "password"}) def test_question_password_optional_strong_enough(): @@ -620,10 +708,10 @@ def test_question_password_optional_strong_enough(): with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): # too short - parse_args_in_yunohost_format({"some_password": "a"}, questions) + ask_questions_and_parse_answers(questions, {"some_password": "a"}) with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format({"some_password": "password"}, questions) + ask_questions_and_parse_answers(questions, {"some_password": "password"}) def test_question_path(): @@ -633,9 +721,12 @@ def test_question_path(): "type": "path", } ] - answers = {"some_path": "some_value"} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + answers = {"some_path": "/some_value"} + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_no_input(): @@ -648,7 +739,7 @@ def test_question_path_no_input(): answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_path_input(): @@ -660,12 +751,15 @@ def test_question_path_input(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( + with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_input_no_ask(): @@ -676,12 +770,15 @@ def test_question_path_input_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( + with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_no_input_optional(): @@ -693,9 +790,12 @@ def test_question_path_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("", "path")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "" def test_question_path_optional_with_input(): @@ -708,12 +808,15 @@ def test_question_path_optional_with_input(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( + with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_optional_with_empty_input(): @@ -726,12 +829,15 @@ def test_question_path_optional_with_empty_input(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("", "path")}) with patch.object(Moulinette, "prompt", return_value=""), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "" def test_question_path_optional_with_input_without_ask(): @@ -743,12 +849,15 @@ def test_question_path_optional_with_input_without_ask(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( + with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_no_input_default(): @@ -761,9 +870,12 @@ def test_question_path_no_input_default(): } ] answers = {} - expected_result = OrderedDict({"some_path": ("some_value", "path")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_path" + assert out.type == "path" + assert out.value == "/some_value" def test_question_path_input_test_ask(): @@ -780,19 +892,21 @@ def test_question_path_input_test_ask(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill="", is_multiline=False, + autocomplete=[], + help=None, ) def test_question_path_input_test_ask_with_default(): ask_text = "some question" - default_text = "some example" + default_text = "someexample" questions = [ { "name": "some_path", @@ -806,13 +920,15 @@ def test_question_path_input_test_ask_with_default(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill=default_text, is_multiline=False, + autocomplete=[], + help=None, ) @@ -833,7 +949,7 @@ def test_question_path_input_test_ask_with_example(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert example_text in prompt.call_args[1]["message"] @@ -855,7 +971,7 @@ def test_question_path_input_test_ask_with_help(): with patch.object( Moulinette, "prompt", return_value="some_value" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert help_text in prompt.call_args[1]["message"] @@ -868,8 +984,11 @@ def test_question_boolean(): } ] answers = {"some_boolean": "y"} - expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_boolean" + assert out.type == "boolean" + assert out.value == 1 def test_question_boolean_all_yes(): @@ -879,50 +998,12 @@ def test_question_boolean_all_yes(): "type": "boolean", } ] - expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - assert ( - parse_args_in_yunohost_format({"some_boolean": "y"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "Y"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "yes"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "Yes"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "YES"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "1"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": 1}, questions) == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": True}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "True"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "TRUE"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "true"}, questions) - == expected_result - ) + + for value in ["Y", "yes", "Yes", "YES", "1", 1, True, "True", "TRUE", "true"]: + out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] + assert out.name == "some_boolean" + assert out.type == "boolean" + assert out.value == 1 def test_question_boolean_all_no(): @@ -932,50 +1013,12 @@ def test_question_boolean_all_no(): "type": "boolean", } ] - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - assert ( - parse_args_in_yunohost_format({"some_boolean": "n"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "N"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "no"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "No"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "No"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "0"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": 0}, questions) == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": False}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "False"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "FALSE"}, questions) - == expected_result - ) - assert ( - parse_args_in_yunohost_format({"some_boolean": "false"}, questions) - == expected_result - ) + + for value in ["n", "N", "no", "No", "No", "0", 0, False, "False", "FALSE", "false"]: + out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] + assert out.name == "some_boolean" + assert out.type == "boolean" + assert out.value == 0 # XXX apparently boolean are always False (0) by default, I'm not sure what to think about that @@ -988,9 +1031,10 @@ def test_question_boolean_no_input(): ] answers = {} - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.value == 0 def test_question_boolean_bad_input(): @@ -1003,7 +1047,7 @@ def test_question_boolean_bad_input(): answers = {"some_boolean": "stuff"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_boolean_input(): @@ -1016,17 +1060,17 @@ def test_question_boolean_input(): ] answers = {} - expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette, "prompt", return_value="y"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + assert out.value == 1 - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) with patch.object(Moulinette, "prompt", return_value="n"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + assert out.value == 0 def test_question_boolean_input_no_ask(): @@ -1037,12 +1081,12 @@ def test_question_boolean_input_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette, "prompt", return_value="y"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + assert out.value == 1 def test_question_boolean_no_input_optional(): @@ -1054,9 +1098,9 @@ def test_question_boolean_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + assert out.value == 0 def test_question_boolean_optional_with_input(): @@ -1069,12 +1113,12 @@ def test_question_boolean_optional_with_input(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette, "prompt", return_value="y"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + assert out.value == 1 def test_question_boolean_optional_with_empty_input(): @@ -1087,12 +1131,13 @@ def test_question_boolean_optional_with_empty_input(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false with patch.object(Moulinette, "prompt", return_value=""), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.value == 0 def test_question_boolean_optional_with_input_without_ask(): @@ -1104,12 +1149,13 @@ def test_question_boolean_optional_with_input_without_ask(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) with patch.object(Moulinette, "prompt", return_value="n"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.value == 0 def test_question_boolean_no_input_default(): @@ -1122,9 +1168,11 @@ def test_question_boolean_no_input_default(): } ] answers = {} - expected_result = OrderedDict({"some_boolean": (0, "boolean")}) + with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.value == 0 def test_question_boolean_bad_default(): @@ -1138,7 +1186,7 @@ def test_question_boolean_bad_default(): ] answers = {} with pytest.raises(YunohostError): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_boolean_input_test_ask(): @@ -1155,13 +1203,15 @@ def test_question_boolean_input_test_ask(): with patch.object(Moulinette, "prompt", return_value=0) as prompt, patch.object( os, "isatty", return_value=True ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text + " [yes | no]", is_password=False, confirm=False, prefill="no", is_multiline=False, + autocomplete=[], + help=None, ) @@ -1181,13 +1231,15 @@ def test_question_boolean_input_test_ask_with_default(): with patch.object(Moulinette, "prompt", return_value=1) as prompt, patch.object( os, "isatty", return_value=True ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text + " [yes | no]", is_password=False, confirm=False, prefill="yes", is_multiline=False, + autocomplete=[], + help=None, ) @@ -1199,7 +1251,6 @@ def test_question_domain_empty(): } ] main_domain = "my_main_domain.com" - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) answers = {} with patch.object( @@ -1209,7 +1260,11 @@ def test_question_domain_empty(): ), patch.object( os, "isatty", return_value=False ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain def test_question_domain(): @@ -1223,12 +1278,15 @@ def test_question_domain(): ] answers = {"some_domain": main_domain} - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain def test_question_domain_two_domains(): @@ -1243,20 +1301,26 @@ def test_question_domain_two_domains(): } ] answers = {"some_domain": other_domain} - expected_result = OrderedDict({"some_domain": (other_domain, "domain")}) with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == other_domain answers = {"some_domain": main_domain} - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain def test_question_domain_two_domains_wrong_answer(): @@ -1278,7 +1342,7 @@ def test_question_domain_two_domains_wrong_answer(): with pytest.raises(YunohostError), patch.object( os, "isatty", return_value=False ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_domain_two_domains_default_no_ask(): @@ -1293,7 +1357,6 @@ def test_question_domain_two_domains_default_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( domain, "_get_maindomain", return_value=main_domain @@ -1302,7 +1365,11 @@ def test_question_domain_two_domains_default_no_ask(): ), patch.object( os, "isatty", return_value=False ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain def test_question_domain_two_domains_default(): @@ -1312,7 +1379,6 @@ def test_question_domain_two_domains_default(): questions = [{"name": "some_domain", "type": "domain", "ask": "choose a domain"}] answers = {} - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( domain, "_get_maindomain", return_value=main_domain @@ -1321,7 +1387,11 @@ def test_question_domain_two_domains_default(): ), patch.object( os, "isatty", return_value=False ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain def test_question_domain_two_domains_default_input(): @@ -1339,13 +1409,19 @@ def test_question_domain_two_domains_default_input(): ), patch.object( os, "isatty", return_value=True ): - expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object(Moulinette, "prompt", return_value=main_domain): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == main_domain - expected_result = OrderedDict({"some_domain": (other_domain, "domain")}) with patch.object(Moulinette, "prompt", return_value=other_domain): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_domain" + assert out.type == "domain" + assert out.value == other_domain def test_question_user_empty(): @@ -1371,7 +1447,7 @@ def test_question_user_empty(): with pytest.raises(YunohostError), patch.object( os, "isatty", return_value=False ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_user(): @@ -1394,12 +1470,14 @@ def test_question_user(): ] answers = {"some_user": username} - expected_result = OrderedDict({"some_user": (username, "user")}) - with patch.object(user, "user_list", return_value={"users": users}), patch.object( user, "user_info", return_value={} ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_user" + assert out.type == "user" + assert out.value == username def test_question_user_two_users(): @@ -1429,20 +1507,26 @@ def test_question_user_two_users(): } ] answers = {"some_user": other_user} - expected_result = OrderedDict({"some_user": (other_user, "user")}) with patch.object(user, "user_list", return_value={"users": users}), patch.object( user, "user_info", return_value={} ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_user" + assert out.type == "user" + assert out.value == other_user answers = {"some_user": username} - expected_result = OrderedDict({"some_user": (username, "user")}) with patch.object(user, "user_list", return_value={"users": users}), patch.object( user, "user_info", return_value={} ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_user" + assert out.type == "user" + assert out.value == username def test_question_user_two_users_wrong_answer(): @@ -1477,7 +1561,7 @@ def test_question_user_two_users_wrong_answer(): with pytest.raises(YunohostError), patch.object( os, "isatty", return_value=False ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_user_two_users_no_default(): @@ -1507,7 +1591,7 @@ def test_question_user_two_users_no_default(): with pytest.raises(YunohostError), patch.object( os, "isatty", return_value=False ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_user_two_users_default_input(): @@ -1537,17 +1621,20 @@ def test_question_user_two_users_default_input(): os, "isatty", return_value=True ): with patch.object(user, "user_info", return_value={}): - expected_result = OrderedDict({"some_user": (username, "user")}) - with patch.object(Moulinette, "prompt", return_value=username): - assert ( - parse_args_in_yunohost_format(answers, questions) == expected_result - ) - expected_result = OrderedDict({"some_user": (other_user, "user")}) + with patch.object(Moulinette, "prompt", return_value=username): + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_user" + assert out.type == "user" + assert out.value == username + with patch.object(Moulinette, "prompt", return_value=other_user): - assert ( - parse_args_in_yunohost_format(answers, questions) == expected_result - ) + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_user" + assert out.type == "user" + assert out.value == other_user def test_question_number(): @@ -1558,8 +1645,11 @@ def test_question_number(): } ] answers = {"some_number": 1337} - expected_result = OrderedDict({"some_number": (1337, "number")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 def test_question_number_no_input(): @@ -1572,7 +1662,7 @@ def test_question_number_no_input(): answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_number_bad_input(): @@ -1585,11 +1675,11 @@ def test_question_number_bad_input(): answers = {"some_number": "stuff"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) answers = {"some_number": 1.5} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_number_input(): @@ -1602,22 +1692,32 @@ def test_question_number_input(): ] answers = {} - expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 with patch.object(Moulinette, "prompt", return_value=1337), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 - expected_result = OrderedDict({"some_number": (0, "number")}) with patch.object(Moulinette, "prompt", return_value="0"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 0 def test_question_number_input_no_ask(): @@ -1628,12 +1728,15 @@ def test_question_number_input_no_ask(): } ] answers = {} - expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 def test_question_number_no_input_optional(): @@ -1645,9 +1748,12 @@ def test_question_number_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_number": (None, "number")}) # default to 0 with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value is None def test_question_number_optional_with_input(): @@ -1660,12 +1766,15 @@ def test_question_number_optional_with_input(): } ] answers = {} - expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 def test_question_number_optional_with_input_without_ask(): @@ -1677,12 +1786,15 @@ def test_question_number_optional_with_input_without_ask(): } ] answers = {} - expected_result = OrderedDict({"some_number": (0, "number")}) with patch.object(Moulinette, "prompt", return_value="0"), patch.object( os, "isatty", return_value=True ): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 0 def test_question_number_no_input_default(): @@ -1695,9 +1807,12 @@ def test_question_number_no_input_default(): } ] answers = {} - expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(os, "isatty", return_value=False): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_number" + assert out.type == "number" + assert out.value == 1337 def test_question_number_bad_default(): @@ -1711,7 +1826,7 @@ def test_question_number_bad_default(): ] answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) def test_question_number_input_test_ask(): @@ -1728,13 +1843,15 @@ def test_question_number_input_test_ask(): with patch.object( Moulinette, "prompt", return_value="1111" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill="", is_multiline=False, + autocomplete=[], + help=None, ) @@ -1754,13 +1871,15 @@ def test_question_number_input_test_ask_with_default(): with patch.object( Moulinette, "prompt", return_value="1111" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) prompt.assert_called_with( message=ask_text, is_password=False, confirm=False, prefill=str(default_value), is_multiline=False, + autocomplete=[], + help=None, ) @@ -1781,7 +1900,7 @@ def test_question_number_input_test_ask_with_example(): with patch.object( Moulinette, "prompt", return_value="1111" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert example_value in prompt.call_args[1]["message"] @@ -1803,7 +1922,7 @@ def test_question_number_input_test_ask_with_help(): with patch.object( Moulinette, "prompt", return_value="1111" ) as prompt, patch.object(os, "isatty", return_value=True): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert ask_text in prompt.call_args[1]["message"] assert help_value in prompt.call_args[1]["message"] @@ -1815,5 +1934,162 @@ def test_question_display_text(): with patch.object(sys, "stdout", new_callable=StringIO) as stdout, patch.object( os, "isatty", return_value=True ): - parse_args_in_yunohost_format(answers, questions) + ask_questions_and_parse_answers(questions, answers) assert "foobar" in stdout.getvalue() + + +def test_question_file_from_cli(): + + FileQuestion.clean_upload_dirs() + + filename = "/tmp/ynh_test_question_file" + os.system(f"rm -f {filename}") + os.system(f"echo helloworld > {filename}") + + questions = [ + { + "name": "some_file", + "type": "file", + } + ] + answers = {"some_file": filename} + + out = ask_questions_and_parse_answers(questions, answers)[0] + + assert out.name == "some_file" + assert out.type == "file" + + # The file is supposed to be copied somewhere else + assert out.value != filename + assert out.value.startswith("/tmp/") + assert os.path.exists(out.value) + assert "helloworld" in open(out.value).read().strip() + + FileQuestion.clean_upload_dirs() + + assert not os.path.exists(out.value) + + +def test_question_file_from_api(): + + FileQuestion.clean_upload_dirs() + + from base64 import b64encode + + b64content = b64encode("helloworld".encode()) + questions = [ + { + "name": "some_file", + "type": "file", + } + ] + answers = {"some_file": b64content} + + interface_type_bkp = Moulinette.interface.type + try: + Moulinette.interface.type = "api" + out = ask_questions_and_parse_answers(questions, answers)[0] + finally: + Moulinette.interface.type = interface_type_bkp + + assert out.name == "some_file" + assert out.type == "file" + + assert out.value.startswith("/tmp/") + assert os.path.exists(out.value) + assert "helloworld" in open(out.value).read().strip() + + FileQuestion.clean_upload_dirs() + + assert not os.path.exists(out.value) + + +def test_normalize_boolean_nominal(): + + assert BooleanQuestion.normalize("yes") == 1 + assert BooleanQuestion.normalize("Yes") == 1 + assert BooleanQuestion.normalize(" yes ") == 1 + assert BooleanQuestion.normalize("y") == 1 + assert BooleanQuestion.normalize("true") == 1 + assert BooleanQuestion.normalize("True") == 1 + assert BooleanQuestion.normalize("on") == 1 + assert BooleanQuestion.normalize("1") == 1 + assert BooleanQuestion.normalize(1) == 1 + + assert BooleanQuestion.normalize("no") == 0 + assert BooleanQuestion.normalize("No") == 0 + assert BooleanQuestion.normalize(" no ") == 0 + assert BooleanQuestion.normalize("n") == 0 + assert BooleanQuestion.normalize("false") == 0 + assert BooleanQuestion.normalize("False") == 0 + assert BooleanQuestion.normalize("off") == 0 + assert BooleanQuestion.normalize("0") == 0 + assert BooleanQuestion.normalize(0) == 0 + + assert BooleanQuestion.normalize("") is None + assert BooleanQuestion.normalize(" ") is None + assert BooleanQuestion.normalize(" none ") is None + assert BooleanQuestion.normalize("None") is None + assert BooleanQuestion.normalize("noNe") is None + assert BooleanQuestion.normalize(None) is None + + +def test_normalize_boolean_humanize(): + + assert BooleanQuestion.humanize("yes") == "yes" + assert BooleanQuestion.humanize("true") == "yes" + assert BooleanQuestion.humanize("on") == "yes" + + assert BooleanQuestion.humanize("no") == "no" + assert BooleanQuestion.humanize("false") == "no" + assert BooleanQuestion.humanize("off") == "no" + + +def test_normalize_boolean_invalid(): + + with pytest.raises(YunohostValidationError): + BooleanQuestion.normalize("yesno") + with pytest.raises(YunohostValidationError): + BooleanQuestion.normalize("foobar") + with pytest.raises(YunohostValidationError): + BooleanQuestion.normalize("enabled") + + +def test_normalize_boolean_special_yesno(): + + customyesno = {"yes": "enabled", "no": "disabled"} + + assert BooleanQuestion.normalize("yes", customyesno) == "enabled" + assert BooleanQuestion.normalize("true", customyesno) == "enabled" + assert BooleanQuestion.normalize("enabled", customyesno) == "enabled" + assert BooleanQuestion.humanize("yes", customyesno) == "yes" + assert BooleanQuestion.humanize("true", customyesno) == "yes" + assert BooleanQuestion.humanize("enabled", customyesno) == "yes" + + assert BooleanQuestion.normalize("no", customyesno) == "disabled" + assert BooleanQuestion.normalize("false", customyesno) == "disabled" + assert BooleanQuestion.normalize("disabled", customyesno) == "disabled" + assert BooleanQuestion.humanize("no", customyesno) == "no" + assert BooleanQuestion.humanize("false", customyesno) == "no" + assert BooleanQuestion.humanize("disabled", customyesno) == "no" + + +def test_normalize_domain(): + + assert DomainQuestion.normalize("https://yolo.swag/") == "yolo.swag" + assert DomainQuestion.normalize("http://yolo.swag") == "yolo.swag" + assert DomainQuestion.normalize("yolo.swag/") == "yolo.swag" + + +def test_normalize_path(): + + assert PathQuestion.normalize("") == "/" + assert PathQuestion.normalize("") == "/" + assert PathQuestion.normalize("macnuggets") == "/macnuggets" + assert PathQuestion.normalize("/macnuggets") == "/macnuggets" + assert PathQuestion.normalize(" /macnuggets ") == "/macnuggets" + assert PathQuestion.normalize("/macnuggets") == "/macnuggets" + assert PathQuestion.normalize("mac/nuggets") == "/mac/nuggets" + assert PathQuestion.normalize("/macnuggets/") == "/macnuggets" + assert PathQuestion.normalize("macnuggets/") == "/macnuggets" + assert PathQuestion.normalize("////macnuggets///") == "/macnuggets" diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8c113aee6..799f736f9 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -34,13 +34,15 @@ from typing import List from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_yaml, write_to_yaml +from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm from yunohost.app import ( - _update_apps_catalog, app_info, app_upgrade, +) +from yunohost.app_catalog import ( _initialize_apps_catalog_system, + _update_apps_catalog, ) from yunohost.domain import domain_add from yunohost.dyndns import _dyndns_available, _dyndns_provides @@ -1118,12 +1120,14 @@ class Migration(object): backup_folder = "/home/yunohost.backup/premigration/" + time.strftime( "%Y%m%d-%H%M%S", time.gmtime() ) - os.makedirs(backup_folder, 0o750) + mkdir(backup_folder, 0o750, parents=True) os.system("systemctl stop slapd") - os.system(f"cp -r --preserve /etc/ldap {backup_folder}/ldap_config") - os.system(f"cp -r --preserve /var/lib/ldap {backup_folder}/ldap_db") - os.system( - f"cp -r --preserve /etc/yunohost/apps {backup_folder}/apps_settings" + cp("/etc/ldap", f"{backup_folder}/ldap_config", recursive=True) + cp("/var/lib/ldap", f"{backup_folder}/ldap_db", recursive=True) + cp( + "/etc/yunohost/apps", + f"{backup_folder}/apps_settings", + recursive=True, ) except Exception as e: raise YunohostError( @@ -1140,17 +1144,19 @@ class Migration(object): ) os.system("systemctl stop slapd") # To be sure that we don't keep some part of the old config - os.system("rm -r /etc/ldap/slapd.d") - os.system(f"cp -r --preserve {backup_folder}/ldap_config/. /etc/ldap/") - os.system(f"cp -r --preserve {backup_folder}/ldap_db/. /var/lib/ldap/") - os.system( - f"cp -r --preserve {backup_folder}/apps_settings/. /etc/yunohost/apps/" + rm("/etc/ldap/slapd.d", force=True, recursive=True) + cp(f"{backup_folder}/ldap_config", "/etc/ldap", recursive=True) + cp(f"{backup_folder}/ldap_db", "/var/lib/ldap", recursive=True) + cp( + f"{backup_folder}/apps_settings", + "/etc/yunohost/apps", + recursive=True, ) os.system("systemctl start slapd") - os.system(f"rm -r {backup_folder}") + rm(backup_folder, force=True, recursive=True) logger.info(m18n.n("migration_ldap_rollback_success")) raise else: - os.system(f"rm -r {backup_folder}") + rm(backup_folder, force=True, recursive=True) return func diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 413919d61..8bedfb5f6 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -413,7 +413,9 @@ def user_update( # without a specified value, change_password will be set to the const 0. # In this case we prompt for the new password. if Moulinette.interface.type == "cli" and not change_password: - change_password = Moulinette.prompt(m18n.n("ask_password"), True, True) + change_password = Moulinette.prompt( + m18n.n("ask_password"), is_password=True, confirm=True + ) # Ensure sufficiently complex password assert_password_is_strong_enough("user", change_password) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 99c898d15..27a9e1533 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -25,12 +25,13 @@ import urllib.parse import tempfile import shutil from collections import OrderedDict -from typing import Optional, Dict, List +from typing import Optional, Dict, List, Union, Any, Mapping from moulinette.interfaces.cli import colorize from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( + read_file, write_to_file, read_toml, read_yaml, @@ -99,6 +100,11 @@ class ConfigPanel: result[key]["value"] = question_class.humanize( option["current_value"], option ) + # FIXME: semantics, technically here this is not about a prompt... + if question_class.hide_user_input_in_prompt: + result[key][ + "value" + ] = "**************" # Prevent displaying password in `config get` if mode == "full": return self.config @@ -164,6 +170,9 @@ class ConfigPanel: raise finally: # Delete files uploaded from API + # FIXME : this is currently done in the context of config panels, + # but could also happen in the context of app install ... (or anywhere else + # where we may parse args etc...) FileQuestion.clean_upload_dirs() self._reload_services() @@ -198,20 +207,20 @@ class ConfigPanel: # Transform toml format into internal format format_description = { - "toml": { + "root": { "properties": ["version", "i18n"], - "default": {"version": 1.0}, + "defaults": {"version": 1.0}, }, "panels": { "properties": ["name", "services", "actions", "help"], - "default": { + "defaults": { "services": [], "actions": {"apply": {"en": "Apply"}}, }, }, "sections": { "properties": ["name", "services", "optional", "help", "visible"], - "default": { + "defaults": { "name": "", "services": [], "optional": True, @@ -241,11 +250,11 @@ class ConfigPanel: "accept", "redact", ], - "default": {}, + "defaults": {}, }, } - def convert(toml_node, node_type): + def _build_internal_config_panel(raw_infos, level): """Convert TOML in internal format ('full' mode used by webadmin) Here are some properties of 1.0 config panel in toml: - node properties and node children are mixed, @@ -253,48 +262,47 @@ class ConfigPanel: - some properties have default values This function detects all children nodes and put them in a list """ - # Prefill the node default keys if needed - default = format_description[node_type]["default"] - node = {key: toml_node.get(key, value) for key, value in default.items()} - properties = format_description[node_type]["properties"] + defaults = format_description[level]["defaults"] + properties = format_description[level]["properties"] - # Define the filter_key part to use and the children type - i = list(format_description).index(node_type) - subnode_type = ( - list(format_description)[i + 1] if node_type != "options" else None - ) + # Start building the ouput (merging the raw infos + defaults) + out = {key: raw_infos.get(key, value) for key, value in defaults.items()} + + # Now fill the sublevels (+ apply filter_key) + i = list(format_description).index(level) + sublevel = list(format_description)[i + 1] if level != "options" else None search_key = filter_key[i] if len(filter_key) > i else False - for key, value in toml_node.items(): + for key, value in raw_infos.items(): # Key/value are a child node if ( isinstance(value, OrderedDict) and key not in properties - and subnode_type + and sublevel ): # We exclude all nodes not referenced by the filter_key if search_key and key != search_key: continue - subnode = convert(value, subnode_type) + subnode = _build_internal_config_panel(value, sublevel) subnode["id"] = key - if node_type == "toml": + if level == "root": subnode.setdefault("name", {"en": key.capitalize()}) - elif node_type == "sections": + elif level == "sections": subnode["name"] = key # legacy - subnode.setdefault("optional", toml_node.get("optional", True)) - node.setdefault(subnode_type, []).append(subnode) + subnode.setdefault("optional", raw_infos.get("optional", True)) + out.setdefault(sublevel, []).append(subnode) # Key/value are a property else: if key not in properties: - logger.warning(f"Unknown key '{key}' found in config toml") + logger.warning(f"Unknown key '{key}' found in config panel") # Todo search all i18n keys - node[key] = ( + out[key] = ( value if key not in ["ask", "help", "name"] else {"en": value} ) - return node + return out - self.config = convert(toml_config_panel, "toml") + self.config = _build_internal_config_panel(toml_config_panel, "root") try: self.config["panels"][0]["sections"][0]["options"][0] @@ -376,14 +384,15 @@ class ConfigPanel: display_header(f"\n# {name}") # Check and ask unanswered questions + questions = ask_questions_and_parse_answers(section["options"], self.args) self.new_values.update( - parse_args_in_yunohost_format(self.args, section["options"]) + { + question.name: question.value + for question in questions + if question.value is not None + } ) - self.new_values = { - key: value[0] - for key, value in self.new_values.items() - if not value[0] is None - } + self.errors = None def _get_default_values(self): @@ -457,18 +466,20 @@ class Question(object): hide_user_input_in_prompt = False pattern: Optional[Dict] = None - def __init__(self, question, user_answers): + def __init__(self, question: Dict[str, Any]): self.name = question["name"] self.type = question.get("type", "string") self.default = question.get("default", None) - self.current_value = question.get("current_value") self.optional = question.get("optional", False) self.choices = question.get("choices", []) self.pattern = question.get("pattern", self.pattern) self.ask = question.get("ask", {"en": self.name}) self.help = question.get("help") - self.value = user_answers.get(self.name) self.redact = question.get("redact", False) + # .current_value is the currently stored value + self.current_value = question.get("current_value") + # .value is the "proposed" value which we got from the user + self.value = question.get("value") # Empty value is parsed as empty string if self.default == "": @@ -480,6 +491,8 @@ class Question(object): @staticmethod def normalize(value, option={}): + if isinstance(value, str): + value = value.strip() return value def _prompt(self, text): @@ -491,9 +504,11 @@ class Question(object): self.value = Moulinette.prompt( message=text, is_password=self.hide_user_input_in_prompt, - confirm=False, # We doesn't want to confirm this kind of password like in webadmin + confirm=False, prefill=prefill, is_multiline=(self.type == "text"), + autocomplete=self.choices, + help=_value_for_locale(self.help), ) def ask_if_needed(self): @@ -513,12 +528,9 @@ class Question(object): ): self.value = class_default if self.default is None else self.default - # Normalization - # This is done to enforce a certain formating like for boolean - self.value = self.normalize(self.value, self) - - # Prevalidation try: + # Normalize and validate + self.value = self.normalize(self.value, self) self._prevalidate() except YunohostValidationError as e: # If in interactive cli, re-ask the current question @@ -531,9 +543,10 @@ class Question(object): raise break + self.value = self._post_parse_value() - return (self.value, self.argument_type) + return self.value def _prevalidate(self): if self.value in [None, ""] and not self.optional: @@ -542,7 +555,12 @@ class Question(object): # we have an answer, do some post checks if self.value not in [None, ""]: if self.choices and self.value not in self.choices: - self._raise_invalid_answer() + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices=", ".join(self.choices), + ) if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): raise YunohostValidationError( self.pattern["error"], @@ -550,25 +568,31 @@ class Question(object): value=self.value, ) - def _raise_invalid_answer(self): - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.name, - value=self.value, - choices=", ".join(self.choices), - ) + def _format_text_for_user_input_in_cli(self): - def _format_text_for_user_input_in_cli(self, column=False): text_for_user_input_in_cli = _value_for_locale(self.ask) if self.choices: - text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices)) - if self.help or column: - text_for_user_input_in_cli += ":\033[m" - if self.help: - text_for_user_input_in_cli += "\n - " - text_for_user_input_in_cli += _value_for_locale(self.help) + # Prevent displaying a shitload of choices + # (e.g. 100+ available users when choosing an app admin...) + choices = ( + list(self.choices.values()) + if isinstance(self.choices, dict) + else self.choices + ) + choices_to_display = choices[:20] + remaining_choices = len(choices[20:]) + + if remaining_choices > 0: + choices_to_display += [ + m18n.n("other_available_options", n=remaining_choices) + ] + + choices_to_display = " | ".join(choices_to_display) + + text_for_user_input_in_cli += f" [{choices_to_display}]" + return text_for_user_input_in_cli def _post_parse_value(self): @@ -659,6 +683,8 @@ class TagsQuestion(Question): def normalize(value, option={}): if isinstance(value, list): return ",".join(value) + if isinstance(value, str): + value = value.strip() return value def _prevalidate(self): @@ -684,20 +710,14 @@ class PasswordQuestion(Question): default_value = "" forbidden_chars = "{}" - def __init__(self, question, user_answers): - super().__init__(question, user_answers) + def __init__(self, question): + super().__init__(question) self.redact = True if self.default is not None: raise YunohostValidationError( "app_argument_password_no_default", name=self.name ) - @staticmethod - def humanize(value, option={}): - if value: - return "********" # Avoid to display the password on screen - return "" - def _prevalidate(self): super()._prevalidate() @@ -712,34 +732,31 @@ class PasswordQuestion(Question): assert_password_is_strong_enough("user", self.value) - def _format_text_for_user_input_in_cli(self): - need_column = self.current_value or self.optional - text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli( - need_column - ) - if self.current_value: - text_for_user_input_in_cli += "\n - " + m18n.n( - "app_argument_password_help_keep" - ) - if self.optional: - text_for_user_input_in_cli += "\n - " + m18n.n( - "app_argument_password_help_optional" - ) - - return text_for_user_input_in_cli - - def _prompt(self, text): - super()._prompt(text) - if self.current_value and self.value == "": - self.value = self.current_value - elif self.value == " ": - self.value = "" - class PathQuestion(Question): argument_type = "path" default_value = "" + @staticmethod + def normalize(value, option={}): + + option = option.__dict__ if isinstance(option, Question) else option + + if not value.strip(): + if option.get("optional"): + return "" + # Hmpf here we could just have a "else" case + # but we also want PathQuestion.normalize("") to return "/" + # (i.e. if no option is provided, hence .get("optional") is None + elif option.get("optional") is False: + raise YunohostValidationError( + "app_argument_invalid", + name=option.get("name"), + error="Question is mandatory", + ) + + return "/" + value.strip().strip(" /") + class BooleanQuestion(Question): argument_type = "boolean" @@ -750,50 +767,70 @@ class BooleanQuestion(Question): @staticmethod def humanize(value, option={}): + option = option.__dict__ if isinstance(option, Question) else option + yes = option.get("yes", 1) no = option.get("no", 0) - value = str(value).lower() - if value == str(yes).lower(): - return "yes" - if value == str(no).lower(): - return "no" - if value in BooleanQuestion.yes_answers: - return "yes" - if value in BooleanQuestion.no_answers: - return "no" - if value in ["none", ""]: + value = BooleanQuestion.normalize(value, option) + + if value == yes: + return "yes" + if value == no: + return "no" + if value is None: return "" raise YunohostValidationError( "app_argument_choice_invalid", - name=option.get("name", ""), + name=option.get("name"), value=value, - choices="yes, no, y, n, 1, 0", + choices="yes/no", ) @staticmethod def normalize(value, option={}): - yes = option.get("yes", 1) - no = option.get("no", 0) - if str(value).lower() in BooleanQuestion.yes_answers: - return yes + option = option.__dict__ if isinstance(option, Question) else option - if str(value).lower() in BooleanQuestion.no_answers: - return no + if isinstance(value, str): + value = value.strip() - if value in [None, ""]: + technical_yes = option.get("yes", 1) + technical_no = option.get("no", 0) + + no_answers = BooleanQuestion.no_answers + yes_answers = BooleanQuestion.yes_answers + + assert ( + str(technical_yes).lower() not in no_answers + ), f"'yes' value can't be in {no_answers}" + assert ( + str(technical_no).lower() not in yes_answers + ), f"'no' value can't be in {yes_answers}" + + no_answers += [str(technical_no).lower()] + yes_answers += [str(technical_yes).lower()] + + strvalue = str(value).lower() + + if strvalue in yes_answers: + return technical_yes + if strvalue in no_answers: + return technical_no + + if strvalue in ["none", ""]: return None + raise YunohostValidationError( "app_argument_choice_invalid", - name=option.get("name", ""), - value=value, - choices="yes, no, y, n, 1, 0", + name=option.get("name"), + value=strvalue, + choices="yes/no", ) - def __init__(self, question, user_answers): - super().__init__(question, user_answers) + def __init__(self, question): + super().__init__(question) self.yes = question.get("yes", 1) self.no = question.get("no", 0) if self.default is None: @@ -807,42 +844,44 @@ class BooleanQuestion(Question): return text_for_user_input_in_cli def get(self, key, default=None): - try: - return getattr(self, key) - except AttributeError: - return default + return getattr(self, key, default) class DomainQuestion(Question): argument_type = "domain" - def __init__(self, question, user_answers): + def __init__(self, question): from yunohost.domain import domain_list, _get_maindomain - super().__init__(question, user_answers) + super().__init__(question) if self.default is None: self.default = _get_maindomain() self.choices = domain_list()["domains"] - def _raise_invalid_answer(self): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("domain_name_unknown", domain=self.value), - ) + @staticmethod + def normalize(value, option={}): + if value.startswith("https://"): + value = value[len("https://") :] + elif value.startswith("http://"): + value = value[len("http://") :] + + # Remove trailing slashes + value = value.rstrip("/").lower() + + return value class UserQuestion(Question): argument_type = "user" - def __init__(self, question, user_answers): + def __init__(self, question): from yunohost.user import user_list, user_info from yunohost.domain import _get_maindomain - super().__init__(question, user_answers) - self.choices = user_list()["users"] + super().__init__(question) + self.choices = list(user_list()["users"].keys()) if not self.choices: raise YunohostValidationError( @@ -853,42 +892,42 @@ class UserQuestion(Question): if self.default is None: root_mail = "root@%s" % _get_maindomain() - for user in self.choices.keys(): + for user in self.choices: if root_mail in user_info(user).get("mail-aliases", []): self.default = user break - def _raise_invalid_answer(self): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("user_unknown", user=self.value), - ) - class NumberQuestion(Question): argument_type = "number" default_value = None - def __init__(self, question, user_answers): - super().__init__(question, user_answers) + def __init__(self, question): + super().__init__(question) self.min = question.get("min", None) self.max = question.get("max", None) self.step = question.get("step", None) @staticmethod def normalize(value, option={}): + if isinstance(value, int): return value + if isinstance(value, str): + value = value.strip() + if isinstance(value, str) and value.isdigit(): return int(value) if value in [None, ""]: return value + option = option.__dict__ if isinstance(option, Question) else option raise YunohostValidationError( - "app_argument_invalid", name=option.name, error=m18n.n("invalid_number") + "app_argument_invalid", + name=option.get("name"), + error=m18n.n("invalid_number"), ) def _prevalidate(self): @@ -915,8 +954,8 @@ class DisplayTextQuestion(Question): argument_type = "display_text" readonly = True - def __init__(self, question, user_answers): - super().__init__(question, user_answers) + def __init__(self, question): + super().__init__(question) self.optional = True self.style = question.get( @@ -946,90 +985,50 @@ class FileQuestion(Question): @classmethod def clean_upload_dirs(cls): # Delete files uploaded from API - if Moulinette.interface.type == "api": - for upload_dir in cls.upload_dirs: - if os.path.exists(upload_dir): - shutil.rmtree(upload_dir) + for upload_dir in cls.upload_dirs: + if os.path.exists(upload_dir): + shutil.rmtree(upload_dir) - def __init__(self, question, user_answers): - super().__init__(question, user_answers) - if question.get("accept"): - self.accept = question.get("accept") - else: - self.accept = "" - if Moulinette.interface.type == "api": - if user_answers.get(f"{self.name}[name]"): - self.value = { - "content": self.value, - "filename": user_answers.get(f"{self.name}[name]", self.name), - } + def __init__(self, question): + super().__init__(question) + self.accept = question.get("accept", "") def _prevalidate(self): if self.value is None: self.value = self.current_value super()._prevalidate() - if ( - isinstance(self.value, str) - and self.value - and not os.path.exists(self.value) - ): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("file_does_not_exist", path=self.value), - ) - if self.value in [None, ""] or not self.accept: - return - filename = self.value if isinstance(self.value, str) else self.value["filename"] - if "." not in filename or "." + filename.split(".")[ - -1 - ] not in self.accept.replace(" ", "").split(","): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n( - "file_extension_not_accepted", file=filename, accept=self.accept - ), - ) + if Moulinette.interface.type != "api": + if not self.value or not os.path.exists(str(self.value)): + raise YunohostValidationError( + "app_argument_invalid", + name=self.name, + error=m18n.n("file_does_not_exist", path=str(self.value)), + ) def _post_parse_value(self): from base64 import b64decode - # Upload files from API - # A file arg contains a string with "FILENAME:BASE64_CONTENT" if not self.value: return self.value - if Moulinette.interface.type == "api" and isinstance(self.value, dict): + upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") + _, file_path = tempfile.mkstemp(dir=upload_dir) - upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_") - FileQuestion.upload_dirs += [upload_dir] - filename = self.value["filename"] - logger.debug( - f"Save uploaded file {self.value['filename']} from API into {upload_dir}" - ) + FileQuestion.upload_dirs += [upload_dir] - # Filename is given by user of the API. For security reason, we have replaced - # os.path.join to avoid the user to be able to rewrite a file in filesystem - # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" - file_path = os.path.normpath(upload_dir + "/" + filename) - if not file_path.startswith(upload_dir + "/"): - raise YunohostError( - f"Filename '{filename}' received from the API got a relative parent path, which is forbidden", - raw_msg=True, - ) - i = 2 - while os.path.exists(file_path): - file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) - i += 1 + logger.debug(f"Saving file {self.name} for file question into {file_path}") + if Moulinette.interface.type != "api": + content = read_file(str(self.value), file_mode="rb") - content = self.value["content"] + if Moulinette.interface.type == "api": + content = b64decode(self.value) - write_to_file(file_path, b64decode(content), file_mode="wb") + write_to_file(file_path, content, file_mode="wb") + + self.value = file_path - self.value = file_path return self.value @@ -1057,25 +1056,41 @@ ARGUMENTS_TYPE_PARSERS = { } -def parse_args_in_yunohost_format(user_answers, argument_questions): +def ask_questions_and_parse_answers( + questions: Dict, prefilled_answers: Union[str, Mapping[str, Any]] = {} +) -> List[Question]: """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. Keyword arguments: - user_answers -- a dictionnary of arguments from the user (generally - empty in CLI, filed from the admin interface) - argument_questions -- the arguments description store in yunohost + questions -- the arguments description store in yunohost format from actions.json/toml, manifest.json/toml or config_panel.json/toml + prefilled_answers -- a url "query-string" such as "domain=yolo.test&path=/foobar&admin=sam" + or a dict such as {"domain": "yolo.test", "path": "/foobar", "admin": "sam"} """ - parsed_answers_dict = OrderedDict() - for question in argument_questions: + if isinstance(prefilled_answers, str): + # FIXME FIXME : this is not uniform with config_set() which uses parse.qs (no l) + # parse_qsl parse single values + # whereas parse.qs return list of values (which is useful for tags, etc) + # For now, let's not migrate this piece of code to parse_qs + # Because Aleks believes some bits of the app CI rely on overriding values (e.g. foo=foo&...&foo=bar) + prefilled_answers = dict( + urllib.parse.parse_qsl(prefilled_answers or "", keep_blank_values=True) + ) + + if not prefilled_answers: + prefilled_answers = {} + + out = [] + + for question in questions: question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")] - question = question_class(question, user_answers) + question["value"] = prefilled_answers.get(question["name"]) + question = question_class(question) - answer = question.ask_if_needed() - if answer is not None: - parsed_answers_dict[question.name] = answer + question.ask_if_needed() + out.append(question) - return parsed_answers_dict + return out diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 3db75f949..ccb6c5406 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -23,6 +23,8 @@ from typing import List from moulinette.utils.filesystem import read_file +SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] + YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] # Lazy dev caching to avoid re-reading the file multiple time when calling @@ -30,6 +32,18 @@ YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] external_resolvers_: List[str] = [] +def is_yunohost_dyndns_domain(domain): + + return any( + domain.endswith(f".{dyndns_domain}") for dyndns_domain in YNH_DYNDNS_DOMAINS + ) + + +def is_special_use_tld(domain): + + return any(domain.endswith(f".{tld}") for tld in SPECIAL_USE_TLDS) + + def external_resolvers(): global external_resolvers_ diff --git a/src/yunohost/utils/legacy.py b/src/yunohost/utils/legacy.py index f3243eb52..4186fa336 100644 --- a/src/yunohost/utils/legacy.py +++ b/src/yunohost/utils/legacy.py @@ -1,3 +1,20 @@ +import os +import re +import glob +from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import ( + read_file, + write_to_file, + write_to_yaml, + read_yaml, +) + +from yunohost.utils.error import YunohostValidationError + + +logger = getActionLogger("yunohost.legacy") + LEGACY_PERMISSION_LABEL = { ("nextcloud", "skipped"): "api", # .well-known ("libreto", "skipped"): "pad access", # /[^/]+ @@ -49,3 +66,184 @@ def legacy_permission_label(app, permission_type): return LEGACY_PERMISSION_LABEL.get( (app, permission_type), "Legacy %s urls" % permission_type ) + + +LEGACY_PHP_VERSION_REPLACEMENTS = [ + ("/etc/php5", "/etc/php/7.4"), + ("/etc/php/7.0", "/etc/php/7.4"), + ("/etc/php/7.3", "/etc/php/7.4"), + ("/var/run/php5-fpm", "/var/run/php/php7.4-fpm"), + ("/var/run/php/php7.0-fpm", "/var/run/php/php7.4-fpm"), + ("/var/run/php/php7.3-fpm", "/var/run/php/php7.4-fpm"), + ("php5", "php7.4"), + ("php7.0", "php7.4"), + ("php7.3", "php7.4"), + ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'), + ( + 'phpversion="${phpversion:-7.0}"', + 'phpversion="${phpversion:-7.4}"', + ), # Many helpers like the composer ones use 7.0 by default ... + ( + 'phpversion="${phpversion:-7.3}"', + 'phpversion="${phpversion:-7.4}"', + ), # Many helpers like the composer ones use 7.0 by default ... + ( + '"$phpversion" == "7.0"', + '$(bc <<< "$phpversion >= 7.4") -eq 1', + ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ( + '"$phpversion" == "7.3"', + '$(bc <<< "$phpversion >= 7.4") -eq 1', + ), # patch ynh_install_php to refuse installing/removing php <= 7.3 +] + + +def _patch_legacy_php_versions(app_folder): + + files_to_patch = [] + files_to_patch.extend(glob.glob("%s/conf/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) + files_to_patch.append("%s/manifest.json" % app_folder) + files_to_patch.append("%s/manifest.toml" % app_folder) + + for filename in files_to_patch: + + # Ignore non-regular files + if not os.path.isfile(filename): + continue + + c = ( + "sed -i " + + "".join( + "-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r) + for p, r in LEGACY_PHP_VERSION_REPLACEMENTS + ) + + "%s" % filename + ) + os.system(c) + + +def _patch_legacy_php_versions_in_settings(app_folder): + + settings = read_yaml(os.path.join(app_folder, "settings.yml")) + + if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]: + settings["fpm_config_dir"] = "/etc/php/7.4/fpm" + if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]: + settings["fpm_service"] = "php7.4-fpm" + if settings.get("phpversion") in ["7.0", "7.3"]: + settings["phpversion"] = "7.4" + + # We delete these checksums otherwise the file will appear as manually modified + list_to_remove = ["checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"] + settings = { + k: v + for k, v in settings.items() + if not any(k.startswith(to_remove) for to_remove in list_to_remove) + } + + write_to_yaml(app_folder + "/settings.yml", settings) + + +def _patch_legacy_helpers(app_folder): + + files_to_patch = [] + files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder)) + files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder)) + + stuff_to_replace = { + "yunohost app initdb": {"important": True}, + "yunohost app checkport": {"important": True}, + "yunohost tools port-available": {"important": True}, + "yunohost app checkurl": {"important": True}, + # Remove + # Automatic diagnosis data from YunoHost + # __PRE_TAG1__$(yunohost tools diagnosis | ...)__PRE_TAG2__" + # + "yunohost tools diagnosis": { + "pattern": r"(Automatic diagnosis data from YunoHost( *\n)*)? *(__\w+__)? *\$\(yunohost tools diagnosis.*\)(__\w+__)?", + "replace": r"", + "important": False, + }, + # Old $1, $2 in backup/restore scripts... + "app=$2": {"only_for": ["scripts/backup", "scripts/restore"], "important": True}, + # Old $1, $2 in backup/restore scripts... + "backup_dir=$1": {"only_for": ["scripts/backup", "scripts/restore"], "important": True}, + # Old $1, $2 in backup/restore scripts... + "restore_dir=$1": {"only_for": ["scripts/restore"], "important": True}, + # Old $1, $2 in install scripts... + # We ain't patching that shit because it ain't trivial to patch all args... + "domain=$1": {"only_for": ["scripts/install"], "important": True}, + } + + for helper, infos in stuff_to_replace.items(): + infos["pattern"] = ( + re.compile(infos["pattern"]) if infos.get("pattern") else None + ) + infos["replace"] = infos.get("replace") + + for filename in files_to_patch: + + # Ignore non-regular files + if not os.path.isfile(filename): + continue + + try: + content = read_file(filename) + except MoulinetteError: + continue + + replaced_stuff = False + show_warning = False + + for helper, infos in stuff_to_replace.items(): + + # Ignore if not relevant for this file + if infos.get("only_for") and not any( + filename.endswith(f) for f in infos["only_for"] + ): + continue + + # If helper is used, attempt to patch the file + if helper in content and infos["pattern"]: + content = infos["pattern"].sub(infos["replace"], content) + replaced_stuff = True + if infos["important"]: + show_warning = True + + # If the helper is *still* in the content, it means that we + # couldn't patch the deprecated helper in the previous lines. In + # that case, abort the install or whichever step is performed + if helper in content and infos["important"]: + raise YunohostValidationError( + "This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.", + raw_msg=True, + ) + + if replaced_stuff: + + # Check the app do load the helper + # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there... + if filename.split("/")[-1] in [ + "install", + "remove", + "upgrade", + "backup", + "restore", + ]: + source_helpers = "source /usr/share/yunohost/helpers" + if source_helpers not in content: + content.replace("#!/bin/bash", "#!/bin/bash\n" + source_helpers) + if source_helpers not in content: + content = source_helpers + "\n" + content + + # Actually write the new content in the file + write_to_file(filename, content) + + if show_warning: + # And complain about those damn deprecated helpers + logger.error( + r"/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ..." + )