YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'dev' into bullseye

Browse source
This commit is contained in:
Alexandre Aubin 2021-10-02 20:08:36 +02:00
commit 3591a01fdb
37 changed files with 2156 additions and 1721 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.coveragerc Normal file
View file

@ -0,0 +1,2 @@
[report]
omit=src/yunohost/tests/*,src/yunohost/vendor/*,/usr/lib/moulinette/yunohost/*

6
.gitlab/ci/test.gitlab-ci.yml
View file

@ -36,7 +36,7 @@ full-tests:
- *install_debs - *install_debs
- yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns --force-diskspace - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns --force-diskspace
script: script:
- python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ data/hooks/diagnosis/ --junitxml=report.xml
- cd tests - cd tests
- bash test_helpers.sh - bash test_helpers.sh
needs: needs:
@ -113,10 +113,10 @@ test-apps:
test-appscatalog: test-appscatalog:
extends: .test-stage extends: .test-stage
script: script:
- python3 -m pytest src/yunohost/tests/test_appscatalog.py - python3 -m pytest src/yunohost/tests/test_app_catalog.py
only: only:
changes: changes:
- src/yunohost/app.py - src/yunohost/app_calalog.py
test-appurl: test-appurl:
extends: .test-stage extends: .test-stage

2
README.md
View file

@ -6,7 +6,9 @@
<div align="center"> <div align="center">
![Version](https://img.shields.io/github/v/tag/yunohost/yunohost?label=version&sort=semver)
[![Build status](https://shields.io/gitlab/pipeline/yunohost/yunohost/dev)](https://gitlab.com/yunohost/yunohost/-/pipelines) [![Build status](https://shields.io/gitlab/pipeline/yunohost/yunohost/dev)](https://gitlab.com/yunohost/yunohost/-/pipelines)
![Test coverage](https://img.shields.io/gitlab/coverage/yunohost/yunohost/dev)
[![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE) [![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE)
[![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost) [![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost)

233
bin/yunomdns
View file

@ -4,160 +4,152 @@
Pythonic declaration of mDNS .local domains for YunoHost Pythonic declaration of mDNS .local domains for YunoHost
""" """
import subprocess
import re
import sys import sys
import yaml import yaml
import socket
from time import sleep from time import sleep
from typing import List, Dict from typing import List, Dict
from zeroconf import Zeroconf, ServiceInfo import ifaddr
from ipaddress import ip_address
from zeroconf import Zeroconf, ServiceInfo, ServiceBrowser
# Helper command taken from Moulinette
def check_output(args, stderr=subprocess.STDOUT, shell=True, **kwargs): def get_network_local_interfaces() -> Dict[str, Dict[str, List[str]]]:
"""Run command with arguments and return its output as a byte string
Overwrite some of the arguments to capture standard error in the result
and use shell by default before calling subprocess.check_output.
""" """
return ( Returns interfaces with their associated local IPs
subprocess.check_output(args, stderr=stderr, shell=shell, **kwargs)
.decode("utf-8")
.strip()
)
# Helper command taken from Moulinette
def _extract_inet(string, skip_netmask=False, skip_loopback=True):
""" """
Extract IP addresses (v4 and/or v6) from a string limited to one
address by protocol
Keyword argument: interfaces = {
string -- String to search in adapter.name: {
skip_netmask -- True to skip subnet mask extraction "ipv4": [ip.ip for ip in adapter.ips if ip.is_IPv4 and ip_address(ip.ip).is_private],
skip_loopback -- False to include addresses reserved for the "ipv6": [ip.ip[0] for ip in adapter.ips if ip.is_IPv6 and ip_address(ip.ip[0]).is_private and not ip_address(ip.ip[0]).is_link_local],
loopback interface
Returns:
A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6'
"""
ip4_pattern = (
r"((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}"
)
ip6_pattern = r"(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::?((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)"
ip4_pattern += r"/[0-9]{1,2})" if not skip_netmask else ")"
ip6_pattern += r"/[0-9]{1,3})" if not skip_netmask else ")"
result = {}
for m in re.finditer(ip4_pattern, string):
addr = m.group(1)
if skip_loopback and addr.startswith("127."):
continue
# Limit to only one result
result["ipv4"] = addr
break
for m in re.finditer(ip6_pattern, string):
addr = m.group(1)
if skip_loopback and addr == "::1":
continue
# Limit to only one result
result["ipv6"] = addr
break
return result
# Helper command taken from Moulinette
def get_network_interfaces():
# Get network devices and their addresses (raw infos from 'ip addr')
devices_raw = {}
output = check_output("ip --brief a").split("\n")
for line in output:
line = line.split()
iname = line[0]
ips = ' '.join(line[2:])
devices_raw[iname] = ips
# Parse relevant informations for each of them
devices = {
name: _extract_inet(addrs)
for name, addrs in devices_raw.items()
if name != "lo"
} }
for adapter in ifaddr.get_adapters()
if adapter.name != "lo"
}
return interfaces
return devices
if __name__ == '__main__': # Listener class, to detect duplicates on the network
# Stores the list of servers in its list property
class Listener:
def __init__(self):
self.list = []
def remove_service(self, zeroconf, type, name):
info = zeroconf.get_service_info(type, name)
self.list.remove(info.server)
def update_service(self, zeroconf, type, name):
pass
def add_service(self, zeroconf, type, name):
info = zeroconf.get_service_info(type, name)
self.list.append(info.server[:-1])
def main() -> bool:
### ###
# CONFIG # CONFIG
### ###
with open('/etc/yunohost/mdns.yml', 'r') as f: with open("/etc/yunohost/mdns.yml", "r") as f:
config = yaml.safe_load(f) or {} config = yaml.safe_load(f) or {}
updated = False
required_fields = ["interfaces", "domains"] required_fields = ["domains"]
missing_fields = [field for field in required_fields if field not in config] missing_fields = [field for field in required_fields if field not in config]
interfaces = get_network_local_interfaces()
if missing_fields: if missing_fields:
print("The fields %s are required" % ', '.join(missing_fields)) print(f"The fields {missing_fields} are required in mdns.yml")
return False
if config['interfaces'] is None: if "interfaces" not in config:
print('No interface listed for broadcast.') config["interfaces"] = [interface
sys.exit(0) for interface, local_ips in interfaces.items()
if local_ips["ipv4"]]
if 'yunohost.local' not in config['domains']: if "ban_interfaces" in config:
config['domains'].append('yunohost.local') config["interfaces"] = [interface
for interface in config["interfaces"]
if interface not in config["ban_interfaces"]]
zcs = {} # Let's discover currently published .local domains accross the network
interfaces = get_network_interfaces() zc = Zeroconf()
for interface in config['interfaces']: listener = Listener()
infos = [] # List of ServiceInfo objects, to feed Zeroconf browser = ServiceBrowser(zc, "_device-info._tcp.local.", listener)
ips = [] # Human-readable IPs sleep(2)
b_ips = [] # Binary-convered IPs browser.cancel()
zc.close()
ipv4 = interfaces[interface]['ipv4'].split('/')[0] # Always attempt to publish yunohost.local
if ipv4: if "yunohost.local" not in config["domains"]:
ips.append(ipv4) config["domains"].append("yunohost.local")
b_ips.append(socket.inet_pton(socket.AF_INET, ipv4))
ipv6 = interfaces[interface]['ipv6'].split('/')[0] def find_domain_not_already_published(domain):
if ipv6:
ips.append(ipv6) # Try domain.local ... but if it's already published by another entity,
b_ips.append(socket.inet_pton(socket.AF_INET6, ipv6)) # try domain-2.local, domain-3.local, ...
i = 1
domain_i = domain
while domain_i in listener.list:
print(f"Uh oh, {domain_i} already exists on the network...")
i += 1
domain_i = domain.replace(".local", f"-{i}.local")
return domain_i
config['domains'] = [find_domain_not_already_published(domain) for domain in config['domains']]
zcs: Dict[Zeroconf, List[ServiceInfo]] = {}
for interface in config["interfaces"]:
if interface not in interfaces:
print(f"Interface {interface} listed in config file is not present on system.")
continue
# Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7
# Buster only ships 0.1.6
# Bullseye ships 0.1.7
# To be re-enabled once we're on bullseye...
# ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"]
ips: List[str] = interfaces[interface]["ipv4"]
# If at least one IP is listed # If at least one IP is listed
if ips: if not ips:
continue
# Create a Zeroconf object, and store the ServiceInfos # Create a Zeroconf object, and store the ServiceInfos
zc = Zeroconf(interfaces=ips) zc = Zeroconf(interfaces=ips) # type: ignore
zcs[zc] = [] zcs[zc] = []
for d in config['domains']:
d_domain=d.replace('.local','') for d in config["domains"]:
if '.' in d_domain: d_domain = d.replace(".local", "")
print(d_domain+'.local: subdomains are not supported.') if "." in d_domain:
else: print(f"{d_domain}.local: subdomains are not supported.")
continue
# Create a ServiceInfo object for each .local domain # Create a ServiceInfo object for each .local domain
zcs[zc].append(ServiceInfo( zcs[zc].append(
type_='_device-info._tcp.local.', ServiceInfo(
name=interface+': '+d_domain+'._device-info._tcp.local.', type_="_device-info._tcp.local.",
addresses=b_ips, name=f"{interface}: {d_domain}._device-info._tcp.local.",
parsed_addresses=ips,
port=80, port=80,
server=d+'.', server=f"{d}.",
)) )
print('Adding '+d+' with addresses '+str(ips)+' on interface '+interface) )
print(f"Adding {d} with addresses {ips} on interface {interface}")
# Run registration # Run registration
print("Registering...") print("Registering...")
for zc, infos in zcs.items(): for zc, infos in zcs.items():
for info in infos: for info in infos:
zc.register_service(info) zc.register_service(info, allow_name_change=True, cooperating_responders=True)
try: try:
print("Registered. Press Ctrl+C or stop service to stop.") print("Registered. Press Ctrl+C or stop service to stop.")
@ -168,6 +160,11 @@ if __name__ == '__main__':
finally: finally:
print("Unregistering...") print("Unregistering...")
for zc, infos in zcs.items(): for zc, infos in zcs.items():
for info in infos: zc.unregister_all_services()
zc.unregister_service(info)
zc.close() zc.close()
return True
if __name__ == "__main__":
sys.exit(0 if main() else 1)

107
data/helpers.d/config
View file

@ -1,45 +1,22 @@
#!/bin/bash #!/bin/bash
_ynh_app_config_get() { _ynh_app_config_get_one() {
# From settings local short_setting="$1"
local lines local type="$2"
lines=$(python3 << EOL local bind="$3"
import toml
from collections import OrderedDict
with open("../config_panel.toml", "r") as f:
file_content = f.read()
loaded_toml = toml.loads(file_content, _dict=OrderedDict)
for panel_name, panel in loaded_toml.items():
if not isinstance(panel, dict): continue
for section_name, section in panel.items():
if not isinstance(section, dict): continue
for name, param in section.items():
if not isinstance(param, dict):
continue
print(';'.join([
name,
param.get('type', 'string'),
param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null')
]))
EOL
)
for line in $lines
do
# Split line into short_setting, type and bind
IFS=';' read short_setting type bind <<< "$line"
local getter="get__${short_setting}" local getter="get__${short_setting}"
binds[${short_setting}]="$bind"
types[${short_setting}]="$type"
file_hash[${short_setting}]=""
formats[${short_setting}]=""
# Get value from getter if exists # Get value from getter if exists
if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null;
then then
old[$short_setting]="$($getter)" old[$short_setting]="$($getter)"
formats[${short_setting}]="yaml" formats[${short_setting}]="yaml"
elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
then
old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)"
formats[${short_setting}]="yaml"
elif [[ "$bind" == "null" ]] elif [[ "$bind" == "null" ]]
then then
old[$short_setting]="YNH_NULL" old[$short_setting]="YNH_NULL"
@ -85,14 +62,9 @@ EOL
old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")" old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")"
fi fi
done
} }
_ynh_app_config_apply_one() {
_ynh_app_config_apply() { local short_setting="$1"
for short_setting in "${!old[@]}"
do
local setter="set__${short_setting}" local setter="set__${short_setting}"
local bind="${binds[$short_setting]}" local bind="${binds[$short_setting]}"
local type="${types[$short_setting]}" local type="${types[$short_setting]}"
@ -103,6 +75,10 @@ _ynh_app_config_apply() {
then then
$setter $setter
elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
then
"set__${bind%%(*}" $short_setting $type $bind
elif [[ "$bind" == "null" ]] elif [[ "$bind" == "null" ]]
then then
continue continue
@ -172,6 +148,49 @@ _ynh_app_config_apply() {
fi fi
fi fi
}
_ynh_app_config_get() {
# From settings
local lines
lines=$(python3 << EOL
import toml
from collections import OrderedDict
with open("../config_panel.toml", "r") as f:
file_content = f.read()
loaded_toml = toml.loads(file_content, _dict=OrderedDict)
for panel_name, panel in loaded_toml.items():
if not isinstance(panel, dict): continue
for section_name, section in panel.items():
if not isinstance(section, dict): continue
for name, param in section.items():
if not isinstance(param, dict):
continue
print(';'.join([
name,
param.get('type', 'string'),
param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null')
]))
EOL
)
for line in $lines
do
# Split line into short_setting, type and bind
IFS=';' read short_setting type bind <<< "$line"
binds[${short_setting}]="$bind"
types[${short_setting}]="$type"
file_hash[${short_setting}]=""
formats[${short_setting}]=""
ynh_app_config_get_one $short_setting $type $bind
done
}
_ynh_app_config_apply() {
for short_setting in "${!old[@]}"
do
ynh_app_config_apply_one $short_setting
done done
} }
@ -253,6 +272,9 @@ _ynh_app_config_validate() {
if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null;
then then
result="$(validate__$short_setting)" result="$(validate__$short_setting)"
elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
then
"validate__${bind%%(*}" $short_setting
fi fi
if [ -n "$result" ] if [ -n "$result" ]
then then
@ -283,6 +305,10 @@ _ynh_app_config_validate() {
} }
ynh_app_config_get_one() {
_ynh_app_config_get_one $1 $2 $3
}
ynh_app_config_get() { ynh_app_config_get() {
_ynh_app_config_get _ynh_app_config_get
} }
@ -295,6 +321,9 @@ ynh_app_config_validate() {
_ynh_app_config_validate _ynh_app_config_validate
} }
ynh_app_config_apply_one() {
_ynh_app_config_apply_one $1
}
ynh_app_config_apply() { ynh_app_config_apply() {
_ynh_app_config_apply _ynh_app_config_apply
} }

4
data/helpers.d/nodejs
View file

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
n_version=7.3.0 n_version=7.5.0
n_install_dir="/opt/node_n" n_install_dir="/opt/node_n"
node_version_path="$n_install_dir/n/versions/node" node_version_path="$n_install_dir/n/versions/node"
# N_PREFIX is the directory of n, it needs to be loaded as a environment variable. # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
@ -17,7 +17,7 @@ ynh_install_n () {
ynh_print_info --message="Installation of N - Node.js version management" ynh_print_info --message="Installation of N - Node.js version management"
# Build an app.src for n # Build an app.src for n
echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz
SOURCE_SUM=b908b0fc86922ede37e89d1030191285209d7d521507bf136e62895e5797847f" > "$YNH_APP_BASEDIR/conf/n.src" SOURCE_SUM=d4da7ea91f680de0c9b5876e097e2a793e8234fcd0f7ca87a0599b925be087a3" > "$YNH_APP_BASEDIR/conf/n.src"
# Download and extract n # Download and extract n
ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
# Install n # Install n

1
data/hooks/conf_regen/01-yunohost
View file

@ -142,6 +142,7 @@ EOF
touch ${pending_dir}/etc/systemd/system/proc-hidepid.service touch ${pending_dir}/etc/systemd/system/proc-hidepid.service
fi fi
mkdir -p ${pending_dir}/etc/dpkg/origins/
cp dpkg-origins ${pending_dir}/etc/dpkg/origins/yunohost cp dpkg-origins ${pending_dir}/etc/dpkg/origins/yunohost
} }

7
data/hooks/conf_regen/37-mdns
View file

@ -12,13 +12,6 @@ _generate_config() {
[[ "$domain" =~ ^[^.]+\.local$ ]] || continue [[ "$domain" =~ ^[^.]+\.local$ ]] || continue
echo " - $domain" echo " - $domain"
done done
echo "interfaces:"
local_network_interfaces="$(ip --brief a | grep ' 10\.\| 192\.168\.' | awk '{print $1}')"
for interface in $local_network_interfaces
do
echo " - $interface"
done
} }
do_init_regen() { do_init_regen() {

54
data/hooks/diagnosis/12-dnsrecords.py
View file

@ -8,13 +8,16 @@ from publicsuffix2 import PublicSuffixList
from moulinette.utils.process import check_output from moulinette.utils.process import check_output
from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.utils.dns import (
dig,
YNH_DYNDNS_DOMAINS,
is_yunohost_dyndns_domain,
is_special_use_tld,
)
from yunohost.diagnosis import Diagnoser from yunohost.diagnosis import Diagnoser
from yunohost.domain import domain_list, _get_maindomain from yunohost.domain import domain_list, _get_maindomain
from yunohost.dns import _build_dns_conf, _get_dns_zone_for_domain from yunohost.dns import _build_dns_conf, _get_dns_zone_for_domain
SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"]
class DNSRecordsDiagnoser(Diagnoser): class DNSRecordsDiagnoser(Diagnoser):
@ -26,23 +29,20 @@ class DNSRecordsDiagnoser(Diagnoser):
main_domain = _get_maindomain() main_domain = _get_maindomain()
all_domains = domain_list(exclude_subdomains=True)["domains"] major_domains = domain_list(exclude_subdomains=True)["domains"]
for domain in all_domains: for domain in major_domains:
self.logger_debug("Diagnosing DNS conf for %s" % domain) self.logger_debug("Diagnosing DNS conf for %s" % domain)
is_specialusedomain = any(
domain.endswith("." + tld) for tld in SPECIAL_USE_TLDS
)
for report in self.check_domain( for report in self.check_domain(
domain, domain,
domain == main_domain, domain == main_domain,
is_specialusedomain=is_specialusedomain,
): ):
yield report yield report
# Check if a domain buy by the user will expire soon # Check if a domain buy by the user will expire soon
psl = PublicSuffixList() psl = PublicSuffixList()
domains_from_registrar = [ domains_from_registrar = [
psl.get_public_suffix(domain) for domain in all_domains psl.get_public_suffix(domain) for domain in major_domains
] ]
domains_from_registrar = [ domains_from_registrar = [
domain for domain in domains_from_registrar if "." in domain domain for domain in domains_from_registrar if "." in domain
@ -53,7 +53,16 @@ class DNSRecordsDiagnoser(Diagnoser):
for report in self.check_expiration_date(domains_from_registrar): for report in self.check_expiration_date(domains_from_registrar):
yield report yield report
def check_domain(self, domain, is_main_domain, is_specialusedomain): def check_domain(self, domain, is_main_domain):
if is_special_use_tld(domain):
categories = []
yield dict(
meta={"domain": domain},
data={},
status="INFO",
summary="diagnosis_dns_specialusedomain",
)
base_dns_zone = _get_dns_zone_for_domain(domain) base_dns_zone = _get_dns_zone_for_domain(domain)
basename = domain.replace(base_dns_zone, "").rstrip(".") or "@" basename = domain.replace(base_dns_zone, "").rstrip(".") or "@"
@ -64,15 +73,6 @@ class DNSRecordsDiagnoser(Diagnoser):
categories = ["basic", "mail", "xmpp", "extra"] categories = ["basic", "mail", "xmpp", "extra"]
if is_specialusedomain:
categories = []
yield dict(
meta={"domain": domain},
data={},
status="INFO",
summary="diagnosis_dns_specialusedomain",
)
for category in categories: for category in categories:
records = expected_configuration[category] records = expected_configuration[category]
@ -84,7 +84,8 @@ class DNSRecordsDiagnoser(Diagnoser):
id_ = r["type"] + ":" + r["name"] id_ = r["type"] + ":" + r["name"]
fqdn = r["name"] + "." + base_dns_zone if r["name"] != "@" else domain fqdn = r["name"] + "." + base_dns_zone if r["name"] != "@" else domain
# Ugly hack to not check mail records for subdomains stuff, otherwise will end up in a shitstorm of errors for people with many subdomains... # Ugly hack to not check mail records for subdomains stuff,
# otherwise will end up in a shitstorm of errors for people with many subdomains...
# Should find a cleaner solution in the suggested conf... # Should find a cleaner solution in the suggested conf...
if r["type"] in ["MX", "TXT"] and fqdn not in [ if r["type"] in ["MX", "TXT"] and fqdn not in [
domain, domain,
@ -131,6 +132,12 @@ class DNSRecordsDiagnoser(Diagnoser):
status = "SUCCESS" status = "SUCCESS"
summary = "diagnosis_dns_good_conf" summary = "diagnosis_dns_good_conf"
# If status is okay and there's actually no expected records
# (e.g. XMPP disabled)
# then let's not yield any diagnosis line
if not records and "status" == "SUCCESS":
continue
output = dict( output = dict(
meta={"domain": domain, "category": category}, meta={"domain": domain, "category": category},
data=results, data=results,
@ -140,10 +147,7 @@ class DNSRecordsDiagnoser(Diagnoser):
if discrepancies: if discrepancies:
# For ynh-managed domains (nohost.me etc...), tell people to try to "yunohost dyndns update --force" # For ynh-managed domains (nohost.me etc...), tell people to try to "yunohost dyndns update --force"
if any( if is_yunohost_dyndns_domain(domain):
domain.endswith(ynh_dyndns_domain)
for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS
):
output["details"] = ["diagnosis_dns_try_dyndns_update_force"] output["details"] = ["diagnosis_dns_try_dyndns_update_force"]
# Otherwise point to the documentation # Otherwise point to the documentation
else: else:

5
data/hooks/diagnosis/21-web.py
View file

@ -8,6 +8,7 @@ from moulinette.utils.filesystem import read_file
from yunohost.diagnosis import Diagnoser from yunohost.diagnosis import Diagnoser
from yunohost.domain import domain_list from yunohost.domain import domain_list
from yunohost.utils.dns import is_special_use_tld
DIAGNOSIS_SERVER = "diagnosis.yunohost.org" DIAGNOSIS_SERVER = "diagnosis.yunohost.org"
@ -34,11 +35,11 @@ class WebDiagnoser(Diagnoser):
summary="diagnosis_http_nginx_conf_not_up_to_date", summary="diagnosis_http_nginx_conf_not_up_to_date",
details=["diagnosis_http_nginx_conf_not_up_to_date_details"], details=["diagnosis_http_nginx_conf_not_up_to_date_details"],
) )
elif domain.endswith(".local"): elif is_special_use_tld(domain):
yield dict( yield dict(
meta={"domain": domain}, meta={"domain": domain},
status="INFO", status="INFO",
summary="diagnosis_http_localdomain", summary="diagnosis_http_special_use_tld",
) )
else: else:
domains_to_check.append(domain) domains_to_check.append(domain)

1
data/templates/mdns/yunomdns.service
View file

@ -6,6 +6,7 @@ After=network.target
User=mdns User=mdns
Group=mdns Group=mdns
Type=simple Type=simple
Environment=PYTHONUNBUFFERED=1
ExecStart=/usr/bin/yunomdns ExecStart=/usr/bin/yunomdns
StandardOutput=syslog StandardOutput=syslog

15
debian/changelog vendored
View file

@ -4,6 +4,21 @@ yunohost (11.0.0~alpha) unstable; urgency=low
-- Alexandre Aubin <alex.aubin@mailoo.org> Fri, 05 Feb 2021 00:02:38 +0100 -- Alexandre Aubin <alex.aubin@mailoo.org> Fri, 05 Feb 2021 00:02:38 +0100
yunohost (4.3.1) testing; urgency=low
- [fix] diagnosis: new app diagnosis grep reporing comments as issues ([#1333](https://github.com/YunoHost/yunohost/pull/1333))
- [enh] configpanel: Bind function for hotspot (79126809)
- [enh] cli: Rework/improve prompt mecanic ([#1338](https://github.com/YunoHost/yunohost/pull/1338))
- [fix] dyndns update broke because of buggy dns record names (da1b9089)
- [enh] dns: general improvement for special-use TLD / ynh dyndns domains (17aafe6f)
- [fix] yunomdns: various fixes/improvements ([#1335](https://github.com/YunoHost/yunohost/pull/1335))
- [fix] certs: Adapt ready_for_ACME check to the new dnsrecord result format... (d75c1a61)
- [i18n] Translations updated for French
Thanks to all contributors <3 ! (Éric Gaspar, Félix Piédallu, Kayou, ljf, tituspijean)
-- Alexandre Aubin <alex.aubin@mailoo.org> Wed, 29 Sep 2021 22:22:42 +0200
yunohost (4.3.0) testing; urgency=low yunohost (4.3.0) testing; urgency=low
- [users] Import/export users from/to CSV ([#1089](https://github.com/YunoHost/yunohost/pull/1089)) - [users] Import/export users from/to CSV ([#1089](https://github.com/YunoHost/yunohost/pull/1089))

2
debian/control vendored
View file

@ -10,7 +10,7 @@ Package: yunohost
Essential: yes Essential: yes
Architecture: all Architecture: all
Depends: ${python3:Depends}, ${misc:Depends} Depends: ${python3:Depends}, ${misc:Depends}
, moulinette (>= 4.2), ssowat (>= 4.0) , moulinette (>= 4.3), ssowat (>= 4.3)
, python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-psutil, python3-requests, python3-dnspython, python3-openssl
, python3-miniupnpc, python3-dbus, python3-jinja2 , python3-miniupnpc, python3-dbus, python3-jinja2
, python3-toml, python3-packaging, python3-publicsuffix2 , python3-toml, python3-packaging, python3-publicsuffix2

16
locales/en.json
View file

@ -13,10 +13,8 @@
"app_already_installed": "{app} is already installed", "app_already_installed": "{app} is already installed",
"app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
"app_already_up_to_date": "{app} is already up-to-date", "app_already_up_to_date": "{app} is already up-to-date",
"app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'", "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})",
"app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
"app_argument_password_help_keep": "Press Enter to keep the current value",
"app_argument_password_help_optional": "Type one space to empty the password",
"app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason",
"app_argument_required": "Argument '{name}' is required", "app_argument_required": "Argument '{name}' is required",
"app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.", "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.",
@ -38,7 +36,6 @@
"app_manifest_install_ask_is_public": "Should this app be exposed to anonymous visitors?", "app_manifest_install_ask_is_public": "Should this app be exposed to anonymous visitors?",
"app_manifest_install_ask_password": "Choose an administration password for this app", "app_manifest_install_ask_password": "Choose an administration password for this app",
"app_manifest_install_ask_path": "Choose the URL path (after the domain) where this app should be installed", "app_manifest_install_ask_path": "Choose the URL path (after the domain) where this app should be installed",
"app_manifest_invalid": "Something is wrong with the app manifest: {error}",
"app_not_correctly_installed": "{app} seems to be incorrectly installed", "app_not_correctly_installed": "{app} seems to be incorrectly installed",
"app_not_installed": "Could not find {app} in the list of installed apps: {all_apps}", "app_not_installed": "Could not find {app} in the list of installed apps: {all_apps}",
"app_not_properly_removed": "{app} has not been properly removed", "app_not_properly_removed": "{app} has not been properly removed",
@ -194,7 +191,7 @@
"diagnosis_dns_good_conf": "DNS records are correctly configured for domain {domain} (category {category})", "diagnosis_dns_good_conf": "DNS records are correctly configured for domain {domain} (category {category})",
"diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with the following info.<br>Type: <code>{type}</code><br>Name: <code>{name}</code><br>Value: <code>{value}</code>", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with the following info.<br>Type: <code>{type}</code><br>Name: <code>{name}</code><br>Value: <code>{value}</code>",
"diagnosis_dns_point_to_doc": "Please check the documentation at <a href='https://yunohost.org/dns_config'>https://yunohost.org/dns_config</a> if you need help about configuring DNS records.", "diagnosis_dns_point_to_doc": "Please check the documentation at <a href='https://yunohost.org/dns_config'>https://yunohost.org/dns_config</a> if you need help about configuring DNS records.",
"diagnosis_dns_specialusedomain": "Domain {domain} is based on a special-use top-level domain (TLD) and is therefore not expected to have actual DNS records.", "diagnosis_dns_specialusedomain": "Domain {domain} is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to have actual DNS records.",
"diagnosis_dns_try_dyndns_update_force": "This domain's DNS configuration should automatically be managed by YunoHost. If that's not the case, you can try to force an update using <cmd>yunohost dyndns update --force</cmd>.", "diagnosis_dns_try_dyndns_update_force": "This domain's DNS configuration should automatically be managed by YunoHost. If that's not the case, you can try to force an update using <cmd>yunohost dyndns update --force</cmd>.",
"diagnosis_domain_expiration_error": "Some domains will expire VERY SOON!", "diagnosis_domain_expiration_error": "Some domains will expire VERY SOON!",
"diagnosis_domain_expiration_not_found": "Unable to check the expiration date for some domains", "diagnosis_domain_expiration_not_found": "Unable to check the expiration date for some domains",
@ -203,7 +200,7 @@
"diagnosis_domain_expiration_warning": "Some domains will expire soon!", "diagnosis_domain_expiration_warning": "Some domains will expire soon!",
"diagnosis_domain_expires_in": "{domain} expires in {days} days.", "diagnosis_domain_expires_in": "{domain} expires in {days} days.",
"diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired!", "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired!",
"diagnosis_everything_ok": "Everything looks good for {category}!", "diagnosis_everything_ok": "Everything looks OK for {category}!",
"diagnosis_failed": "Failed to fetch diagnosis result for category '{category}': {error}", "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}': {error}",
"diagnosis_failed_for_category": "Diagnosis failed for category '{category}': {error}", "diagnosis_failed_for_category": "Diagnosis failed for category '{category}': {error}",
"diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!", "diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!",
@ -216,7 +213,7 @@
"diagnosis_http_could_not_diagnose_details": "Error: {error}", "diagnosis_http_could_not_diagnose_details": "Error: {error}",
"diagnosis_http_hairpinning_issue": "Your local network does not seem to have hairpinning enabled.", "diagnosis_http_hairpinning_issue": "Your local network does not seem to have hairpinning enabled.",
"diagnosis_http_hairpinning_issue_details": "This is probably because of your ISP box / router. As a result, people from outside your local network will be able to access your server as expected, but not people from inside the local network (like you, probably?) when using the domain name or global IP. You may be able to improve the situation by having a look at <a href='https://yunohost.org/dns_local_network'>https://yunohost.org/dns_local_network</a>", "diagnosis_http_hairpinning_issue_details": "This is probably because of your ISP box / router. As a result, people from outside your local network will be able to access your server as expected, but not people from inside the local network (like you, probably?) when using the domain name or global IP. You may be able to improve the situation by having a look at <a href='https://yunohost.org/dns_local_network'>https://yunohost.org/dns_local_network</a>",
"diagnosis_http_localdomain": "Domain {domain}, with a .local TLD, is not expected to be exposed outside the local network.", "diagnosis_http_special_use_tld": "Domain {domain} is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to be exposed outside the local network.",
"diagnosis_http_nginx_conf_not_up_to_date": "This domain's nginx configuration appears to have been modified manually, and prevents YunoHost from diagnosing if it's reachable on HTTP.", "diagnosis_http_nginx_conf_not_up_to_date": "This domain's nginx configuration appears to have been modified manually, and prevents YunoHost from diagnosing if it's reachable on HTTP.",
"diagnosis_http_nginx_conf_not_up_to_date_details": "To fix the situation, inspect the difference with the command line using <cmd>yunohost tools regen-conf nginx --dry-run --with-diff</cmd> and if you're ok, apply the changes with <cmd>yunohost tools regen-conf nginx --force</cmd>.", "diagnosis_http_nginx_conf_not_up_to_date_details": "To fix the situation, inspect the difference with the command line using <cmd>yunohost tools regen-conf nginx --dry-run --with-diff</cmd> and if you're ok, apply the changes with <cmd>yunohost tools regen-conf nginx --force</cmd>.",
"diagnosis_http_ok": "Domain {domain} is reachable through HTTP from outside the local network.", "diagnosis_http_ok": "Domain {domain} is reachable through HTTP from outside the local network.",
@ -310,6 +307,7 @@
"domain_deleted": "Domain deleted", "domain_deleted": "Domain deleted",
"domain_deletion_failed": "Unable to delete domain {domain}: {error}", "domain_deletion_failed": "Unable to delete domain {domain}: {error}",
"domain_dns_conf_is_just_a_recommendation": "This command shows you the *recommended* configuration. It does not actually set up the DNS configuration for you. It is your responsability to configure your DNS zone in your registrar according to this recommendation.", "domain_dns_conf_is_just_a_recommendation": "This command shows you the *recommended* configuration. It does not actually set up the DNS configuration for you. It is your responsability to configure your DNS zone in your registrar according to this recommendation.",
"domain_dns_conf_special_use_tld": "This domain is based on a special-use top-level domain (TLD) such as .local or .test and is therefore not expected to have actual DNS records.",
"domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain",
"domain_dyndns_root_unknown": "Unknown DynDNS root domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain",
"domain_exists": "The domain already exists", "domain_exists": "The domain already exists",
@ -366,7 +364,6 @@
"extracting": "Extracting...", "extracting": "Extracting...",
"field_invalid": "Invalid field '{}'", "field_invalid": "Invalid field '{}'",
"file_does_not_exist": "The file {path} does not exist.", "file_does_not_exist": "The file {path} does not exist.",
"file_extension_not_accepted": "Refusing file '{path}' because its extension is not among the accepted extensions: {accept}",
"firewall_reload_failed": "Could not reload the firewall", "firewall_reload_failed": "Could not reload the firewall",
"firewall_reloaded": "Firewall reloaded", "firewall_reloaded": "Firewall reloaded",
"firewall_rules_cmd_failed": "Some firewall rule commands have failed. More info in log.", "firewall_rules_cmd_failed": "Some firewall rule commands have failed. More info in log.",
@ -560,6 +557,7 @@
"migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations run`.", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations run`.",
"not_enough_disk_space": "Not enough free space on '{path}'", "not_enough_disk_space": "Not enough free space on '{path}'",
"operation_interrupted": "The operation was manually interrupted?", "operation_interrupted": "The operation was manually interrupted?",
"other_available_options": "... and {n} other available options not shown",
"packages_upgrade_failed": "Could not upgrade all the packages", "packages_upgrade_failed": "Could not upgrade all the packages",
"password_listed": "This password is among the most used passwords in the world. Please choose something more unique.", "password_listed": "This password is among the most used passwords in the world. Please choose something more unique.",
"password_too_simple_1": "The password needs to be at least 8 characters long", "password_too_simple_1": "The password needs to be at least 8 characters long",
@ -675,7 +673,7 @@
"service_stop_failed": "Unable to stop the service '{service}'\n\nRecent service logs:{logs}", "service_stop_failed": "Unable to stop the service '{service}'\n\nRecent service logs:{logs}",
"service_stopped": "Service '{service}' stopped", "service_stopped": "Service '{service}' stopped",
"service_unknown": "Unknown service '{service}'", "service_unknown": "Unknown service '{service}'",
"show_tile_cant_be_enabled_for_regex": "You cannot enable 'show_tile' right no, because the URL for the permission '{permission}' is a regex", "show_tile_cant_be_enabled_for_regex": "You cannot enable 'show_tile' right now, because the URL for the permission '{permission}' is a regex",
"show_tile_cant_be_enabled_for_url_not_defined": "You cannot enable 'show_tile' right now, because you must first define an URL for the permission '{permission}'", "show_tile_cant_be_enabled_for_url_not_defined": "You cannot enable 'show_tile' right now, because you must first define an URL for the permission '{permission}'",
"ssowat_conf_generated": "SSOwat configuration regenerated", "ssowat_conf_generated": "SSOwat configuration regenerated",
"ssowat_conf_updated": "SSOwat configuration updated", "ssowat_conf_updated": "SSOwat configuration updated",

34
locales/fr.json
View file

@ -431,7 +431,7 @@
"diagnosis_cache_still_valid": "(Le cache est encore valide pour le diagnostic {category}. Il ne sera pas re-diagnostiqué pour le moment !)", "diagnosis_cache_still_valid": "(Le cache est encore valide pour le diagnostic {category}. Il ne sera pas re-diagnostiqué pour le moment !)",
"diagnosis_ignored_issues": "(+ {nb_ignored} problème(s) ignoré(s))", "diagnosis_ignored_issues": "(+ {nb_ignored} problème(s) ignoré(s))",
"diagnosis_found_warnings": "Trouvé {warnings} objet(s) pouvant être amélioré(s) pour {category}.", "diagnosis_found_warnings": "Trouvé {warnings} objet(s) pouvant être amélioré(s) pour {category}.",
"diagnosis_everything_ok": "Tout semble bien pour {category} !", "diagnosis_everything_ok": "Tout semble OK pour {category} !",
"diagnosis_failed": "Échec de la récupération du résultat du diagnostic pour la catégorie '{category}' : {error}", "diagnosis_failed": "Échec de la récupération du résultat du diagnostic pour la catégorie '{category}' : {error}",
"diagnosis_ip_connected_ipv4": "Le serveur est connecté à Internet en IPv4 !", "diagnosis_ip_connected_ipv4": "Le serveur est connecté à Internet en IPv4 !",
"diagnosis_ip_no_ipv4": "Le serveur ne dispose pas d'une adresse IPv4.", "diagnosis_ip_no_ipv4": "Le serveur ne dispose pas d'une adresse IPv4.",
@ -675,5 +675,35 @@
"log_app_config_set": "Appliquer la configuration à l'application '{}'", "log_app_config_set": "Appliquer la configuration à l'application '{}'",
"service_not_reloading_because_conf_broken": "Le service '{name}' n'a pas été rechargé/redémarré car sa configuration est cassée : {errors}", "service_not_reloading_because_conf_broken": "Le service '{name}' n'a pas été rechargé/redémarré car sa configuration est cassée : {errors}",
"app_argument_password_help_keep": "Tapez sur Entrée pour conserver la valeur actuelle", "app_argument_password_help_keep": "Tapez sur Entrée pour conserver la valeur actuelle",
"app_argument_password_help_optional": "Tapez un espace pour vider le mot de passe" "app_argument_password_help_optional": "Tapez un espace pour vider le mot de passe",
"domain_registrar_is_not_configured": "Le registrar n'est pas encore configuré pour le domaine {domain}.",
"domain_dns_push_not_applicable": "La fonction de configuration DNS automatique n'est pas applicable au domaine {domain}. Vous devez configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns_config.",
"domain_dns_registrar_yunohost": "Ce domaine est nohost.me / nohost.st / ynh.fr et sa configuration DNS est donc automatiquement gérée par YunoHost sans autre configuration. (voir la commande 'yunohost dyndns update')",
"domain_dns_registrar_supported": "YunoHost a détecté automatiquement que ce domaine est géré par le registrar **{registrar}**. Si vous le souhaitez, YunoHost configurera automatiquement cette zone DNS, si vous lui fournissez les identifiants API appropriés. Vous pouvez trouver de la documentation sur la façon d'obtenir vos identifiants API sur cette page : https://yunohost.org/registar_api_{registrar}. (Vous pouvez également configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns )",
"domain_config_features_disclaimer": "Jusqu'à présent, l'activation/désactivation des fonctionnalités de messagerie ou XMPP n'a d'impact que sur la configuration DNS recommandée et automatique, et non sur les configurations système !",
"domain_dns_push_managed_in_parent_domain": "La fonctionnalité de configuration DNS automatique est gérée dans le domaine parent {parent_domain}.",
"domain_dns_registrar_managed_in_parent_domain": "Ce domaine est un sous-domaine de {parent_domain_link}. La configuration du registrar DNS doit être gérée dans le panneau de configuration de {parent_domain}.",
"domain_dns_registrar_not_supported": "YunoHost n'a pas pu détecter automatiquement le bureau d'enregistrement gérant ce domaine. Vous devez configurer manuellement vos enregistrements DNS en suivant la documentation sur https://yunohost.org/dns.",
"domain_dns_registrar_experimental": "Jusqu'à présent, l'interface avec l'API de **{registrar}** n'a pas été correctement testée et revue par la communauté YunoHost. L'assistance est **très expérimentale** - soyez prudent !",
"domain_dns_push_failed_to_authenticate": "Échec de l'authentification sur l'API du bureau d'enregistrement pour le domaine « {domain} ». Très probablement les informations d'identification sont incorrectes ? (Error: {error})",
"domain_dns_push_failed_to_list": "Échec de la liste des enregistrements actuels à l'aide de l'API du registraire : {error}",
"domain_dns_push_already_up_to_date": "Dossiers déjà à jour.",
"domain_dns_pushing": "Transmission des enregistrements DNS...",
"domain_dns_push_record_failed": "Échec de l'enregistrement {action} {type}/{name} : {error}",
"domain_dns_push_success": "Enregistrements DNS mis à jour !",
"domain_dns_push_failed": "La mise à jour des enregistrements DNS a échoué.",
"domain_dns_push_partial_failure": "Enregistrements DNS partiellement mis à jour : certains avertissements/erreurs ont été signalés.",
"domain_config_mail_in": "Emails entrants",
"domain_config_mail_out": "Emails sortants",
"domain_config_xmpp": "Messagerie instantanée (XMPP)",
"domain_config_auth_token": "Jeton d'authentification",
"domain_config_auth_key": "Clé d'authentification",
"domain_config_auth_secret": "Secret d'authentification",
"domain_config_api_protocol": "Protocole API",
"domain_config_auth_entrypoint": "Point d'entrée API",
"domain_config_auth_application_key": "Clé d'application",
"domain_config_auth_application_secret": "Clé secrète de l'application",
"ldap_attribute_already_exists": "L'attribut LDAP '{attribute}' existe déjà avec la valeur '{value}'",
"log_domain_config_set": "Mettre à jour la configuration du domaine '{}'",
"log_domain_dns_push": "Pousser les enregistrements DNS pour le domaine '{}'"
} }

2
src/yunohost/.coveragerc
View file

@ -1,2 +0,0 @@
[report]
omit=tests/*,vendor/*,/usr/lib/moulinette/yunohost/

1168
src/yunohost/app.py
View file

File diff suppressed because it is too large Load diff

255
src/yunohost/app_catalog.py Normal file
View file

@ -0,0 +1,255 @@
import os
import re
from moulinette import m18n
from moulinette.utils.log import getActionLogger
from moulinette.utils.network import download_json
from moulinette.utils.filesystem import (
read_json,
read_yaml,
write_to_json,
write_to_yaml,
mkdir,
)
from yunohost.utils.i18n import _value_for_locale
from yunohost.utils.error import YunohostError
logger = getActionLogger("yunohost.app_catalog")
APPS_CATALOG_CACHE = "/var/cache/yunohost/repo"
APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
APPS_CATALOG_API_VERSION = 2
APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
# Old legacy function...
def app_fetchlist():
logger.warning(
"'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead"
)
from yunohost.tools import tools_update
tools_update(target="apps")
def app_catalog(full=False, with_categories=False):
"""
Return a dict of apps available to installation from Yunohost's app catalog
"""
from yunohost.app import _installed_apps, _set_default_ask_questions
# Get app list from catalog cache
catalog = _load_apps_catalog()
installed_apps = set(_installed_apps())
# Trim info for apps if not using --full
for app, infos in catalog["apps"].items():
infos["installed"] = app in installed_apps
infos["manifest"]["description"] = _value_for_locale(
infos["manifest"]["description"]
)
if not full:
catalog["apps"][app] = {
"description": infos["manifest"]["description"],
"level": infos["level"],
}
else:
infos["manifest"]["arguments"] = _set_default_ask_questions(
infos["manifest"].get("arguments", {})
)
# Trim info for categories if not using --full
for category in catalog["categories"]:
category["title"] = _value_for_locale(category["title"])
category["description"] = _value_for_locale(category["description"])
for subtags in category.get("subtags", []):
subtags["title"] = _value_for_locale(subtags["title"])
if not full:
catalog["categories"] = [
{"id": c["id"], "description": c["description"]}
for c in catalog["categories"]
]
if not with_categories:
return {"apps": catalog["apps"]}
else:
return {"apps": catalog["apps"], "categories": catalog["categories"]}
def app_search(string):
"""
Return a dict of apps whose description or name match the search string
"""
# Retrieve a simple dict listing all apps
catalog_of_apps = app_catalog()
# Selecting apps according to a match in app name or description
matching_apps = {"apps": {}}
for app in catalog_of_apps["apps"].items():
if re.search(string, app[0], flags=re.IGNORECASE) or re.search(
string, app[1]["description"], flags=re.IGNORECASE
):
matching_apps["apps"][app[0]] = app[1]
return matching_apps
def _initialize_apps_catalog_system():
"""
This function is meant to intialize the apps_catalog system with YunoHost's default app catalog.
"""
default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}]
try:
logger.debug(
"Initializing apps catalog system with YunoHost's default app list"
)
write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list)
except Exception as e:
raise YunohostError(
"Could not initialize the apps catalog system... : %s" % str(e)
)
logger.success(m18n.n("apps_catalog_init_success"))
def _read_apps_catalog_list():
"""
Read the json corresponding to the list of apps catalogs
"""
try:
list_ = read_yaml(APPS_CATALOG_CONF)
# Support the case where file exists but is empty
# by returning [] if list_ is None
return list_ if list_ else []
except Exception as e:
raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e))
def _actual_apps_catalog_api_url(base_url):
return "{base_url}/v{version}/apps.json".format(
base_url=base_url, version=APPS_CATALOG_API_VERSION
)
def _update_apps_catalog():
"""
Fetches the json for each apps_catalog and update the cache
apps_catalog_list is for example :
[ {"id": "default", "url": "https://app.yunohost.org/default/"} ]
Then for each apps_catalog, the actual json URL to be fetched is like :
https://app.yunohost.org/default/vX/apps.json
And store it in :
/var/cache/yunohost/repo/default.json
"""
apps_catalog_list = _read_apps_catalog_list()
logger.info(m18n.n("apps_catalog_updating"))
# Create cache folder if needed
if not os.path.exists(APPS_CATALOG_CACHE):
logger.debug("Initialize folder for apps catalog cache")
mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root")
for apps_catalog in apps_catalog_list:
apps_catalog_id = apps_catalog["id"]
actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"])
# Fetch the json
try:
apps_catalog_content = download_json(actual_api_url)
except Exception as e:
raise YunohostError(
"apps_catalog_failed_to_download",
apps_catalog=apps_catalog_id,
error=str(e),
)
# Remember the apps_catalog api version for later
apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION
# Save the apps_catalog data in the cache
cache_file = "{cache_folder}/{list}.json".format(
cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
)
try:
write_to_json(cache_file, apps_catalog_content)
except Exception as e:
raise YunohostError(
"Unable to write cache data for %s apps_catalog : %s"
% (apps_catalog_id, str(e))
)
logger.success(m18n.n("apps_catalog_update_success"))
def _load_apps_catalog():
"""
Read all the apps catalog cache files and build a single dict (merged_catalog)
corresponding to all known apps and categories
"""
merged_catalog = {"apps": {}, "categories": []}
for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]:
# Let's load the json from cache for this catalog
cache_file = "{cache_folder}/{list}.json".format(
cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
)
try:
apps_catalog_content = (
read_json(cache_file) if os.path.exists(cache_file) else None
)
except Exception as e:
raise YunohostError(
"Unable to read cache for apps_catalog %s : %s" % (cache_file, e),
raw_msg=True,
)
# Check that the version of the data matches version ....
# ... otherwise it means we updated yunohost in the meantime
# and need to update the cache for everything to be consistent
if (
not apps_catalog_content
or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION
):
logger.info(m18n.n("apps_catalog_obsolete_cache"))
_update_apps_catalog()
apps_catalog_content = read_json(cache_file)
del apps_catalog_content["from_api_version"]
# Add apps from this catalog to the output
for app, info in apps_catalog_content["apps"].items():
# (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ...
# in which case we keep only the first one found)
if app in merged_catalog["apps"]:
logger.warning(
"Duplicate app %s found between apps catalog %s and %s"
% (app, apps_catalog_id, merged_catalog["apps"][app]["repository"])
)
continue
info["repository"] = apps_catalog_id
merged_catalog["apps"][app] = info
# Annnnd categories
merged_catalog["categories"] += apps_catalog_content["categories"]
return merged_catalog

25
src/yunohost/backup.py
View file

@ -49,10 +49,6 @@ from yunohost.app import (
app_info, app_info,
_is_installed, _is_installed,
_make_environment_for_app_script, _make_environment_for_app_script,
_patch_legacy_helpers,
_patch_legacy_php_versions,
_patch_legacy_php_versions_in_settings,
LEGACY_PHP_VERSION_REPLACEMENTS,
_make_tmp_workdir_for_app, _make_tmp_workdir_for_app,
) )
from yunohost.hook import ( from yunohost.hook import (
@ -1194,6 +1190,7 @@ class RestoreManager:
""" """
Apply dirty patch to redirect php5 and php7.0 files to php7.4 Apply dirty patch to redirect php5 and php7.0 files to php7.4
""" """
from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS
backup_csv = os.path.join(self.work_dir, "backup.csv") backup_csv = os.path.join(self.work_dir, "backup.csv")
@ -1355,6 +1352,11 @@ class RestoreManager:
app_instance_name -- (string) The app name to restore (no app with this app_instance_name -- (string) The app name to restore (no app with this
name should be already install) name should be already install)
""" """
from yunohost.utils.legacy import (
_patch_legacy_php_versions,
_patch_legacy_php_versions_in_settings,
_patch_legacy_helpers,
)
from yunohost.user import user_group_list from yunohost.user import user_group_list
from yunohost.permission import ( from yunohost.permission import (
permission_create, permission_create,
@ -1489,7 +1491,11 @@ class RestoreManager:
logger.debug(m18n.n("restore_running_app_script", app=app_instance_name)) logger.debug(m18n.n("restore_running_app_script", app=app_instance_name))
# Prepare env. var. to pass to script # Prepare env. var. to pass to script
env_dict = _make_environment_for_app_script(app_instance_name) # FIXME : workdir should be a tmp workdir
app_workdir = os.path.join(self.work_dir, "apps", app_instance_name, "settings")
env_dict = _make_environment_for_app_script(
app_instance_name, workdir=app_workdir
)
env_dict.update( env_dict.update(
{ {
"YNH_BACKUP_DIR": self.work_dir, "YNH_BACKUP_DIR": self.work_dir,
@ -1497,9 +1503,6 @@ class RestoreManager:
"YNH_APP_BACKUP_DIR": os.path.join( "YNH_APP_BACKUP_DIR": os.path.join(
self.work_dir, "apps", app_instance_name, "backup" self.work_dir, "apps", app_instance_name, "backup"
), ),
"YNH_APP_BASEDIR": os.path.join(
self.work_dir, "apps", app_instance_name, "settings"
),
} }
) )
@ -1536,11 +1539,9 @@ class RestoreManager:
remove_script = os.path.join(app_scripts_in_archive, "remove") remove_script = os.path.join(app_scripts_in_archive, "remove")
# Setup environment for remove script # Setup environment for remove script
env_dict_remove = _make_environment_for_app_script(app_instance_name) env_dict_remove = _make_environment_for_app_script(
env_dict_remove["YNH_APP_BASEDIR"] = os.path.join( app_instance_name, workdir=app_workdir
self.work_dir, "apps", app_instance_name, "settings"
) )
remove_operation_logger = OperationLogger( remove_operation_logger = OperationLogger(
"remove_on_failed_restore", "remove_on_failed_restore",
[("app", app_instance_name)], [("app", app_instance_name)],

52
src/yunohost/certificate.py
View file

@ -845,14 +845,9 @@ def _backup_current_cert(domain):
def _check_domain_is_ready_for_ACME(domain): def _check_domain_is_ready_for_ACME(domain):
dnsrecords = ( from yunohost.domain import _get_parent_domain_of
Diagnoser.get_cached_report( from yunohost.dns import _get_dns_zone_for_domain
"dnsrecords",
item={"domain": domain, "category": "basic"},
warn_if_no_cache=False,
)
or {}
)
httpreachable = ( httpreachable = (
Diagnoser.get_cached_report( Diagnoser.get_cached_report(
"web", item={"domain": domain}, warn_if_no_cache=False "web", item={"domain": domain}, warn_if_no_cache=False
@ -860,16 +855,47 @@ def _check_domain_is_ready_for_ACME(domain):
or {} or {}
) )
if not dnsrecords or not httpreachable: parent_domain = _get_parent_domain_of(domain)
dnsrecords = (
Diagnoser.get_cached_report(
"dnsrecords",
item={"domain": parent_domain, "category": "basic"},
warn_if_no_cache=False,
)
or {}
)
base_dns_zone = _get_dns_zone_for_domain(domain)
record_name = (
domain.replace(f".{base_dns_zone}", "") if domain != base_dns_zone else "@"
)
A_record_status = dnsrecords.get("data").get(f"A:{record_name}")
AAAA_record_status = dnsrecords.get("data").get(f"AAAA:{record_name}")
# Fallback to wildcard in case no result yet for the DNS name?
if not A_record_status:
A_record_status = dnsrecords.get("data").get("A:*")
if not AAAA_record_status:
AAAA_record_status = dnsrecords.get("data").get("AAAA:*")
if (
not httpreachable
or not dnsrecords.get("data")
or (A_record_status, AAAA_record_status) == (None, None)
):
raise YunohostValidationError( raise YunohostValidationError(
"certmanager_domain_not_diagnosed_yet", domain=domain "certmanager_domain_not_diagnosed_yet", domain=domain
) )
# Check if IP from DNS matches public IP # Check if IP from DNS matches public IP
if not dnsrecords.get("status") in [ # - 'MISSING' for IPv6 ain't critical for ACME
"SUCCESS", # - IPv4 can be None assuming there's at least an IPv6, and viveversa
"WARNING", # - (the case where both are None is checked before)
]: # Warning is for missing IPv6 record which ain't critical for ACME if not (
A_record_status in [None, "OK"]
and AAAA_record_status in [None, "OK", "MISSING"]
):
raise YunohostValidationError( raise YunohostValidationError(
"certmanager_domain_dns_ip_differs_from_public_ip", domain=domain "certmanager_domain_dns_ip_differs_from_public_ip", domain=domain
) )

3
src/yunohost/data_migrations/0022_php73_to_php74_pools.py
View file

@ -4,7 +4,8 @@ from shutil import copy2
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from yunohost.app import _is_installed, _patch_legacy_php_versions_in_settings from yunohost.app import _is_installed
from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings
from yunohost.tools import Migration from yunohost.tools import Migration
from yunohost.service import _run_service_command from yunohost.service import _run_service_command

59
src/yunohost/dns.py
View file

@ -32,7 +32,7 @@ from collections import OrderedDict
from moulinette import m18n, Moulinette from moulinette import m18n, Moulinette
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, write_to_file, read_toml from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir
from yunohost.domain import ( from yunohost.domain import (
domain_list, domain_list,
@ -40,8 +40,9 @@ from yunohost.domain import (
domain_config_get, domain_config_get,
_get_domain_settings, _get_domain_settings,
_set_domain_settings, _set_domain_settings,
_list_subdomains_of,
) )
from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.utils.dns import dig, is_yunohost_dyndns_domain, is_special_use_tld
from yunohost.utils.error import YunohostValidationError, YunohostError from yunohost.utils.error import YunohostValidationError, YunohostError
from yunohost.utils.network import get_public_ip from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation from yunohost.log import is_unit_operation
@ -61,6 +62,9 @@ def domain_dns_suggest(domain):
""" """
if is_special_use_tld(domain):
return m18n.n("domain_dns_conf_special_use_tld")
_assert_domain_exists(domain) _assert_domain_exists(domain)
dns_conf = _build_dns_conf(domain) dns_conf = _build_dns_conf(domain)
@ -104,18 +108,6 @@ def domain_dns_suggest(domain):
return result return result
def _list_subdomains_of(parent_domain):
_assert_domain_exists(parent_domain)
out = []
for domain in domain_list()["domains"]:
if domain.endswith(f".{parent_domain}"):
out.append(domain)
return out
def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False):
""" """
Internal function that will returns a data structure containing the needed Internal function that will returns a data structure containing the needed
@ -169,10 +161,7 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False):
# If this is a ynh_dyndns_domain, we're not gonna include all the subdomains in the conf # If this is a ynh_dyndns_domain, we're not gonna include all the subdomains in the conf
# Because dynette only accept a specific list of name/type # Because dynette only accept a specific list of name/type
# And the wildcard */A already covers the bulk of use cases # And the wildcard */A already covers the bulk of use cases
if any( if is_yunohost_dyndns_domain(base_domain):
base_domain.endswith("." + ynh_dyndns_domain)
for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS
):
subdomains = [] subdomains = []
else: else:
subdomains = _list_subdomains_of(base_domain) subdomains = _list_subdomains_of(base_domain)
@ -297,6 +286,12 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False):
# Defined by custom hooks ships in apps for example ... # Defined by custom hooks ships in apps for example ...
# FIXME : this ain't practical for apps that may want to add
# custom dns records for a subdomain ... there's no easy way for
# an app to compare the base domain is the parent of the subdomain ?
# (On the other hand, in sep 2021, it looks like no app is using
# this mechanism...)
hook_results = hook_callback("custom_dns_rules", args=[base_domain]) hook_results = hook_callback("custom_dns_rules", args=[base_domain])
for hook_name, results in hook_results.items(): for hook_name, results in hook_results.items():
# #
@ -426,9 +421,14 @@ def _get_dns_zone_for_domain(domain):
# First, check if domain is a nohost.me / noho.st / ynh.fr # First, check if domain is a nohost.me / noho.st / ynh.fr
# This is mainly meant to speed up things for "dyndns update" # This is mainly meant to speed up things for "dyndns update"
# ... otherwise we end up constantly doing a bunch of dig requests # ... otherwise we end up constantly doing a bunch of dig requests
for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS: if is_yunohost_dyndns_domain(domain):
if domain.endswith("." + ynh_dyndns_domain): # Keep only foo.nohost.me even if we have subsub.sub.foo.nohost.me
return ynh_dyndns_domain return ".".join(domain.rsplit(".", 3)[-3:])
# Same thing with .local, .test, ... domains
if is_special_use_tld(domain):
# Keep only foo.local even if we have subsub.sub.foo.local
return ".".join(domain.rsplit(".", 2)[-2:])
# Check cache # Check cache
cache_folder = "/var/cache/yunohost/dns_zones" cache_folder = "/var/cache/yunohost/dns_zones"
@ -471,7 +471,7 @@ def _get_dns_zone_for_domain(domain):
# Check if there's a NS record for that domain # Check if there's a NS record for that domain
answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external") answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external")
if answer[0] == "ok": if answer[0] == "ok":
os.system(f"mkdir -p {cache_folder}") mkdir(cache_folder, parents=True, force=True)
write_to_file(cache_file, parent) write_to_file(cache_file, parent)
return parent return parent
@ -520,7 +520,7 @@ def _get_registrar_config_section(domain):
# TODO big project, integrate yunohost's dynette as a registrar-like provider # TODO big project, integrate yunohost's dynette as a registrar-like provider
# TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron...
if dns_zone in YNH_DYNDNS_DOMAINS: if is_yunohost_dyndns_domain(dns_zone):
registrar_infos["registrar"] = OrderedDict( registrar_infos["registrar"] = OrderedDict(
{ {
"type": "alert", "type": "alert",
@ -530,6 +530,15 @@ def _get_registrar_config_section(domain):
} }
) )
return OrderedDict(registrar_infos) return OrderedDict(registrar_infos)
elif is_special_use_tld(dns_zone):
registrar_infos["registrar"] = OrderedDict(
{
"type": "alert",
"style": "info",
"ask": m18n.n("domain_dns_conf_special_use_tld"),
"value": None,
}
)
try: try:
registrar = _relevant_provider_for_domain(dns_zone)[0] registrar = _relevant_provider_for_domain(dns_zone)[0]
@ -603,6 +612,10 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge=
_assert_domain_exists(domain) _assert_domain_exists(domain)
if is_special_use_tld(domain):
logger.info(m18n.n("domain_dns_conf_special_use_tld"))
return {}
if not registrar or registrar == "None": # yes it's None as a string if not registrar or registrar == "None": # yes it's None as a string
raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain)

31
src/yunohost/domain.py
View file

@ -29,7 +29,7 @@ from typing import Dict, Any
from moulinette import m18n, Moulinette from moulinette import m18n, Moulinette
from moulinette.core import MoulinetteError from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm
from yunohost.app import ( from yunohost.app import (
app_ssowatconf, app_ssowatconf,
@ -105,6 +105,33 @@ def _assert_domain_exists(domain):
raise YunohostValidationError("domain_name_unknown", domain=domain) raise YunohostValidationError("domain_name_unknown", domain=domain)
def _list_subdomains_of(parent_domain):
_assert_domain_exists(parent_domain)
out = []
for domain in domain_list()["domains"]:
if domain.endswith(f".{parent_domain}"):
out.append(domain)
return out
def _get_parent_domain_of(domain):
_assert_domain_exists(domain)
if "." not in domain:
return domain
parent_domain = domain.split(".", 1)[-1]
if parent_domain not in domain_list()["domains"]:
return domain # Domain is its own parent
else:
return _get_parent_domain_of(parent_domain)
@is_unit_operation() @is_unit_operation()
def domain_add(operation_logger, domain, dyndns=False): def domain_add(operation_logger, domain, dyndns=False):
""" """
@ -301,7 +328,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False):
] ]
for stuff in stuff_to_delete: for stuff in stuff_to_delete:
os.system("rm -rf {stuff}") rm(stuff, force=True, recursive=True)
# Sometime we have weird issues with the regenconf where some files # Sometime we have weird issues with the regenconf where some files
# appears as manually modified even though they weren't touched ... # appears as manually modified even though they weren't touched ...

19
src/yunohost/dyndns.py
View file

@ -33,7 +33,7 @@ import subprocess
from moulinette import m18n from moulinette import m18n
from moulinette.core import MoulinetteError from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import write_to_file, read_file from moulinette.utils.filesystem import write_to_file, read_file, rm, chown, chmod
from moulinette.utils.network import download_json from moulinette.utils.network import download_json
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
@ -152,13 +152,12 @@ def dyndns_subscribe(
os.system( os.system(
"cd /etc/yunohost/dyndns && " "cd /etc/yunohost/dyndns && "
"dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s" f"dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER {domain}"
% domain
)
os.system(
"chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private"
) )
chmod("/etc/yunohost/dyndns", 0o600, recursive=True)
chown("/etc/yunohost/dyndns", "root", recursive=True)
private_file = glob.glob("/etc/yunohost/dyndns/*%s*.private" % domain)[0] private_file = glob.glob("/etc/yunohost/dyndns/*%s*.private" % domain)[0]
key_file = glob.glob("/etc/yunohost/dyndns/*%s*.key" % domain)[0] key_file = glob.glob("/etc/yunohost/dyndns/*%s*.key" % domain)[0]
with open(key_file) as f: with open(key_file) as f:
@ -175,12 +174,12 @@ def dyndns_subscribe(
timeout=30, timeout=30,
) )
except Exception as e: except Exception as e:
os.system("rm -f %s" % private_file) rm(private_file, force=True)
os.system("rm -f %s" % key_file) rm(key_file, force=True)
raise YunohostError("dyndns_registration_failed", error=str(e)) raise YunohostError("dyndns_registration_failed", error=str(e))
if r.status_code != 201: if r.status_code != 201:
os.system("rm -f %s" % private_file) rm(private_file, force=True)
os.system("rm -f %s" % key_file) rm(key_file, force=True)
try: try:
error = json.loads(r.text)["error"] error = json.loads(r.text)["error"]
except Exception: except Exception:

7
src/yunohost/firewall.py
View file

@ -31,7 +31,6 @@ from moulinette import m18n
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from moulinette.utils import process from moulinette.utils import process
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.text import prependlines
FIREWALL_FILE = "/etc/yunohost/firewall.yml" FIREWALL_FILE = "/etc/yunohost/firewall.yml"
UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp" UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp"
@ -240,7 +239,7 @@ def firewall_reload(skip_upnp=False):
except process.CalledProcessError as e: except process.CalledProcessError as e:
logger.debug( logger.debug(
"iptables seems to be not available, it outputs:\n%s", "iptables seems to be not available, it outputs:\n%s",
prependlines(e.output.rstrip(), "> "), e.output.decode().strip(),
) )
logger.warning(m18n.n("iptables_unavailable")) logger.warning(m18n.n("iptables_unavailable"))
else: else:
@ -273,7 +272,7 @@ def firewall_reload(skip_upnp=False):
except process.CalledProcessError as e: except process.CalledProcessError as e:
logger.debug( logger.debug(
"ip6tables seems to be not available, it outputs:\n%s", "ip6tables seems to be not available, it outputs:\n%s",
prependlines(e.output.rstrip(), "> "), e.output.decode().strip(),
) )
logger.warning(m18n.n("ip6tables_unavailable")) logger.warning(m18n.n("ip6tables_unavailable"))
else: else:
@ -526,6 +525,6 @@ def _on_rule_command_error(returncode, cmd, output):
'"%s" returned non-zero exit status %d:\n%s', '"%s" returned non-zero exit status %d:\n%s',
cmd, cmd,
returncode, returncode,
prependlines(output.rstrip(), "> "), output.decode().strip(),
) )
return True return True

5
src/yunohost/hook.py
View file

@ -34,7 +34,7 @@ from importlib import import_module
from moulinette import m18n, Moulinette from moulinette import m18n, Moulinette
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from moulinette.utils import log from moulinette.utils import log
from moulinette.utils.filesystem import read_yaml from moulinette.utils.filesystem import read_yaml, cp
HOOK_FOLDER = "/usr/share/yunohost/hooks/" HOOK_FOLDER = "/usr/share/yunohost/hooks/"
CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/"
@ -60,8 +60,7 @@ def hook_add(app, file):
os.makedirs(CUSTOM_HOOK_FOLDER + action) os.makedirs(CUSTOM_HOOK_FOLDER + action)
finalpath = CUSTOM_HOOK_FOLDER + action + "/" + priority + "-" + app finalpath = CUSTOM_HOOK_FOLDER + action + "/" + priority + "-" + app
os.system("cp %s %s" % (file, finalpath)) cp(file, finalpath)
os.system("chown -hR admin: %s" % HOOK_FOLDER)
return {"hook": finalpath} return {"hook": finalpath}

2
src/yunohost/tests/test_appscatalog.py → src/yunohost/tests/test_app_catalog.py
View file

@ -9,7 +9,7 @@ from moulinette import m18n
from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml
from yunohost.utils.error import YunohostError from yunohost.utils.error import YunohostError
from yunohost.app import ( from yunohost.app_catalog import (
_initialize_apps_catalog_system, _initialize_apps_catalog_system,
_read_apps_catalog_list, _read_apps_catalog_list,
_update_apps_catalog, _update_apps_catalog,

6
src/yunohost/tests/test_app_config.py
View file

@ -2,9 +2,11 @@ import glob
import os import os
import shutil import shutil
import pytest import pytest
from mock import patch
from .conftest import get_test_apps_dir from .conftest import get_test_apps_dir
from moulinette import Moulinette
from moulinette.utils.filesystem import read_file from moulinette.utils.filesystem import read_file
from yunohost.domain import _get_maindomain from yunohost.domain import _get_maindomain
@ -146,7 +148,9 @@ def test_app_config_regular_setting(config_app):
assert app_config_get(config_app, "main.components.boolean") == "1" assert app_config_get(config_app, "main.components.boolean") == "1"
assert app_setting(config_app, "boolean") == "1" assert app_setting(config_app, "boolean") == "1"
with pytest.raises(YunohostValidationError): with pytest.raises(YunohostValidationError), patch.object(
os, "isatty", return_value=False
), patch.object(Moulinette, "prompt", return_value="pwet"):
app_config_set(config_app, "main.components.boolean", "pwet") app_config_set(config_app, "main.components.boolean", "pwet")

48
src/yunohost/tests/test_appurl.py
View file

@ -4,7 +4,7 @@ import os
from .conftest import get_test_apps_dir from .conftest import get_test_apps_dir
from yunohost.utils.error import YunohostError from yunohost.utils.error import YunohostError
from yunohost.app import app_install, app_remove, _normalize_domain_path from yunohost.app import app_install, app_remove, _is_app_repo_url
from yunohost.domain import _get_maindomain, domain_url_available from yunohost.domain import _get_maindomain, domain_url_available
from yunohost.permission import _validate_and_sanitize_permission_url from yunohost.permission import _validate_and_sanitize_permission_url
@ -28,20 +28,42 @@ def teardown_function(function):
pass pass
def test_normalize_domain_path(): def test_repo_url_definition():
assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh")
assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh/")
assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh.git")
assert _is_app_repo_url(
"https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing"
)
assert _is_app_repo_url(
"https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing/"
)
assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo-bar-123_ynh")
assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo_bar_123_ynh")
assert _is_app_repo_url("https://github.com/YunoHost-Apps/FooBar123_ynh")
assert _is_app_repo_url("https://github.com/labriqueinternet/vpnclient_ynh")
assert _is_app_repo_url("https://framagit.org/YunoHost/apps/nodebb_ynh")
assert _is_app_repo_url(
"https://framagit.org/YunoHost/apps/nodebb_ynh/-/tree/testing"
)
assert _is_app_repo_url("https://gitlab.com/yunohost-apps/foobar_ynh")
assert _is_app_repo_url("https://code.antopie.org/miraty/qr_ynh")
assert _is_app_repo_url(
"https://gitlab.domainepublic.net/Neutrinet/neutrinet_ynh/-/tree/unstable"
)
assert _is_app_repo_url("git@github.com:YunoHost-Apps/foobar_ynh.git")
assert _normalize_domain_path("https://yolo.swag/", "macnuggets") == ( assert not _is_app_repo_url("github.com/YunoHost-Apps/foobar_ynh")
"yolo.swag", assert not _is_app_repo_url("http://github.com/YunoHost-Apps/foobar_ynh")
"/macnuggets", assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_wat")
) assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_ynh_wat")
assert _normalize_domain_path("http://yolo.swag", "/macnuggets/") == ( assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar/tree/testing")
"yolo.swag", assert not _is_app_repo_url(
"/macnuggets", "https://github.com/YunoHost-Apps/foobar_ynh_wat/tree/testing"
)
assert _normalize_domain_path("yolo.swag/", "macnuggets/") == (
"yolo.swag",
"/macnuggets",
) )
assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/")
assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet")
assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet_foo")
def test_urlavailable(): def test_urlavailable():

9
src/yunohost/tests/test_dns.py
View file

@ -34,8 +34,13 @@ def test_get_dns_zone_from_domain_existing():
assert ( assert (
_get_dns_zone_for_domain("non-existing-domain.yunohost.org") == "yunohost.org" _get_dns_zone_for_domain("non-existing-domain.yunohost.org") == "yunohost.org"
) )
assert _get_dns_zone_for_domain("yolo.nohost.me") == "nohost.me" assert _get_dns_zone_for_domain("yolo.nohost.me") == "yolo.nohost.me"
assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "nohost.me" assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "yolo.nohost.me"
assert _get_dns_zone_for_domain("bar.foo.yolo.nohost.me") == "yolo.nohost.me"
assert _get_dns_zone_for_domain("yolo.test") == "yolo.test"
assert _get_dns_zone_for_domain("foo.yolo.test") == "yolo.test"
assert _get_dns_zone_for_domain("yolo.tld") == "yolo.tld" assert _get_dns_zone_for_domain("yolo.tld") == "yolo.tld"
assert _get_dns_zone_for_domain("foo.yolo.tld") == "yolo.tld" assert _get_dns_zone_for_domain("foo.yolo.tld") == "yolo.tld"

6
src/yunohost/tests/test_permission.py
View file

@ -1049,7 +1049,7 @@ def test_permission_app_remove():
def test_permission_app_change_url(): def test_permission_app_change_url():
app_install( app_install(
os.path.join(get_test_apps_dir(), "permissions_app_ynh"), os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
args="domain=%s&domain_2=%s&path=%s&admin=%s" args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s"
% (maindomain, other_domains[0], "/urlpermissionapp", "alice"), % (maindomain, other_domains[0], "/urlpermissionapp", "alice"),
force=True, force=True,
) )
@ -1072,7 +1072,7 @@ def test_permission_app_change_url():
def test_permission_protection_management_by_helper(): def test_permission_protection_management_by_helper():
app_install( app_install(
os.path.join(get_test_apps_dir(), "permissions_app_ynh"), os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
args="domain=%s&domain_2=%s&path=%s&admin=%s" args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s"
% (maindomain, other_domains[0], "/urlpermissionapp", "alice"), % (maindomain, other_domains[0], "/urlpermissionapp", "alice"),
force=True, force=True,
) )
@ -1135,7 +1135,7 @@ def test_permission_legacy_app_propagation_on_ssowat():
app_install( app_install(
os.path.join(get_test_apps_dir(), "legacy_app_ynh"), os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
args="domain=%s&domain_2=%s&path=%s" args="domain=%s&domain_2=%s&path=%s&is_public=1"
% (maindomain, other_domains[0], "/legacy"), % (maindomain, other_domains[0], "/legacy"),
force=True, force=True,
) )

802
src/yunohost/tests/test_questions.py
View file

File diff suppressed because it is too large Load diff

34
src/yunohost/tools.py
View file

@ -34,13 +34,15 @@ from typing import List
from moulinette import Moulinette, m18n from moulinette import Moulinette, m18n
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.process import check_output, call_async_output from moulinette.utils.process import check_output, call_async_output
from moulinette.utils.filesystem import read_yaml, write_to_yaml from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm
from yunohost.app import ( from yunohost.app import (
_update_apps_catalog,
app_info, app_info,
app_upgrade, app_upgrade,
)
from yunohost.app_catalog import (
_initialize_apps_catalog_system, _initialize_apps_catalog_system,
_update_apps_catalog,
) )
from yunohost.domain import domain_add from yunohost.domain import domain_add
from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.dyndns import _dyndns_available, _dyndns_provides
@ -1118,12 +1120,14 @@ class Migration(object):
backup_folder = "/home/yunohost.backup/premigration/" + time.strftime( backup_folder = "/home/yunohost.backup/premigration/" + time.strftime(
"%Y%m%d-%H%M%S", time.gmtime() "%Y%m%d-%H%M%S", time.gmtime()
) )
os.makedirs(backup_folder, 0o750) mkdir(backup_folder, 0o750, parents=True)
os.system("systemctl stop slapd") os.system("systemctl stop slapd")
os.system(f"cp -r --preserve /etc/ldap {backup_folder}/ldap_config") cp("/etc/ldap", f"{backup_folder}/ldap_config", recursive=True)
os.system(f"cp -r --preserve /var/lib/ldap {backup_folder}/ldap_db") cp("/var/lib/ldap", f"{backup_folder}/ldap_db", recursive=True)
os.system( cp(
f"cp -r --preserve /etc/yunohost/apps {backup_folder}/apps_settings" "/etc/yunohost/apps",
f"{backup_folder}/apps_settings",
recursive=True,
) )
except Exception as e: except Exception as e:
raise YunohostError( raise YunohostError(
@ -1140,17 +1144,19 @@ class Migration(object):
) )
os.system("systemctl stop slapd") os.system("systemctl stop slapd")
# To be sure that we don't keep some part of the old config # To be sure that we don't keep some part of the old config
os.system("rm -r /etc/ldap/slapd.d") rm("/etc/ldap/slapd.d", force=True, recursive=True)
os.system(f"cp -r --preserve {backup_folder}/ldap_config/. /etc/ldap/") cp(f"{backup_folder}/ldap_config", "/etc/ldap", recursive=True)
os.system(f"cp -r --preserve {backup_folder}/ldap_db/. /var/lib/ldap/") cp(f"{backup_folder}/ldap_db", "/var/lib/ldap", recursive=True)
os.system( cp(
f"cp -r --preserve {backup_folder}/apps_settings/. /etc/yunohost/apps/" f"{backup_folder}/apps_settings",
"/etc/yunohost/apps",
recursive=True,
) )
os.system("systemctl start slapd") os.system("systemctl start slapd")
os.system(f"rm -r {backup_folder}") rm(backup_folder, force=True, recursive=True)
logger.info(m18n.n("migration_ldap_rollback_success")) logger.info(m18n.n("migration_ldap_rollback_success"))
raise raise
else: else:
os.system(f"rm -r {backup_folder}") rm(backup_folder, force=True, recursive=True)
return func return func

4
src/yunohost/user.py
View file

@ -413,7 +413,9 @@ def user_update(
# without a specified value, change_password will be set to the const 0. # without a specified value, change_password will be set to the const 0.
# In this case we prompt for the new password. # In this case we prompt for the new password.
if Moulinette.interface.type == "cli" and not change_password: if Moulinette.interface.type == "cli" and not change_password:
change_password = Moulinette.prompt(m18n.n("ask_password"), True, True) change_password = Moulinette.prompt(
m18n.n("ask_password"), is_password=True, confirm=True
)
# Ensure sufficiently complex password # Ensure sufficiently complex password
assert_password_is_strong_enough("user", change_password) assert_password_is_strong_enough("user", change_password)

429
src/yunohost/utils/config.py
View file

@ -25,12 +25,13 @@ import urllib.parse
import tempfile import tempfile
import shutil import shutil
from collections import OrderedDict from collections import OrderedDict
from typing import Optional, Dict, List from typing import Optional, Dict, List, Union, Any, Mapping
from moulinette.interfaces.cli import colorize from moulinette.interfaces.cli import colorize
from moulinette import Moulinette, m18n from moulinette import Moulinette, m18n
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import ( from moulinette.utils.filesystem import (
read_file,
write_to_file, write_to_file,
read_toml, read_toml,
read_yaml, read_yaml,
@ -99,6 +100,11 @@ class ConfigPanel:
result[key]["value"] = question_class.humanize( result[key]["value"] = question_class.humanize(
option["current_value"], option option["current_value"], option
) )
# FIXME: semantics, technically here this is not about a prompt...
if question_class.hide_user_input_in_prompt:
result[key][
"value"
] = "**************" # Prevent displaying password in `config get`
if mode == "full": if mode == "full":
return self.config return self.config
@ -164,6 +170,9 @@ class ConfigPanel:
raise raise
finally: finally:
# Delete files uploaded from API # Delete files uploaded from API
# FIXME : this is currently done in the context of config panels,
# but could also happen in the context of app install ... (or anywhere else
# where we may parse args etc...)
FileQuestion.clean_upload_dirs() FileQuestion.clean_upload_dirs()
self._reload_services() self._reload_services()
@ -198,20 +207,20 @@ class ConfigPanel:
# Transform toml format into internal format # Transform toml format into internal format
format_description = { format_description = {
"toml": { "root": {
"properties": ["version", "i18n"], "properties": ["version", "i18n"],
"default": {"version": 1.0}, "defaults": {"version": 1.0},
}, },
"panels": { "panels": {
"properties": ["name", "services", "actions", "help"], "properties": ["name", "services", "actions", "help"],
"default": { "defaults": {
"services": [], "services": [],
"actions": {"apply": {"en": "Apply"}}, "actions": {"apply": {"en": "Apply"}},
}, },
}, },
"sections": { "sections": {
"properties": ["name", "services", "optional", "help", "visible"], "properties": ["name", "services", "optional", "help", "visible"],
"default": { "defaults": {
"name": "", "name": "",
"services": [], "services": [],
"optional": True, "optional": True,
@ -241,11 +250,11 @@ class ConfigPanel:
"accept", "accept",
"redact", "redact",
], ],
"default": {}, "defaults": {},
}, },
} }
def convert(toml_node, node_type): def _build_internal_config_panel(raw_infos, level):
"""Convert TOML in internal format ('full' mode used by webadmin) """Convert TOML in internal format ('full' mode used by webadmin)
Here are some properties of 1.0 config panel in toml: Here are some properties of 1.0 config panel in toml:
- node properties and node children are mixed, - node properties and node children are mixed,
@ -253,48 +262,47 @@ class ConfigPanel:
- some properties have default values - some properties have default values
This function detects all children nodes and put them in a list This function detects all children nodes and put them in a list
""" """
# Prefill the node default keys if needed
default = format_description[node_type]["default"]
node = {key: toml_node.get(key, value) for key, value in default.items()}
properties = format_description[node_type]["properties"] defaults = format_description[level]["defaults"]
properties = format_description[level]["properties"]
# Define the filter_key part to use and the children type # Start building the ouput (merging the raw infos + defaults)
i = list(format_description).index(node_type) out = {key: raw_infos.get(key, value) for key, value in defaults.items()}
subnode_type = (
list(format_description)[i + 1] if node_type != "options" else None # Now fill the sublevels (+ apply filter_key)
) i = list(format_description).index(level)
sublevel = list(format_description)[i + 1] if level != "options" else None
search_key = filter_key[i] if len(filter_key) > i else False search_key = filter_key[i] if len(filter_key) > i else False
for key, value in toml_node.items(): for key, value in raw_infos.items():
# Key/value are a child node # Key/value are a child node
if ( if (
isinstance(value, OrderedDict) isinstance(value, OrderedDict)
and key not in properties and key not in properties
and subnode_type and sublevel
): ):
# We exclude all nodes not referenced by the filter_key # We exclude all nodes not referenced by the filter_key
if search_key and key != search_key: if search_key and key != search_key:
continue continue
subnode = convert(value, subnode_type) subnode = _build_internal_config_panel(value, sublevel)
subnode["id"] = key subnode["id"] = key
if node_type == "toml": if level == "root":
subnode.setdefault("name", {"en": key.capitalize()}) subnode.setdefault("name", {"en": key.capitalize()})
elif node_type == "sections": elif level == "sections":
subnode["name"] = key # legacy subnode["name"] = key # legacy
subnode.setdefault("optional", toml_node.get("optional", True)) subnode.setdefault("optional", raw_infos.get("optional", True))
node.setdefault(subnode_type, []).append(subnode) out.setdefault(sublevel, []).append(subnode)
# Key/value are a property # Key/value are a property
else: else:
if key not in properties: if key not in properties:
logger.warning(f"Unknown key '{key}' found in config toml") logger.warning(f"Unknown key '{key}' found in config panel")
# Todo search all i18n keys # Todo search all i18n keys
node[key] = ( out[key] = (
value if key not in ["ask", "help", "name"] else {"en": value} value if key not in ["ask", "help", "name"] else {"en": value}
) )
return node return out
self.config = convert(toml_config_panel, "toml") self.config = _build_internal_config_panel(toml_config_panel, "root")
try: try:
self.config["panels"][0]["sections"][0]["options"][0] self.config["panels"][0]["sections"][0]["options"][0]
@ -376,14 +384,15 @@ class ConfigPanel:
display_header(f"\n# {name}") display_header(f"\n# {name}")
# Check and ask unanswered questions # Check and ask unanswered questions
questions = ask_questions_and_parse_answers(section["options"], self.args)
self.new_values.update( self.new_values.update(
parse_args_in_yunohost_format(self.args, section["options"]) {
) question.name: question.value
self.new_values = { for question in questions
key: value[0] if question.value is not None
for key, value in self.new_values.items()
if not value[0] is None
} }
)
self.errors = None self.errors = None
def _get_default_values(self): def _get_default_values(self):
@ -457,18 +466,20 @@ class Question(object):
hide_user_input_in_prompt = False hide_user_input_in_prompt = False
pattern: Optional[Dict] = None pattern: Optional[Dict] = None
def __init__(self, question, user_answers): def __init__(self, question: Dict[str, Any]):
self.name = question["name"] self.name = question["name"]
self.type = question.get("type", "string") self.type = question.get("type", "string")
self.default = question.get("default", None) self.default = question.get("default", None)
self.current_value = question.get("current_value")
self.optional = question.get("optional", False) self.optional = question.get("optional", False)
self.choices = question.get("choices", []) self.choices = question.get("choices", [])
self.pattern = question.get("pattern", self.pattern) self.pattern = question.get("pattern", self.pattern)
self.ask = question.get("ask", {"en": self.name}) self.ask = question.get("ask", {"en": self.name})
self.help = question.get("help") self.help = question.get("help")
self.value = user_answers.get(self.name)
self.redact = question.get("redact", False) self.redact = question.get("redact", False)
# .current_value is the currently stored value
self.current_value = question.get("current_value")
# .value is the "proposed" value which we got from the user
self.value = question.get("value")
# Empty value is parsed as empty string # Empty value is parsed as empty string
if self.default == "": if self.default == "":
@ -480,6 +491,8 @@ class Question(object):
@staticmethod @staticmethod
def normalize(value, option={}): def normalize(value, option={}):
if isinstance(value, str):
value = value.strip()
return value return value
def _prompt(self, text): def _prompt(self, text):
@ -491,9 +504,11 @@ class Question(object):
self.value = Moulinette.prompt( self.value = Moulinette.prompt(
message=text, message=text,
is_password=self.hide_user_input_in_prompt, is_password=self.hide_user_input_in_prompt,
confirm=False, # We doesn't want to confirm this kind of password like in webadmin confirm=False,
prefill=prefill, prefill=prefill,
is_multiline=(self.type == "text"), is_multiline=(self.type == "text"),
autocomplete=self.choices,
help=_value_for_locale(self.help),
) )
def ask_if_needed(self): def ask_if_needed(self):
@ -513,12 +528,9 @@ class Question(object):
): ):
self.value = class_default if self.default is None else self.default self.value = class_default if self.default is None else self.default
# Normalization
# This is done to enforce a certain formating like for boolean
self.value = self.normalize(self.value, self)
# Prevalidation
try: try:
# Normalize and validate
self.value = self.normalize(self.value, self)
self._prevalidate() self._prevalidate()
except YunohostValidationError as e: except YunohostValidationError as e:
# If in interactive cli, re-ask the current question # If in interactive cli, re-ask the current question
@ -531,9 +543,10 @@ class Question(object):
raise raise
break break
self.value = self._post_parse_value() self.value = self._post_parse_value()
return (self.value, self.argument_type) return self.value
def _prevalidate(self): def _prevalidate(self):
if self.value in [None, ""] and not self.optional: if self.value in [None, ""] and not self.optional:
@ -542,7 +555,12 @@ class Question(object):
# we have an answer, do some post checks # we have an answer, do some post checks
if self.value not in [None, ""]: if self.value not in [None, ""]:
if self.choices and self.value not in self.choices: if self.choices and self.value not in self.choices:
self._raise_invalid_answer() raise YunohostValidationError(
"app_argument_choice_invalid",
name=self.name,
value=self.value,
choices=", ".join(self.choices),
)
if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): if self.pattern and not re.match(self.pattern["regexp"], str(self.value)):
raise YunohostValidationError( raise YunohostValidationError(
self.pattern["error"], self.pattern["error"],
@ -550,25 +568,31 @@ class Question(object):
value=self.value, value=self.value,
) )
def _raise_invalid_answer(self): def _format_text_for_user_input_in_cli(self):
raise YunohostValidationError(
"app_argument_choice_invalid",
name=self.name,
value=self.value,
choices=", ".join(self.choices),
)
def _format_text_for_user_input_in_cli(self, column=False):
text_for_user_input_in_cli = _value_for_locale(self.ask) text_for_user_input_in_cli = _value_for_locale(self.ask)
if self.choices: if self.choices:
text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices))
if self.help or column: # Prevent displaying a shitload of choices
text_for_user_input_in_cli += ":\033[m" # (e.g. 100+ available users when choosing an app admin...)
if self.help: choices = (
text_for_user_input_in_cli += "\n - " list(self.choices.values())
text_for_user_input_in_cli += _value_for_locale(self.help) if isinstance(self.choices, dict)
else self.choices
)
choices_to_display = choices[:20]
remaining_choices = len(choices[20:])
if remaining_choices > 0:
choices_to_display += [
m18n.n("other_available_options", n=remaining_choices)
]
choices_to_display = " | ".join(choices_to_display)
text_for_user_input_in_cli += f" [{choices_to_display}]"
return text_for_user_input_in_cli return text_for_user_input_in_cli
def _post_parse_value(self): def _post_parse_value(self):
@ -659,6 +683,8 @@ class TagsQuestion(Question):
def normalize(value, option={}): def normalize(value, option={}):
if isinstance(value, list): if isinstance(value, list):
return ",".join(value) return ",".join(value)
if isinstance(value, str):
value = value.strip()
return value return value
def _prevalidate(self): def _prevalidate(self):
@ -684,20 +710,14 @@ class PasswordQuestion(Question):
default_value = "" default_value = ""
forbidden_chars = "{}" forbidden_chars = "{}"
def __init__(self, question, user_answers): def __init__(self, question):
super().__init__(question, user_answers) super().__init__(question)
self.redact = True self.redact = True
if self.default is not None: if self.default is not None:
raise YunohostValidationError( raise YunohostValidationError(
"app_argument_password_no_default", name=self.name "app_argument_password_no_default", name=self.name
) )
@staticmethod
def humanize(value, option={}):
if value:
return "********" # Avoid to display the password on screen
return ""
def _prevalidate(self): def _prevalidate(self):
super()._prevalidate() super()._prevalidate()
@ -712,34 +732,31 @@ class PasswordQuestion(Question):
assert_password_is_strong_enough("user", self.value) assert_password_is_strong_enough("user", self.value)
def _format_text_for_user_input_in_cli(self):
need_column = self.current_value or self.optional
text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli(
need_column
)
if self.current_value:
text_for_user_input_in_cli += "\n - " + m18n.n(
"app_argument_password_help_keep"
)
if self.optional:
text_for_user_input_in_cli += "\n - " + m18n.n(
"app_argument_password_help_optional"
)
return text_for_user_input_in_cli
def _prompt(self, text):
super()._prompt(text)
if self.current_value and self.value == "":
self.value = self.current_value
elif self.value == " ":
self.value = ""
class PathQuestion(Question): class PathQuestion(Question):
argument_type = "path" argument_type = "path"
default_value = "" default_value = ""
@staticmethod
def normalize(value, option={}):
option = option.__dict__ if isinstance(option, Question) else option
if not value.strip():
if option.get("optional"):
return ""
# Hmpf here we could just have a "else" case
# but we also want PathQuestion.normalize("") to return "/"
# (i.e. if no option is provided, hence .get("optional") is None
elif option.get("optional") is False:
raise YunohostValidationError(
"app_argument_invalid",
name=option.get("name"),
error="Question is mandatory",
)
return "/" + value.strip().strip(" /")
class BooleanQuestion(Question): class BooleanQuestion(Question):
argument_type = "boolean" argument_type = "boolean"
@ -750,50 +767,70 @@ class BooleanQuestion(Question):
@staticmethod @staticmethod
def humanize(value, option={}): def humanize(value, option={}):
option = option.__dict__ if isinstance(option, Question) else option
yes = option.get("yes", 1) yes = option.get("yes", 1)
no = option.get("no", 0) no = option.get("no", 0)
value = str(value).lower()
if value == str(yes).lower():
return "yes"
if value == str(no).lower():
return "no"
if value in BooleanQuestion.yes_answers:
return "yes"
if value in BooleanQuestion.no_answers:
return "no"
if value in ["none", ""]: value = BooleanQuestion.normalize(value, option)
if value == yes:
return "yes"
if value == no:
return "no"
if value is None:
return "" return ""
raise YunohostValidationError( raise YunohostValidationError(
"app_argument_choice_invalid", "app_argument_choice_invalid",
name=option.get("name", ""), name=option.get("name"),
value=value, value=value,
choices="yes, no, y, n, 1, 0", choices="yes/no",
) )
@staticmethod @staticmethod
def normalize(value, option={}): def normalize(value, option={}):
yes = option.get("yes", 1)
no = option.get("no", 0)
if str(value).lower() in BooleanQuestion.yes_answers: option = option.__dict__ if isinstance(option, Question) else option
return yes
if str(value).lower() in BooleanQuestion.no_answers: if isinstance(value, str):
return no value = value.strip()
if value in [None, ""]: technical_yes = option.get("yes", 1)
technical_no = option.get("no", 0)
no_answers = BooleanQuestion.no_answers
yes_answers = BooleanQuestion.yes_answers
assert (
str(technical_yes).lower() not in no_answers
), f"'yes' value can't be in {no_answers}"
assert (
str(technical_no).lower() not in yes_answers
), f"'no' value can't be in {yes_answers}"
no_answers += [str(technical_no).lower()]
yes_answers += [str(technical_yes).lower()]
strvalue = str(value).lower()
if strvalue in yes_answers:
return technical_yes
if strvalue in no_answers:
return technical_no
if strvalue in ["none", ""]:
return None return None
raise YunohostValidationError( raise YunohostValidationError(
"app_argument_choice_invalid", "app_argument_choice_invalid",
name=option.get("name", ""), name=option.get("name"),
value=value, value=strvalue,
choices="yes, no, y, n, 1, 0", choices="yes/no",
) )
def __init__(self, question, user_answers): def __init__(self, question):
super().__init__(question, user_answers) super().__init__(question)
self.yes = question.get("yes", 1) self.yes = question.get("yes", 1)
self.no = question.get("no", 0) self.no = question.get("no", 0)
if self.default is None: if self.default is None:
@ -807,42 +844,44 @@ class BooleanQuestion(Question):
return text_for_user_input_in_cli return text_for_user_input_in_cli
def get(self, key, default=None): def get(self, key, default=None):
try: return getattr(self, key, default)
return getattr(self, key)
except AttributeError:
return default
class DomainQuestion(Question): class DomainQuestion(Question):
argument_type = "domain" argument_type = "domain"
def __init__(self, question, user_answers): def __init__(self, question):
from yunohost.domain import domain_list, _get_maindomain from yunohost.domain import domain_list, _get_maindomain
super().__init__(question, user_answers) super().__init__(question)
if self.default is None: if self.default is None:
self.default = _get_maindomain() self.default = _get_maindomain()
self.choices = domain_list()["domains"] self.choices = domain_list()["domains"]
def _raise_invalid_answer(self): @staticmethod
raise YunohostValidationError( def normalize(value, option={}):
"app_argument_invalid", if value.startswith("https://"):
name=self.name, value = value[len("https://") :]
error=m18n.n("domain_name_unknown", domain=self.value), elif value.startswith("http://"):
) value = value[len("http://") :]
# Remove trailing slashes
value = value.rstrip("/").lower()
return value
class UserQuestion(Question): class UserQuestion(Question):
argument_type = "user" argument_type = "user"
def __init__(self, question, user_answers): def __init__(self, question):
from yunohost.user import user_list, user_info from yunohost.user import user_list, user_info
from yunohost.domain import _get_maindomain from yunohost.domain import _get_maindomain
super().__init__(question, user_answers) super().__init__(question)
self.choices = user_list()["users"] self.choices = list(user_list()["users"].keys())
if not self.choices: if not self.choices:
raise YunohostValidationError( raise YunohostValidationError(
@ -853,42 +892,42 @@ class UserQuestion(Question):
if self.default is None: if self.default is None:
root_mail = "root@%s" % _get_maindomain() root_mail = "root@%s" % _get_maindomain()
for user in self.choices.keys(): for user in self.choices:
if root_mail in user_info(user).get("mail-aliases", []): if root_mail in user_info(user).get("mail-aliases", []):
self.default = user self.default = user
break break
def _raise_invalid_answer(self):
raise YunohostValidationError(
"app_argument_invalid",
name=self.name,
error=m18n.n("user_unknown", user=self.value),
)
class NumberQuestion(Question): class NumberQuestion(Question):
argument_type = "number" argument_type = "number"
default_value = None default_value = None
def __init__(self, question, user_answers): def __init__(self, question):
super().__init__(question, user_answers) super().__init__(question)
self.min = question.get("min", None) self.min = question.get("min", None)
self.max = question.get("max", None) self.max = question.get("max", None)
self.step = question.get("step", None) self.step = question.get("step", None)
@staticmethod @staticmethod
def normalize(value, option={}): def normalize(value, option={}):
if isinstance(value, int): if isinstance(value, int):
return value return value
if isinstance(value, str):
value = value.strip()
if isinstance(value, str) and value.isdigit(): if isinstance(value, str) and value.isdigit():
return int(value) return int(value)
if value in [None, ""]: if value in [None, ""]:
return value return value
option = option.__dict__ if isinstance(option, Question) else option
raise YunohostValidationError( raise YunohostValidationError(
"app_argument_invalid", name=option.name, error=m18n.n("invalid_number") "app_argument_invalid",
name=option.get("name"),
error=m18n.n("invalid_number"),
) )
def _prevalidate(self): def _prevalidate(self):
@ -915,8 +954,8 @@ class DisplayTextQuestion(Question):
argument_type = "display_text" argument_type = "display_text"
readonly = True readonly = True
def __init__(self, question, user_answers): def __init__(self, question):
super().__init__(question, user_answers) super().__init__(question)
self.optional = True self.optional = True
self.style = question.get( self.style = question.get(
@ -946,90 +985,50 @@ class FileQuestion(Question):
@classmethod @classmethod
def clean_upload_dirs(cls): def clean_upload_dirs(cls):
# Delete files uploaded from API # Delete files uploaded from API
if Moulinette.interface.type == "api":
for upload_dir in cls.upload_dirs: for upload_dir in cls.upload_dirs:
if os.path.exists(upload_dir): if os.path.exists(upload_dir):
shutil.rmtree(upload_dir) shutil.rmtree(upload_dir)
def __init__(self, question, user_answers): def __init__(self, question):
super().__init__(question, user_answers) super().__init__(question)
if question.get("accept"): self.accept = question.get("accept", "")
self.accept = question.get("accept")
else:
self.accept = ""
if Moulinette.interface.type == "api":
if user_answers.get(f"{self.name}[name]"):
self.value = {
"content": self.value,
"filename": user_answers.get(f"{self.name}[name]", self.name),
}
def _prevalidate(self): def _prevalidate(self):
if self.value is None: if self.value is None:
self.value = self.current_value self.value = self.current_value
super()._prevalidate() super()._prevalidate()
if (
isinstance(self.value, str)
and self.value
and not os.path.exists(self.value)
):
raise YunohostValidationError(
"app_argument_invalid",
name=self.name,
error=m18n.n("file_does_not_exist", path=self.value),
)
if self.value in [None, ""] or not self.accept:
return
filename = self.value if isinstance(self.value, str) else self.value["filename"] if Moulinette.interface.type != "api":
if "." not in filename or "." + filename.split(".")[ if not self.value or not os.path.exists(str(self.value)):
-1
] not in self.accept.replace(" ", "").split(","):
raise YunohostValidationError( raise YunohostValidationError(
"app_argument_invalid", "app_argument_invalid",
name=self.name, name=self.name,
error=m18n.n( error=m18n.n("file_does_not_exist", path=str(self.value)),
"file_extension_not_accepted", file=filename, accept=self.accept
),
) )
def _post_parse_value(self): def _post_parse_value(self):
from base64 import b64decode from base64 import b64decode
# Upload files from API
# A file arg contains a string with "FILENAME:BASE64_CONTENT"
if not self.value: if not self.value:
return self.value return self.value
if Moulinette.interface.type == "api" and isinstance(self.value, dict): upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_")
_, file_path = tempfile.mkstemp(dir=upload_dir)
upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_")
FileQuestion.upload_dirs += [upload_dir] FileQuestion.upload_dirs += [upload_dir]
filename = self.value["filename"]
logger.debug(
f"Save uploaded file {self.value['filename']} from API into {upload_dir}"
)
# Filename is given by user of the API. For security reason, we have replaced logger.debug(f"Saving file {self.name} for file question into {file_path}")
# os.path.join to avoid the user to be able to rewrite a file in filesystem if Moulinette.interface.type != "api":
# i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" content = read_file(str(self.value), file_mode="rb")
file_path = os.path.normpath(upload_dir + "/" + filename)
if not file_path.startswith(upload_dir + "/"):
raise YunohostError(
f"Filename '{filename}' received from the API got a relative parent path, which is forbidden",
raw_msg=True,
)
i = 2
while os.path.exists(file_path):
file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
i += 1
content = self.value["content"] if Moulinette.interface.type == "api":
content = b64decode(self.value)
write_to_file(file_path, b64decode(content), file_mode="wb") write_to_file(file_path, content, file_mode="wb")
self.value = file_path self.value = file_path
return self.value return self.value
@ -1057,25 +1056,41 @@ ARGUMENTS_TYPE_PARSERS = {
} }
def parse_args_in_yunohost_format(user_answers, argument_questions): def ask_questions_and_parse_answers(
questions: Dict, prefilled_answers: Union[str, Mapping[str, Any]] = {}
) -> List[Question]:
"""Parse arguments store in either manifest.json or actions.json or from a """Parse arguments store in either manifest.json or actions.json or from a
config panel against the user answers when they are present. config panel against the user answers when they are present.
Keyword arguments: Keyword arguments:
user_answers -- a dictionnary of arguments from the user (generally questions -- the arguments description store in yunohost
empty in CLI, filed from the admin interface)
argument_questions -- the arguments description store in yunohost
format from actions.json/toml, manifest.json/toml format from actions.json/toml, manifest.json/toml
or config_panel.json/toml or config_panel.json/toml
prefilled_answers -- a url "query-string" such as "domain=yolo.test&path=/foobar&admin=sam"
or a dict such as {"domain": "yolo.test", "path": "/foobar", "admin": "sam"}
""" """
parsed_answers_dict = OrderedDict()
for question in argument_questions: if isinstance(prefilled_answers, str):
# FIXME FIXME : this is not uniform with config_set() which uses parse.qs (no l)
# parse_qsl parse single values
# whereas parse.qs return list of values (which is useful for tags, etc)
# For now, let's not migrate this piece of code to parse_qs
# Because Aleks believes some bits of the app CI rely on overriding values (e.g. foo=foo&...&foo=bar)
prefilled_answers = dict(
urllib.parse.parse_qsl(prefilled_answers or "", keep_blank_values=True)
)
if not prefilled_answers:
prefilled_answers = {}
out = []
for question in questions:
question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")] question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]
question = question_class(question, user_answers) question["value"] = prefilled_answers.get(question["name"])
question = question_class(question)
answer = question.ask_if_needed() question.ask_if_needed()
if answer is not None: out.append(question)
parsed_answers_dict[question.name] = answer
return parsed_answers_dict return out

14
src/yunohost/utils/dns.py
View file

@ -23,6 +23,8 @@ from typing import List
from moulinette.utils.filesystem import read_file from moulinette.utils.filesystem import read_file
SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"]
YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"]
# Lazy dev caching to avoid re-reading the file multiple time when calling # Lazy dev caching to avoid re-reading the file multiple time when calling
@ -30,6 +32,18 @@ YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"]
external_resolvers_: List[str] = [] external_resolvers_: List[str] = []
def is_yunohost_dyndns_domain(domain):
return any(
domain.endswith(f".{dyndns_domain}") for dyndns_domain in YNH_DYNDNS_DOMAINS
)
def is_special_use_tld(domain):
return any(domain.endswith(f".{tld}") for tld in SPECIAL_USE_TLDS)
def external_resolvers(): def external_resolvers():
global external_resolvers_ global external_resolvers_

198
src/yunohost/utils/legacy.py
View file

@ -1,3 +1,20 @@
import os
import re
import glob
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import (
read_file,
write_to_file,
write_to_yaml,
read_yaml,
)
from yunohost.utils.error import YunohostValidationError
logger = getActionLogger("yunohost.legacy")
LEGACY_PERMISSION_LABEL = { LEGACY_PERMISSION_LABEL = {
("nextcloud", "skipped"): "api", # .well-known ("nextcloud", "skipped"): "api", # .well-known
("libreto", "skipped"): "pad access", # /[^/]+ ("libreto", "skipped"): "pad access", # /[^/]+
@ -49,3 +66,184 @@ def legacy_permission_label(app, permission_type):
return LEGACY_PERMISSION_LABEL.get( return LEGACY_PERMISSION_LABEL.get(
(app, permission_type), "Legacy %s urls" % permission_type (app, permission_type), "Legacy %s urls" % permission_type
) )
LEGACY_PHP_VERSION_REPLACEMENTS = [
("/etc/php5", "/etc/php/7.4"),
("/etc/php/7.0", "/etc/php/7.4"),
("/etc/php/7.3", "/etc/php/7.4"),
("/var/run/php5-fpm", "/var/run/php/php7.4-fpm"),
("/var/run/php/php7.0-fpm", "/var/run/php/php7.4-fpm"),
("/var/run/php/php7.3-fpm", "/var/run/php/php7.4-fpm"),
("php5", "php7.4"),
("php7.0", "php7.4"),
("php7.3", "php7.4"),
('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'),
(
'phpversion="${phpversion:-7.0}"',
'phpversion="${phpversion:-7.4}"',
), # Many helpers like the composer ones use 7.0 by default ...
(
'phpversion="${phpversion:-7.3}"',
'phpversion="${phpversion:-7.4}"',
), # Many helpers like the composer ones use 7.0 by default ...
(
'"$phpversion" == "7.0"',
'$(bc <<< "$phpversion >= 7.4") -eq 1',
), # patch ynh_install_php to refuse installing/removing php <= 7.3
(
'"$phpversion" == "7.3"',
'$(bc <<< "$phpversion >= 7.4") -eq 1',
), # patch ynh_install_php to refuse installing/removing php <= 7.3
]
def _patch_legacy_php_versions(app_folder):
files_to_patch = []
files_to_patch.extend(glob.glob("%s/conf/*" % app_folder))
files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder))
files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
files_to_patch.append("%s/manifest.json" % app_folder)
files_to_patch.append("%s/manifest.toml" % app_folder)
for filename in files_to_patch:
# Ignore non-regular files
if not os.path.isfile(filename):
continue
c = (
"sed -i "
+ "".join(
"-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r)
for p, r in LEGACY_PHP_VERSION_REPLACEMENTS
)
+ "%s" % filename
)
os.system(c)
def _patch_legacy_php_versions_in_settings(app_folder):
settings = read_yaml(os.path.join(app_folder, "settings.yml"))
if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]:
settings["fpm_config_dir"] = "/etc/php/7.4/fpm"
if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]:
settings["fpm_service"] = "php7.4-fpm"
if settings.get("phpversion") in ["7.0", "7.3"]:
settings["phpversion"] = "7.4"
# We delete these checksums otherwise the file will appear as manually modified
list_to_remove = ["checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"]
settings = {
k: v
for k, v in settings.items()
if not any(k.startswith(to_remove) for to_remove in list_to_remove)
}
write_to_yaml(app_folder + "/settings.yml", settings)
def _patch_legacy_helpers(app_folder):
files_to_patch = []
files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
stuff_to_replace = {
"yunohost app initdb": {"important": True},
"yunohost app checkport": {"important": True},
"yunohost tools port-available": {"important": True},
"yunohost app checkurl": {"important": True},
# Remove
# Automatic diagnosis data from YunoHost
# __PRE_TAG1__$(yunohost tools diagnosis | ...)__PRE_TAG2__"
#
"yunohost tools diagnosis": {
"pattern": r"(Automatic diagnosis data from YunoHost( *\n)*)? *(__\w+__)? *\$\(yunohost tools diagnosis.*\)(__\w+__)?",
"replace": r"",
"important": False,
},
# Old $1, $2 in backup/restore scripts...
"app=$2": {"only_for": ["scripts/backup", "scripts/restore"], "important": True},
# Old $1, $2 in backup/restore scripts...
"backup_dir=$1": {"only_for": ["scripts/backup", "scripts/restore"], "important": True},
# Old $1, $2 in backup/restore scripts...
"restore_dir=$1": {"only_for": ["scripts/restore"], "important": True},
# Old $1, $2 in install scripts...
# We ain't patching that shit because it ain't trivial to patch all args...
"domain=$1": {"only_for": ["scripts/install"], "important": True},
}
for helper, infos in stuff_to_replace.items():
infos["pattern"] = (
re.compile(infos["pattern"]) if infos.get("pattern") else None
)
infos["replace"] = infos.get("replace")
for filename in files_to_patch:
# Ignore non-regular files
if not os.path.isfile(filename):
continue
try:
content = read_file(filename)
except MoulinetteError:
continue
replaced_stuff = False
show_warning = False
for helper, infos in stuff_to_replace.items():
# Ignore if not relevant for this file
if infos.get("only_for") and not any(
filename.endswith(f) for f in infos["only_for"]
):
continue
# If helper is used, attempt to patch the file
if helper in content and infos["pattern"]:
content = infos["pattern"].sub(infos["replace"], content)
replaced_stuff = True
if infos["important"]:
show_warning = True
# If the helper is *still* in the content, it means that we
# couldn't patch the deprecated helper in the previous lines. In
# that case, abort the install or whichever step is performed
if helper in content and infos["important"]:
raise YunohostValidationError(
"This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.",
raw_msg=True,
)
if replaced_stuff:
# Check the app do load the helper
# If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there...
if filename.split("/")[-1] in [
"install",
"remove",
"upgrade",
"backup",
"restore",
]:
source_helpers = "source /usr/share/yunohost/helpers"
if source_helpers not in content:
content.replace("#!/bin/bash", "#!/bin/bash\n" + source_helpers)
if source_helpers not in content:
content = source_helpers + "\n" + content
# Actually write the new content in the file
write_to_file(filename, content)
if show_warning:
# And complain about those damn deprecated helpers
logger.error(
r"/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ..."
)