[^";]+))'
- '(?=.*(;[\s]*|")p=(?P[^";]+))'), dkim_content, re.M|re.S
- )
- if dkim:
- result += '\n{host}. {ttl} IN TXT "v={v}; k={k}; p={p}"'.format(
- host='{0}.{1}'.format(dkim.group('host'), domain), ttl=ttl,
- v=dkim.group('v'), k=dkim.group('k'), p=dkim.group('p')
- )
+ result += "\n\n"
+ result += "; Mail"
+ for record in dns_conf["mail"]:
+ result += "\n{name} {ttl} IN {type} {value}".format(**record)
- # If DKIM is set, add dummy DMARC support
- result += '\n_dmarc {ttl} IN TXT "v=DMARC1; p=none"'.format(
- ttl=ttl
- )
+ is_cli = True if msettings.get('interface') == 'cli' else False
+ if is_cli:
+ logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation"))
return result
-def get_public_ip(protocol=4):
- """Retrieve the public IP address from ip.yunohost.org"""
- if protocol == 4:
- url = 'https://ip.yunohost.org'
- elif protocol == 6:
- # FIXME: Let's Encrypt does not support IPv6-only hosts yet
- url = 'http://ip6.yunohost.org'
+def domain_cert_status(auth, domain_list, full=False):
+ return yunohost.certificate.certificate_status(auth, domain_list, full)
+
+
+def domain_cert_install(auth, domain_list, force=False, no_checks=False, self_signed=False, staging=False):
+ return yunohost.certificate.certificate_install(auth, domain_list, force, no_checks, self_signed, staging)
+
+
+def domain_cert_renew(auth, domain_list, force=False, no_checks=False, email=False, staging=False):
+ return yunohost.certificate.certificate_renew(auth, domain_list, force, no_checks, email, staging)
+
+
+def _get_conflicting_apps(auth, domain, path):
+ """
+ Return a list of all conflicting apps with a domain/path (it can be empty)
+
+ Keyword argument:
+ domain -- The domain for the web path (e.g. your.domain.tld)
+ path -- The path to check (e.g. /coffee)
+ """
+
+ domain, path = _normalize_domain_path(domain, path)
+
+ # Abort if domain is unknown
+ if domain not in domain_list(auth)['domains']:
+ raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
+
+ # This import cannot be put on top of file because it would create a
+ # recursive import...
+ from yunohost.app import app_map
+
+ # Fetch apps map
+ apps_map = app_map(raw=True)
+
+ # Loop through all apps to check if path is taken by one of them
+ conflicts = []
+ if domain in apps_map:
+ # Loop through apps
+ for p, a in apps_map[domain].items():
+ if path == p:
+ conflicts.append((p, a["id"], a["label"]))
+ # We also don't want conflicts with other apps starting with
+ # same name
+ elif path.startswith(p) or p.startswith(path):
+ conflicts.append((p, a["id"], a["label"]))
+
+ return conflicts
+
+
+def domain_url_available(auth, domain, path):
+ """
+ Check availability of a web path
+
+ Keyword argument:
+ domain -- The domain for the web path (e.g. your.domain.tld)
+ path -- The path to check (e.g. /coffee)
+ """
+
+ return len(_get_conflicting_apps(auth, domain, path)) == 0
+
+
+def _get_maindomain():
+ with open('/etc/yunohost/current_host', 'r') as f:
+ maindomain = f.readline().rstrip()
+ return maindomain
+
+
+def _set_maindomain(domain):
+ with open('/etc/yunohost/current_host', 'w') as f:
+ f.write(domain)
+
+
+def _normalize_domain_path(domain, path):
+
+ # We want url to be of the format :
+ # some.domain.tld/foo
+
+ # Remove http/https prefix if it's there
+ if domain.startswith("https://"):
+ domain = domain[len("https://"):]
+ elif domain.startswith("http://"):
+ domain = domain[len("http://"):]
+
+ # Remove trailing slashes
+ domain = domain.rstrip("/")
+ path = "/" + path.strip("/")
+
+ return domain, path
+
+
+def _build_dns_conf(domain, ttl=3600):
+ """
+ Internal function that will returns a data structure containing the needed
+ information to generate/adapt the dns configuration
+
+ The returned datastructure will have the following form:
+ {
+ "basic": [
+ # if ipv4 available
+ {"type": "A", "name": "@", "value": "123.123.123.123", "ttl": 3600},
+ {"type": "A", "name": "*", "value": "123.123.123.123", "ttl": 3600},
+ # if ipv6 available
+ {"type": "AAAA", "name": "@", "value": "valid-ipv6", "ttl": 3600},
+ {"type": "AAAA", "name": "*", "value": "valid-ipv6", "ttl": 3600},
+ ],
+ "xmpp": [
+ {"type": "SRV", "name": "_xmpp-client._tcp", "value": "0 5 5222 domain.tld.", "ttl": 3600},
+ {"type": "SRV", "name": "_xmpp-server._tcp", "value": "0 5 5269 domain.tld.", "ttl": 3600},
+ {"type": "CNAME", "name": "muc", "value": "@", "ttl": 3600},
+ {"type": "CNAME", "name": "pubsub", "value": "@", "ttl": 3600},
+ {"type": "CNAME", "name": "vjud", "value": "@", "ttl": 3600}
+ ],
+ "mail": [
+ {"type": "MX", "name": "@", "value": "10 domain.tld.", "ttl": 3600},
+ {"type": "TXT", "name": "@", "value": "\"v=spf1 a mx ip4:123.123.123.123 ipv6:valid-ipv6 -all\"", "ttl": 3600 },
+ {"type": "TXT", "name": "mail._domainkey", "value": "\"v=DKIM1; k=rsa; p=some-super-long-key\"", "ttl": 3600},
+ {"type": "TXT", "name": "_dmarc", "value": "\"v=DMARC1; p=none\"", "ttl": 3600}
+ ],
+ }
+ """
+
+ ipv4 = get_public_ip()
+ ipv6 = get_public_ip(6)
+
+ basic = []
+
+ # Basic ipv4/ipv6 records
+ if ipv4:
+ basic += [
+ ["@", ttl, "A", ipv4],
+ ["*", ttl, "A", ipv4],
+ ]
+
+ if ipv6:
+ basic += [
+ ["@", ttl, "AAAA", ipv6],
+ ["*", ttl, "AAAA", ipv6],
+ ]
+
+ # XMPP
+ xmpp = [
+ ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain],
+ ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain],
+ ["muc", ttl, "CNAME", "@"],
+ ["pubsub", ttl, "CNAME", "@"],
+ ["vjud", ttl, "CNAME", "@"],
+ ]
+
+ # SPF record
+ spf_record = '"v=spf1 a mx'
+ if ipv4:
+ spf_record += ' ip4:{ip4}'.format(ip4=ipv4)
+ if ipv6:
+ spf_record += ' ip6:{ip6}'.format(ip6=ipv6)
+ spf_record += ' -all"'
+
+ # Email
+ mail = [
+ ["@", ttl, "MX", "10 %s." % domain],
+ ["@", ttl, "TXT", spf_record],
+ ]
+
+ # DKIM/DMARC record
+ dkim_host, dkim_publickey = _get_DKIM(domain)
+
+ if dkim_host:
+ mail += [
+ [dkim_host, ttl, "TXT", dkim_publickey],
+ ["_dmarc", ttl, "TXT", '"v=DMARC1; p=none"'],
+ ]
+
+ return {
+ "basic": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in basic],
+ "xmpp": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in xmpp],
+ "mail": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in mail],
+ }
+
+
+def _get_DKIM(domain):
+ DKIM_file = '/etc/dkim/{domain}.mail.txt'.format(domain=domain)
+
+ if not os.path.isfile(DKIM_file):
+ return (None, None)
+
+ with open(DKIM_file) as f:
+ dkim_content = f.read()
+
+ # Gotta manage two formats :
+ #
+ # Legacy
+ # -----
+ #
+ # mail._domainkey IN TXT ( "v=DKIM1; k=rsa; "
+ # "p=" )
+ #
+ # New
+ # ------
+ #
+ # mail._domainkey IN TXT ( "v=DKIM1; h=sha256; k=rsa; "
+ # "p=" )
+
+ is_legacy_format = " h=sha256; " not in dkim_content
+
+ # Legacy DKIM format
+ if is_legacy_format:
+ dkim = re.match((
+ r'^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+'
+ '[^"]*"v=(?P[^";]+);'
+ '[\s"]*k=(?P[^";]+);'
+ '[\s"]*p=(?P[^";]+)'), dkim_content, re.M | re.S
+ )
else:
- raise ValueError("invalid protocol version")
- try:
- return urlopen(url).read().strip()
- except IOError:
- logger.debug('cannot retrieve public IPv%d' % protocol, exc_info=1)
- raise MoulinetteError(errno.ENETUNREACH,
- m18n.n('no_internet_connection'))
+ dkim = re.match((
+ r'^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+'
+ '[^"]*"v=(?P[^";]+);'
+ '[\s"]*h=(?P[^";]+);'
+ '[\s"]*k=(?P[^";]+);'
+ '[\s"]*p=(?P[^";]+)'), dkim_content, re.M | re.S
+ )
+
+ if not dkim:
+ return (None, None)
+
+ if is_legacy_format:
+ return (
+ dkim.group('host'),
+ '"v={v}; k={k}; p={p}"'.format(v=dkim.group('v'),
+ k=dkim.group('k'),
+ p=dkim.group('p'))
+ )
+ else:
+ return (
+ dkim.group('host'),
+ '"v={v}; h={h}; k={k}; p={p}"'.format(v=dkim.group('v'),
+ h=dkim.group('h'),
+ k=dkim.group('k'),
+ p=dkim.group('p'))
+ )
diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py
index 878bc577e..88547b4db 100644
--- a/src/yunohost/dyndns.py
+++ b/src/yunohost/dyndns.py
@@ -27,47 +27,94 @@ import os
import re
import json
import glob
+import time
import base64
import errno
-import requests
import subprocess
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file, write_to_file, rm
+from moulinette.utils.network import download_json
-from yunohost.domain import get_public_ip
+from yunohost.domain import _get_maindomain, _build_dns_conf
+from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.dyndns')
+OLD_IPV4_FILE = '/etc/yunohost/dyndns/old_ip'
+OLD_IPV6_FILE = '/etc/yunohost/dyndns/old_ipv6'
+DYNDNS_ZONE = '/etc/yunohost/dyndns/zone'
-class IPRouteLine(object):
- """ Utility class to parse an ip route output line
-
- The output of ip ro is variable and hard to parse completly, it would
- require a real parser, not just a regexp, so do minimal parsing here...
-
- >>> a = IPRouteLine('2001:: from :: via fe80::c23f:fe:1e:cafe dev eth0 src 2000:de:beef:ca:0:fe:1e:cafe metric 0')
- >>> a.src_addr
- "2000:de:beef:ca:0:fe:1e:cafe"
- """
- regexp = re.compile(
- r'(?Punreachable)?.*src\s+(?P[0-9a-f:]+).*')
-
- def __init__(self, line):
- self.m = self.regexp.match(line)
- if not self.m:
- raise ValueError("Not a valid ip route get line")
-
- # make regexp group available as object attributes
- for k, v in self.m.groupdict().items():
- setattr(self, k, v)
-
-re_dyndns_private_key = re.compile(
+RE_DYNDNS_PRIVATE_KEY_MD5 = re.compile(
r'.*/K(?P[^\s\+]+)\.\+157.+\.private$'
)
+RE_DYNDNS_PRIVATE_KEY_SHA512 = re.compile(
+ r'.*/K(?P[^\s\+]+)\.\+165.+\.private$'
+)
-def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None):
+
+def _dyndns_provides(provider, domain):
+ """
+ Checks if a provider provide/manage a given domain.
+
+ Keyword arguments:
+ provider -- The url of the provider, e.g. "dyndns.yunohost.org"
+ domain -- The full domain that you'd like.. e.g. "foo.nohost.me"
+
+ Returns:
+ True if the provider provide/manages the domain. False otherwise.
+ """
+
+ logger.debug("Checking if %s is managed by %s ..." % (domain, provider))
+
+ try:
+ # Dyndomains will be a list of domains supported by the provider
+ # e.g. [ "nohost.me", "noho.st" ]
+ dyndomains = download_json('https://%s/domains' % provider, timeout=30)
+ except MoulinetteError as e:
+ logger.error(str(e))
+ raise MoulinetteError(errno.EIO,
+ m18n.n('dyndns_could_not_check_provide',
+ domain=domain, provider=provider))
+
+ # Extract 'dyndomain' from 'domain', e.g. 'nohost.me' from 'foo.nohost.me'
+ dyndomain = '.'.join(domain.split('.')[1:])
+
+ return dyndomain in dyndomains
+
+
+def _dyndns_available(provider, domain):
+ """
+ Checks if a domain is available from a given provider.
+
+ Keyword arguments:
+ provider -- The url of the provider, e.g. "dyndns.yunohost.org"
+ domain -- The full domain that you'd like.. e.g. "foo.nohost.me"
+
+ Returns:
+ True if the domain is avaible, False otherwise.
+ """
+ logger.debug("Checking if domain %s is available on %s ..."
+ % (domain, provider))
+
+ try:
+ r = download_json('https://%s/test/%s' % (provider, domain),
+ expected_status_code=None)
+ except MoulinetteError as e:
+ logger.error(str(e))
+ raise MoulinetteError(errno.EIO,
+ m18n.n('dyndns_could_not_check_available',
+ domain=domain, provider=provider))
+
+ return r == u"Domain %s is available" % domain
+
+
+@is_unit_operation()
+def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", domain=None, key=None):
"""
Subscribe to a DynDNS service
@@ -78,38 +125,48 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
"""
if domain is None:
- with open('/etc/yunohost/current_host', 'r') as f:
- domain = f.readline().rstrip()
+ domain = _get_maindomain()
+ operation_logger.related_to.append(('domain', domain))
+
+ # Verify if domain is provided by subscribe_host
+ if not _dyndns_provides(subscribe_host, domain):
+ raise MoulinetteError(errno.ENOENT,
+ m18n.n('dyndns_domain_not_provided',
+ domain=domain, provider=subscribe_host))
# Verify if domain is available
- try:
- if requests.get('https://%s/test/%s' % (subscribe_host, domain)).status_code != 200:
- raise MoulinetteError(errno.EEXIST, m18n.n('dyndns_unavailable'))
- except requests.ConnectionError:
- raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection'))
+ if not _dyndns_available(subscribe_host, domain):
+ raise MoulinetteError(errno.ENOENT,
+ m18n.n('dyndns_unavailable', domain=domain))
+
+ operation_logger.start()
if key is None:
if len(glob.glob('/etc/yunohost/dyndns/*.key')) == 0:
- os.makedirs('/etc/yunohost/dyndns')
+ if not os.path.exists('/etc/yunohost/dyndns'):
+ os.makedirs('/etc/yunohost/dyndns')
- logger.info(m18n.n('dyndns_key_generating'))
+ logger.debug(m18n.n('dyndns_key_generating'))
- os.system('cd /etc/yunohost/dyndns && ' \
- 'dnssec-keygen -a hmac-md5 -b 128 -n USER %s' % domain)
+ os.system('cd /etc/yunohost/dyndns && '
+ 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain)
os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private')
key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0]
with open(key_file) as f:
- key = f.readline().strip().split(' ')[-1]
+ key = f.readline().strip().split(' ', 6)[-1]
+ import requests # lazy loading this module for performance reasons
# Send subscription
try:
- r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={ 'subdomain': domain })
+ r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30)
except requests.ConnectionError:
raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection'))
if r.status_code != 201:
- try: error = json.loads(r.text)['error']
- except: error = "Server error"
+ try:
+ error = json.loads(r.text)['error']
+ except:
+ error = "Server error, code: %s. (Message: \"%s\")" % (r.status_code, r.text)
raise MoulinetteError(errno.EPERM,
m18n.n('dyndns_registration_failed', error=error))
@@ -118,7 +175,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
dyndns_installcron()
-def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
+@is_unit_operation()
+def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, key=None,
ipv4=None, ipv6=None):
"""
Update IP on DynDNS platform
@@ -131,127 +189,132 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
ipv6 -- IPv6 address to send
"""
- # IPv4
+ # Get old ipv4/v6
+
+ old_ipv4, old_ipv6 = (None, None) # (default values)
+
+ if os.path.isfile(OLD_IPV4_FILE):
+ old_ipv4 = read_file(OLD_IPV4_FILE).rstrip()
+
+ if os.path.isfile(OLD_IPV6_FILE):
+ old_ipv6 = read_file(OLD_IPV6_FILE).rstrip()
+
+ # Get current IPv4 and IPv6
+ ipv4_ = get_public_ip()
+ ipv6_ = get_public_ip(6)
+
if ipv4 is None:
- ipv4 = get_public_ip()
+ ipv4 = ipv4_
- try:
- with open('/etc/yunohost/dyndns/old_ip', 'r') as f:
- old_ip = f.readline().rstrip()
- except IOError:
- old_ip = '0.0.0.0'
-
- # IPv6
if ipv6 is None:
+ ipv6 = ipv6_
+
+ logger.debug("Old IPv4/v6 are (%s, %s)" % (old_ipv4, old_ipv6))
+ logger.debug("Requested IPv4/v6 are (%s, %s)" % (ipv4, ipv6))
+
+ # no need to update
+ if old_ipv4 == ipv4 and old_ipv6 == ipv6:
+ logger.info("No updated needed.")
+ return
+ else:
+ logger.info("Updated needed, going on...")
+
+ # If domain is not given, try to guess it from keys available...
+ if domain is None:
+ (domain, key) = _guess_current_dyndns_domain(dyn_host)
+ # If key is not given, pick the first file we find with the domain given
+ else:
+ if key is None:
+ keys = glob.glob('/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
+
+ if not keys:
+ raise MoulinetteError(errno.EIO, m18n.n('dyndns_key_not_found'))
+
+ key = keys[0]
+
+ operation_logger.related_to.append(('domain', domain))
+ operation_logger.start()
+
+ # This mean that hmac-md5 is used
+ # (Re?)Trigger the migration to sha256 and return immediately.
+ # The actual update will be done in next run.
+ if "+157" in key:
+ from yunohost.tools import _get_migration_by_name
+ migration = _get_migration_by_name("migrate_to_tsig_sha256")
try:
- ip_route_out = subprocess.check_output(
- ['ip', 'route', 'get', '2000::']).split('\n')
+ migration.migrate(dyn_host, domain, key)
+ except Exception as e:
+ logger.error(m18n.n('migrations_migration_has_failed',
+ exception=e,
+ number=migration.number,
+ name=migration.name),
+ exc_info=1)
+ return
- if len(ip_route_out) > 0:
- route = IPRouteLine(ip_route_out[0])
- if not route.unreachable:
- ipv6 = route.src_addr
+ # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me'
+ host = domain.split('.')[1:]
+ host = '.'.join(host)
- except (OSError, ValueError) as e:
- # Unlikely case "ip route" does not return status 0
- # or produces unexpected output
- raise MoulinetteError(errno.EBADMSG,
- "ip route cmd error : {}".format(e))
+ logger.debug("Building zone update file ...")
- if ipv6 is None:
- logger.info(m18n.n('no_ipv6_connectivity'))
+ lines = [
+ 'server %s' % dyn_host,
+ 'zone %s' % host,
+ ]
+
+ dns_conf = _build_dns_conf(domain)
+
+ # Delete the old records for all domain/subdomains
+
+ # every dns_conf.values() is a list of :
+ # [{"name": "...", "ttl": "...", "type": "...", "value": "..."}]
+ for records in dns_conf.values():
+ for record in records:
+ action = "update delete {name}.{domain}.".format(domain=domain, **record)
+ action = action.replace(" @.", " ")
+ lines.append(action)
+
+ # Add the new records for all domain/subdomains
+
+ for records in dns_conf.values():
+ for record in records:
+ # (For some reason) here we want the format with everytime the
+ # entire, full domain shown explicitly, not just "muc" or "@", it
+ # should be muc.the.domain.tld. or the.domain.tld
+ if record["value"] == "@":
+ record["value"] = domain
+ record["value"] = record["value"].replace(";","\;")
+
+ action = "update add {name}.{domain}. {ttl} {type} {value}".format(domain=domain, **record)
+ action = action.replace(" @.", " ")
+ lines.append(action)
+
+ lines += [
+ 'show',
+ 'send'
+ ]
+
+ # Write the actions to do to update to a file, to be able to pass it
+ # to nsupdate as argument
+ write_to_file(DYNDNS_ZONE, '\n'.join(lines))
+
+ logger.debug("Now pushing new conf to DynDNS host...")
try:
- with open('/etc/yunohost/dyndns/old_ipv6', 'r') as f:
- old_ipv6 = f.readline().rstrip()
- except IOError:
- old_ipv6 = '0000:0000:0000:0000:0000:0000:0000:0000'
+ command = ["/usr/bin/nsupdate", "-k", key, DYNDNS_ZONE]
+ subprocess.check_call(command)
+ except subprocess.CalledProcessError:
+ rm(OLD_IPV4_FILE, force=True) # Remove file (ignore if non-existent)
+ rm(OLD_IPV6_FILE, force=True) # Remove file (ignore if non-existent)
+ raise MoulinetteError(errno.EPERM,
+ m18n.n('dyndns_ip_update_failed'))
- if old_ip != ipv4 or old_ipv6 != ipv6:
- if domain is None:
- # Retrieve the first registered domain
- for path in glob.iglob('/etc/yunohost/dyndns/K*.private'):
- match = re_dyndns_private_key.match(path)
- if not match:
- continue
- _domain = match.group('domain')
- try:
- # Check if domain is registered
- if requests.get('https://{0}/test/{1}'.format(
- dyn_host, _domain)).status_code == 200:
- continue
- except requests.ConnectionError:
- raise MoulinetteError(errno.ENETUNREACH,
- m18n.n('no_internet_connection'))
- domain = _domain
- key = path
- break
- if not domain:
- raise MoulinetteError(errno.EINVAL,
- m18n.n('dyndns_no_domain_registered'))
+ logger.success(m18n.n('dyndns_ip_updated'))
- if key is None:
- keys = glob.glob(
- '/etc/yunohost/dyndns/K{0}.+*.private'.format(domain))
- if len(keys) > 0:
- key = keys[0]
- if not key:
- raise MoulinetteError(errno.EIO,
- m18n.n('dyndns_key_not_found'))
-
- host = domain.split('.')[1:]
- host = '.'.join(host)
- lines = [
- 'server %s' % dyn_host,
- 'zone %s' % host,
- 'update delete %s. A' % domain,
- 'update delete %s. AAAA' % domain,
- 'update delete %s. MX' % domain,
- 'update delete %s. TXT' % domain,
- 'update delete pubsub.%s. A' % domain,
- 'update delete pubsub.%s. AAAA' % domain,
- 'update delete muc.%s. A' % domain,
- 'update delete muc.%s. AAAA' % domain,
- 'update delete vjud.%s. A' % domain,
- 'update delete vjud.%s. AAAA' % domain,
- 'update delete _xmpp-client._tcp.%s. SRV' % domain,
- 'update delete _xmpp-server._tcp.%s. SRV' % domain,
- 'update add %s. 1800 A %s' % (domain, ipv4),
- 'update add %s. 14400 MX 5 %s.' % (domain, domain),
- 'update add %s. 14400 TXT "v=spf1 a mx -all"' % domain,
- 'update add pubsub.%s. 1800 A %s' % (domain, ipv4),
- 'update add muc.%s. 1800 A %s' % (domain, ipv4),
- 'update add vjud.%s. 1800 A %s' % (domain, ipv4),
- 'update add _xmpp-client._tcp.%s. 14400 SRV 0 5 5222 %s.' % (domain, domain),
- 'update add _xmpp-server._tcp.%s. 14400 SRV 0 5 5269 %s.' % (domain, domain)
- ]
- if ipv6 is not None:
- lines += [
- 'update add %s. 1800 AAAA %s' % (domain, ipv6),
- 'update add pubsub.%s. 1800 AAAA %s' % (domain, ipv6),
- 'update add muc.%s. 1800 AAAA %s' % (domain, ipv6),
- 'update add vjud.%s. 1800 AAAA %s' % (domain, ipv6),
- ]
- lines += [
- 'show',
- 'send'
- ]
- with open('/etc/yunohost/dyndns/zone', 'w') as zone:
- for line in lines:
- zone.write(line + '\n')
-
- if os.system('/usr/bin/nsupdate -k %s /etc/yunohost/dyndns/zone' % key) == 0:
- logger.success(m18n.n('dyndns_ip_updated'))
- with open('/etc/yunohost/dyndns/old_ip', 'w') as f:
- f.write(ipv4)
- if ipv6 is not None:
- with open('/etc/yunohost/dyndns/old_ipv6', 'w') as f:
- f.write(ipv6)
- else:
- os.system('rm -f /etc/yunohost/dyndns/old_ip')
- os.system('rm -f /etc/yunohost/dyndns/old_ipv6')
- raise MoulinetteError(errno.EPERM,
- m18n.n('dyndns_ip_update_failed'))
+ if ipv4 is not None:
+ write_to_file(OLD_IPV4_FILE, ipv4)
+ if ipv6 is not None:
+ write_to_file(OLD_IPV6_FILE, ipv6)
def dyndns_installcron():
@@ -278,3 +341,32 @@ def dyndns_removecron():
raise MoulinetteError(errno.EIO, m18n.n('dyndns_cron_remove_failed'))
logger.success(m18n.n('dyndns_cron_removed'))
+
+
+def _guess_current_dyndns_domain(dyn_host):
+ """
+ This function tries to guess which domain should be updated by
+ "dyndns_update()" because there's not proper management of the current
+ dyndns domain :/ (and at the moment the code doesn't support having several
+ dyndns domain, which is sort of a feature so that people don't abuse the
+ dynette...)
+ """
+
+ # Retrieve the first registered domain
+ for path in glob.iglob('/etc/yunohost/dyndns/K*.private'):
+ match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path)
+ if not match:
+ match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path)
+ if not match:
+ continue
+ _domain = match.group('domain')
+
+ # Verify if domain is registered (i.e., if it's available, skip
+ # current domain beause that's not the one we want to update..)
+ if _dyndns_available(dyn_host, _domain):
+ continue
+ else:
+ return (_domain, path)
+
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('dyndns_no_domain_registered'))
diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py
index 1291cf86a..7b1c72170 100644
--- a/src/yunohost/firewall.py
+++ b/src/yunohost/firewall.py
@@ -33,13 +33,14 @@ except ImportError:
sys.stderr.write('Error: Yunohost CLI Require miniupnpc lib\n')
sys.exit(1)
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils import process
from moulinette.utils.log import getActionLogger
from moulinette.utils.text import prependlines
-firewall_file = '/etc/yunohost/firewall.yml'
-upnp_cron_job = '/etc/cron.d/yunohost-firewall-upnp'
+FIREWALL_FILE = '/etc/yunohost/firewall.yml'
+UPNP_CRON_JOB = '/etc/cron.d/yunohost-firewall-upnp'
logger = getActionLogger('yunohost.firewall')
@@ -67,14 +68,14 @@ def firewall_allow(protocol, port, ipv4_only=False, ipv6_only=False,
# Validate protocols
protocols = ['TCP', 'UDP']
if protocol != 'Both' and protocol in protocols:
- protocols = [protocol,]
+ protocols = [protocol, ]
# Validate IP versions
ipvs = ['ipv4', 'ipv6']
if ipv4_only and not ipv6_only:
- ipvs = ['ipv4',]
+ ipvs = ['ipv4', ]
elif ipv6_only and not ipv4_only:
- ipvs = ['ipv6',]
+ ipvs = ['ipv6', ]
for p in protocols:
# Iterate over IP versions to add port
@@ -117,18 +118,18 @@ def firewall_disallow(protocol, port, ipv4_only=False, ipv6_only=False,
# Validate protocols
protocols = ['TCP', 'UDP']
if protocol != 'Both' and protocol in protocols:
- protocols = [protocol,]
+ protocols = [protocol, ]
# Validate IP versions and UPnP
ipvs = ['ipv4', 'ipv6']
upnp = True
if ipv4_only and ipv6_only:
- upnp = True # automatically disallow UPnP
+ upnp = True # automatically disallow UPnP
elif ipv4_only:
- ipvs = ['ipv4',]
+ ipvs = ['ipv4', ]
upnp = upnp_only
elif ipv6_only:
- ipvs = ['ipv6',]
+ ipvs = ['ipv6', ]
upnp = upnp_only
elif upnp_only:
ipvs = []
@@ -161,7 +162,7 @@ def firewall_list(raw=False, by_ip_version=False, list_forwarded=False):
list_forwarded -- List forwarded ports with UPnP
"""
- with open(firewall_file) as f:
+ with open(FIREWALL_FILE) as f:
firewall = yaml.load(f)
if raw:
return firewall
@@ -178,7 +179,7 @@ def firewall_list(raw=False, by_ip_version=False, list_forwarded=False):
ports = sorted(set(ports['ipv4']) | set(ports['ipv6']))
# Format returned dict
- ret = { "opened_ports": ports }
+ ret = {"opened_ports": ports}
if list_forwarded:
# Combine TCP and UDP forwarded ports
ret['forwarded_ports'] = sorted(
@@ -224,8 +225,8 @@ def firewall_reload(skip_upnp=False):
# Iterate over ports and add rule
for protocol in ['TCP', 'UDP']:
for port in firewall['ipv4'][protocol]:
- rules.append("iptables -w -A INPUT -p %s --dport %s -j ACCEPT" \
- % (protocol, process.quote(str(port))))
+ rules.append("iptables -w -A INPUT -p %s --dport %s -j ACCEPT"
+ % (protocol, process.quote(str(port))))
rules += [
"iptables -w -A INPUT -i lo -j ACCEPT",
"iptables -w -A INPUT -p icmp -j ACCEPT",
@@ -233,7 +234,7 @@ def firewall_reload(skip_upnp=False):
]
# Execute each rule
- if process.check_commands(rules, callback=_on_rule_command_error):
+ if process.run_commands(rules, callback=_on_rule_command_error):
errors = True
reloaded = True
@@ -253,8 +254,8 @@ def firewall_reload(skip_upnp=False):
# Iterate over ports and add rule
for protocol in ['TCP', 'UDP']:
for port in firewall['ipv6'][protocol]:
- rules.append("ip6tables -w -A INPUT -p %s --dport %s -j ACCEPT" \
- % (protocol, process.quote(str(port))))
+ rules.append("ip6tables -w -A INPUT -p %s --dport %s -j ACCEPT"
+ % (protocol, process.quote(str(port))))
rules += [
"ip6tables -w -A INPUT -i lo -j ACCEPT",
"ip6tables -w -A INPUT -p icmpv6 -j ACCEPT",
@@ -262,7 +263,7 @@ def firewall_reload(skip_upnp=False):
]
# Execute each rule
- if process.check_commands(rules, callback=_on_rule_command_error):
+ if process.run_commands(rules, callback=_on_rule_command_error):
errors = True
reloaded = True
@@ -304,20 +305,21 @@ def firewall_upnp(action='status', no_refresh=False):
# Compatibility with previous version
if action == 'reload':
- logger.info("'reload' action is deprecated and will be removed")
+ logger.debug("'reload' action is deprecated and will be removed")
try:
# Remove old cron job
os.remove('/etc/cron.d/yunohost-firewall')
- except: pass
+ except:
+ pass
action = 'status'
no_refresh = False
if action == 'status' and no_refresh:
# Only return current state
- return { 'enabled': enabled }
+ return {'enabled': enabled}
elif action == 'enable' or (enabled and action == 'status'):
# Add cron job
- with open(upnp_cron_job, 'w+') as f:
+ with open(UPNP_CRON_JOB, 'w+') as f:
f.write('*/50 * * * * root '
'/usr/bin/yunohost firewall upnp status >>/dev/null\n')
# Open port 1900 to receive discovery message
@@ -329,8 +331,9 @@ def firewall_upnp(action='status', no_refresh=False):
elif action == 'disable' or (not enabled and action == 'status'):
try:
# Remove cron job
- os.remove(upnp_cron_job)
- except: pass
+ os.remove(UPNP_CRON_JOB)
+ except:
+ pass
enabled = False
if action == 'status':
no_refresh = True
@@ -354,7 +357,7 @@ def firewall_upnp(action='status', no_refresh=False):
# Select UPnP device
upnpc.selectigd()
except:
- logger.info('unable to select UPnP device', exc_info=1)
+ logger.debug('unable to select UPnP device', exc_info=1)
enabled = False
else:
# Iterate over ports
@@ -364,7 +367,8 @@ def firewall_upnp(action='status', no_refresh=False):
if upnpc.getspecificportmapping(port, protocol):
try:
upnpc.deleteportmapping(port, protocol)
- except: pass
+ except:
+ pass
if not enabled:
continue
try:
@@ -372,7 +376,7 @@ def firewall_upnp(action='status', no_refresh=False):
upnpc.addportmapping(port, protocol, upnpc.lanaddr,
port, 'yunohost firewall: port %d' % port, '')
except:
- logger.info('unable to add port %d using UPnP',
+ logger.debug('unable to add port %d using UPnP',
port, exc_info=1)
enabled = False
@@ -381,8 +385,8 @@ def firewall_upnp(action='status', no_refresh=False):
firewall['uPnP']['enabled'] = enabled
# Make a backup and update firewall file
- os.system("cp {0} {0}.old".format(firewall_file))
- with open(firewall_file, 'w') as f:
+ os.system("cp {0} {0}.old".format(FIREWALL_FILE))
+ with open(FIREWALL_FILE, 'w') as f:
yaml.safe_dump(firewall, f, default_flow_style=False)
if not no_refresh:
@@ -403,7 +407,7 @@ def firewall_upnp(action='status', no_refresh=False):
if action == 'enable' and not enabled:
raise MoulinetteError(errno.ENXIO, m18n.n('upnp_port_open_failed'))
- return { 'enabled': enabled }
+ return {'enabled': enabled}
def firewall_stop():
@@ -424,7 +428,7 @@ def firewall_stop():
os.system("ip6tables -F")
os.system("ip6tables -X")
- if os.path.exists(upnp_cron_job):
+ if os.path.exists(UPNP_CRON_JOB):
firewall_upnp('disable')
@@ -444,15 +448,17 @@ def _get_ssh_port(default=22):
pass
return default
+
def _update_firewall_file(rules):
"""Make a backup and write new rules to firewall file"""
- os.system("cp {0} {0}.old".format(firewall_file))
- with open(firewall_file, 'w') as f:
+ os.system("cp {0} {0}.old".format(FIREWALL_FILE))
+ with open(FIREWALL_FILE, 'w') as f:
yaml.safe_dump(rules, f, default_flow_style=False)
+
def _on_rule_command_error(returncode, cmd, output):
"""Callback for rules commands error"""
# Log error and continue commands execution
- logger.info('"%s" returned non-zero exit status %d:\n%s',
- cmd, returncode, prependlines(output.rstrip(), '> '))
+ logger.debug('"%s" returned non-zero exit status %d:\n%s',
+ cmd, returncode, prependlines(output.rstrip(), '> '))
return True
diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py
index 2d46cfcd5..87844ce17 100644
--- a/src/yunohost/hook.py
+++ b/src/yunohost/hook.py
@@ -24,18 +24,17 @@
Manage hooks
"""
import os
-import sys
import re
-import json
import errno
-import subprocess
+import tempfile
from glob import iglob
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils import log
-hook_folder = '/usr/share/yunohost/hooks/'
-custom_hook_folder = '/etc/yunohost/hooks.d/'
+HOOK_FOLDER = '/usr/share/yunohost/hooks/'
+CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/'
logger = log.getActionLogger('yunohost.hook')
@@ -52,14 +51,16 @@ def hook_add(app, file):
path, filename = os.path.split(file)
priority, action = _extract_filename_parts(filename)
- try: os.listdir(custom_hook_folder + action)
- except OSError: os.makedirs(custom_hook_folder + action)
+ try:
+ os.listdir(CUSTOM_HOOK_FOLDER + action)
+ except OSError:
+ os.makedirs(CUSTOM_HOOK_FOLDER + action)
- finalpath = custom_hook_folder + action +'/'+ priority +'-'+ app
+ finalpath = CUSTOM_HOOK_FOLDER + action + '/' + priority + '-' + app
os.system('cp %s %s' % (file, finalpath))
- os.system('chown -hR admin: %s' % hook_folder)
+ os.system('chown -hR admin: %s' % HOOK_FOLDER)
- return { 'hook': finalpath }
+ return {'hook': finalpath}
def hook_remove(app):
@@ -71,11 +72,12 @@ def hook_remove(app):
"""
try:
- for action in os.listdir(custom_hook_folder):
- for script in os.listdir(custom_hook_folder + action):
+ for action in os.listdir(CUSTOM_HOOK_FOLDER):
+ for script in os.listdir(CUSTOM_HOOK_FOLDER + action):
if script.endswith(app):
- os.remove(custom_hook_folder + action +'/'+ script)
- except OSError: pass
+ os.remove(CUSTOM_HOOK_FOLDER + action + '/' + script)
+ except OSError:
+ pass
def hook_info(action, name):
@@ -92,7 +94,7 @@ def hook_info(action, name):
# Search in custom folder first
for h in iglob('{:s}{:s}/*-{:s}'.format(
- custom_hook_folder, action, name)):
+ CUSTOM_HOOK_FOLDER, action, name)):
priority, _ = _extract_filename_parts(os.path.basename(h))
priorities.add(priority)
hooks.append({
@@ -101,7 +103,7 @@ def hook_info(action, name):
})
# Append non-overwritten system hooks
for h in iglob('{:s}{:s}/*-{:s}'.format(
- hook_folder, action, name)):
+ HOOK_FOLDER, action, name)):
priority, _ = _extract_filename_parts(os.path.basename(h))
if priority not in priorities:
hooks.append({
@@ -136,11 +138,11 @@ def hook_list(action, list_by='name', show_info=False):
def _append_hook(d, priority, name, path):
# Use the priority as key and a dict of hooks names
# with their info as value
- value = { 'path': path }
+ value = {'path': path}
try:
d[priority][name] = value
except KeyError:
- d[priority] = { name: value }
+ d[priority] = {name: value}
else:
def _append_hook(d, priority, name, path):
# Use the priority as key and the name as value
@@ -162,11 +164,12 @@ def hook_list(action, list_by='name', show_info=False):
if h['path'] != path:
h['path'] = path
return
- l.append({ 'priority': priority, 'path': path })
+ l.append({'priority': priority, 'path': path})
d[name] = l
else:
if list_by == 'name':
result = set()
+
def _append_hook(d, priority, name, path):
# Add only the name
d.add(name)
@@ -186,25 +189,25 @@ def hook_list(action, list_by='name', show_info=False):
# Append system hooks first
if list_by == 'folder':
result['system'] = dict() if show_info else set()
- _append_folder(result['system'], hook_folder)
+ _append_folder(result['system'], HOOK_FOLDER)
else:
- _append_folder(result, hook_folder)
+ _append_folder(result, HOOK_FOLDER)
except OSError:
logger.debug("system hook folder not found for action '%s' in %s",
- action, hook_folder)
+ action, HOOK_FOLDER)
try:
# Append custom hooks
if list_by == 'folder':
result['custom'] = dict() if show_info else set()
- _append_folder(result['custom'], custom_hook_folder)
+ _append_folder(result['custom'], CUSTOM_HOOK_FOLDER)
else:
- _append_folder(result, custom_hook_folder)
+ _append_folder(result, CUSTOM_HOOK_FOLDER)
except OSError:
logger.debug("custom hook folder not found for action '%s' in %s",
- action, custom_hook_folder)
+ action, CUSTOM_HOOK_FOLDER)
- return { 'hooks': result }
+ return {'hooks': result}
def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
@@ -226,7 +229,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
(name, priority, path, succeed) as arguments
"""
- result = { 'succeed': {}, 'failed': {} }
+ result = {'succeed': {}, 'failed': {}}
hooks_dict = {}
# Retrieve hooks
@@ -244,7 +247,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
for n in hooks:
for key in hooks_names.keys():
if key == n or key.startswith("%s_" % n) \
- and key not in all_hooks:
+ and key not in all_hooks:
all_hooks.append(key)
# Iterate over given hooks names list
@@ -258,7 +261,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
for h in hl:
# Update hooks dict
d = hooks_dict.get(h['priority'], dict())
- d.update({ n: { 'path': h['path'] }})
+ d.update({n: {'path': h['path']}})
hooks_dict[h['priority']] = d
if not hooks_dict:
return result
@@ -278,7 +281,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
hook_args = pre_callback(name=name, priority=priority,
path=path, args=args)
hook_exec(path, args=hook_args, chdir=chdir, env=env,
- no_trace=no_trace, raise_on_error=True)
+ no_trace=no_trace, raise_on_error=True, user="root")
except MoulinetteError as e:
state = 'failed'
logger.error(e.strerror, exc_info=1)
@@ -295,7 +298,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
- chdir=None, env=None):
+ chdir=None, env=None, user="admin", stdout_callback=None,
+ stderr_callback=None):
"""
Execute hook from a file with arguments
@@ -306,10 +310,10 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
no_trace -- Do not print each command that will be executed
chdir -- The directory from where the script will be executed
env -- Dictionnary of environment variables to export
+ user -- User with which to run the command
"""
from moulinette.utils.process import call_async_output
- from yunohost.app import _value_for_locale
# Validate hook path
if path[0] != '/':
@@ -329,32 +333,52 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
else:
cmd_script = path
+ # Add Execution dir to environment var
+ if env is None:
+ env = {}
+ env['YNH_CWD'] = chdir
+
+ stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
+ env['YNH_STDINFO'] = stdinfo
+
# Construct command to execute
- command = ['sudo', '-n', '-u', 'admin', '-H', 'sh', '-c']
+ if user == "root":
+ command = ['sh', '-c']
+ else:
+ command = ['sudo', '-n', '-u', user, '-H', 'sh', '-c']
+
if no_trace:
cmd = '/bin/bash "{script}" {args}'
else:
# use xtrace on fd 7 which is redirected to stdout
cmd = 'BASH_XTRACEFD=7 /bin/bash -x "{script}" {args} 7>&1'
- if env:
- # prepend environment variables
- cmd = '{0} {1}'.format(
- ' '.join(['{0}={1}'.format(k, shell_quote(v)) \
- for k, v in env.items()]), cmd)
+
+ # prepend environment variables
+ cmd = '{0} {1}'.format(
+ ' '.join(['{0}={1}'.format(k, shell_quote(v))
+ for k, v in env.items()]), cmd)
command.append(cmd.format(script=cmd_script, args=cmd_args))
if logger.isEnabledFor(log.DEBUG):
- logger.info(m18n.n('executing_command', command=' '.join(command)))
+ logger.debug(m18n.n('executing_command', command=' '.join(command)))
else:
- logger.info(m18n.n('executing_script', script=path))
+ logger.debug(m18n.n('executing_script', script=path))
# Define output callbacks and call command
callbacks = (
- lambda l: logger.info(l.rstrip()),
- lambda l: logger.warning(l.rstrip()),
+ stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()),
+ stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()),
)
+
+ if stdinfo:
+ callbacks = ( callbacks[0], callbacks[1],
+ lambda l: logger.info(l.rstrip()))
+
+ logger.debug("About to run the command '%s'" % command)
+
returncode = call_async_output(
- command, callbacks, shell=False, cwd=chdir
+ command, callbacks, shell=False, cwd=chdir,
+ stdinfo=stdinfo
)
# Check and return process' return code
@@ -385,6 +409,7 @@ def _extract_filename_parts(filename):
_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.UNICODE).search
+
def shell_quote(s):
"""Return a shell-escaped version of the string *s*."""
s = str(s)
diff --git a/src/yunohost/log.py b/src/yunohost/log.py
new file mode 100644
index 000000000..c105b8279
--- /dev/null
+++ b/src/yunohost/log.py
@@ -0,0 +1,462 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+ Copyright (C) 2018 YunoHost
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+""" yunohost_log.py
+
+ Manage debug logs
+"""
+
+import os
+import yaml
+import errno
+import collections
+
+from datetime import datetime
+from logging import FileHandler, getLogger, Formatter
+from sys import exc_info
+
+from moulinette import m18n, msettings
+from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file
+
+CATEGORIES_PATH = '/var/log/yunohost/categories/'
+OPERATIONS_PATH = '/var/log/yunohost/categories/operation/'
+CATEGORIES = ['operation', 'history', 'package', 'system', 'access', 'service',
+ 'app']
+METADATA_FILE_EXT = '.yml'
+LOG_FILE_EXT = '.log'
+RELATED_CATEGORIES = ['app', 'domain', 'service', 'user']
+
+logger = getActionLogger('yunohost.log')
+
+
+def log_list(category=[], limit=None):
+ """
+ List available logs
+
+ Keyword argument:
+ limit -- Maximum number of logs
+ """
+
+ categories = category
+ is_api = msettings.get('interface') == 'api'
+
+ # In cli we just display `operation` logs by default
+ if not categories:
+ categories = ["operation"] if not is_api else CATEGORIES
+
+ result = collections.OrderedDict()
+ for category in categories:
+ result[category] = []
+
+ category_path = os.path.join(CATEGORIES_PATH, category)
+ if not os.path.exists(category_path):
+ logger.debug(m18n.n('log_category_404', category=category))
+
+ continue
+
+ logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
+ os.listdir(category_path))
+ logs = reversed(sorted(logs))
+
+ if limit is not None:
+ logs = logs[:limit]
+
+ for log in logs:
+
+ base_filename = log[:-len(METADATA_FILE_EXT)]
+ md_filename = log
+ md_path = os.path.join(category_path, md_filename)
+
+ log = base_filename.split("-")
+
+ entry = {
+ "name": base_filename,
+ "path": md_path,
+ }
+ entry["description"] = _get_description_from_name(base_filename)
+ try:
+ log_datetime = datetime.strptime(" ".join(log[:2]),
+ "%Y%m%d %H%M%S")
+ except ValueError:
+ pass
+ else:
+ entry["started_at"] = log_datetime
+
+ result[category].append(entry)
+
+ # Reverse the order of log when in cli, more comfortable to read (avoid
+ # unecessary scrolling)
+ if not is_api:
+ for category in result:
+ result[category] = list(reversed(result[category]))
+
+ return result
+
+
+def log_display(path, number=50, share=False):
+ """
+ Display a log file enriched with metadata if any.
+
+ If the file_name is not an absolute path, it will try to search the file in
+ the unit operations log path (see OPERATIONS_PATH).
+
+ Argument:
+ file_name
+ number
+ share
+ """
+
+ # Normalize log/metadata paths and filenames
+ abs_path = path
+ log_path = None
+ if not path.startswith('/'):
+ for category in CATEGORIES:
+ abs_path = os.path.join(CATEGORIES_PATH, category, path)
+ if os.path.exists(abs_path) or os.path.exists(abs_path + METADATA_FILE_EXT):
+ break
+
+ if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT):
+ log_path = abs_path
+
+ if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT):
+ base_path = ''.join(os.path.splitext(abs_path)[:-1])
+ else:
+ base_path = abs_path
+ base_filename = os.path.basename(base_path)
+ md_path = base_path + METADATA_FILE_EXT
+ if log_path is None:
+ log_path = base_path + LOG_FILE_EXT
+
+ if not os.path.exists(md_path) and not os.path.exists(log_path):
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('log_does_exists', log=path))
+
+ infos = {}
+
+ # If it's a unit operation, display the name and the description
+ if base_path.startswith(CATEGORIES_PATH):
+ infos["description"] = _get_description_from_name(base_filename)
+ infos['name'] = base_filename
+
+ if share:
+ from yunohost.utils.yunopaste import yunopaste
+ content = ""
+ if os.path.exists(md_path):
+ content += read_file(md_path)
+ content += "\n============\n\n"
+ if os.path.exists(log_path):
+ content += read_file(log_path)
+
+ url = yunopaste(content)
+
+ logger.info(m18n.n("log_available_on_yunopaste", url=url))
+ if msettings.get('interface') == 'api':
+ return {"url": url}
+ else:
+ return
+
+ # Display metadata if exist
+ if os.path.exists(md_path):
+ with open(md_path, "r") as md_file:
+ try:
+ metadata = yaml.safe_load(md_file)
+ infos['metadata_path'] = md_path
+ infos['metadata'] = metadata
+ if 'log_path' in metadata:
+ log_path = metadata['log_path']
+ except yaml.YAMLError:
+ error = m18n.n('log_corrupted_md_file', file=md_path)
+ if os.path.exists(log_path):
+ logger.warning(error)
+ else:
+ raise MoulinetteError(errno.EINVAL, error)
+
+ # Display logs if exist
+ if os.path.exists(log_path):
+ from yunohost.service import _tail
+ logs = _tail(log_path, int(number))
+ infos['log_path'] = log_path
+ infos['logs'] = logs
+
+ return infos
+
+
+def is_unit_operation(entities=['app', 'domain', 'service', 'user'],
+ exclude=['auth', 'password'], operation_key=None):
+ """
+ Configure quickly a unit operation
+
+ This decorator help you to configure the record of a unit operations.
+
+ Argument:
+ entities A list of entity types related to the unit operation. The entity
+ type is searched inside argument's names of the decorated function. If
+ something match, the argument value is added as related entity. If the
+ argument name is different you can specify it with a tuple
+ (argname, entity_type) instead of just put the entity type.
+
+ exclude Remove some arguments from the context. By default, arguments
+ called 'password' and 'auth' are removed. If an argument is an object, you
+ need to exclude it or create manually the unit operation without this
+ decorator.
+
+ operation_key A key to describe the unit operation log used to create the
+ filename and search a translation. Please ensure that this key prefixed by
+ 'log_' is present in locales/en.json otherwise it won't be translatable.
+
+ """
+ def decorate(func):
+ def func_wrapper(*args, **kwargs):
+ op_key = operation_key
+ if op_key is None:
+ op_key = func.__name__
+
+ # If the function is called directly from an other part of the code
+ # and not by the moulinette framework, we need to complete kwargs
+ # dictionnary with the args list.
+ # Indeed, we use convention naming in this decorator and we need to
+ # know name of each args (so we need to use kwargs instead of args)
+ if len(args) > 0:
+ from inspect import getargspec
+ keys = getargspec(func).args
+ if 'operation_logger' in keys:
+ keys.remove('operation_logger')
+ for k, arg in enumerate(args):
+ kwargs[keys[k]] = arg
+ args = ()
+
+ # Search related entity in arguments of the decorated function
+ related_to = []
+ for entity in entities:
+ if isinstance(entity, tuple):
+ entity_type = entity[1]
+ entity = entity[0]
+ else:
+ entity_type = entity
+
+ if entity in kwargs and kwargs[entity] is not None:
+ if isinstance(kwargs[entity], basestring):
+ related_to.append((entity_type, kwargs[entity]))
+ else:
+ for x in kwargs[entity]:
+ related_to.append((entity_type, x))
+
+ context = kwargs.copy()
+
+ # Exclude unappropriate data from the context
+ for field in exclude:
+ if field in context:
+ context.pop(field, None)
+ operation_logger = OperationLogger(op_key, related_to, args=context)
+
+ try:
+ # Start the actual function, and give the unit operation
+ # in argument to let the developper start the record itself
+ args = (operation_logger,) + args
+ result = func(*args, **kwargs)
+ except Exception as e:
+ operation_logger.error(e)
+ raise
+ else:
+ operation_logger.success()
+ return result
+ return func_wrapper
+ return decorate
+
+
+class OperationLogger(object):
+ """
+ Instances of this class represents unit operation done on the ynh instance.
+
+ Each time an action of the yunohost cli/api change the system, one or
+ several unit operations should be registered.
+
+ This class record logs and metadata like context or start time/end time.
+ """
+
+ def __init__(self, operation, related_to=None, **kwargs):
+ # TODO add a way to not save password on app installation
+ self.operation = operation
+ self.related_to = related_to
+ self.extra = kwargs
+ self.started_at = None
+ self.ended_at = None
+ self.logger = None
+ self._name = None
+
+ self.path = OPERATIONS_PATH
+
+ if not os.path.exists(self.path):
+ os.makedirs(self.path)
+
+ def start(self):
+ """
+ Start to record logs that change the system
+ Until this start method is run, no unit operation will be registered.
+ """
+
+ if self.started_at is None:
+ self.started_at = datetime.now()
+ self.flush()
+ self._register_log()
+
+ def _register_log(self):
+ """
+ Register log with a handler connected on log system
+ """
+
+ # TODO add a way to not save password on app installation
+ filename = os.path.join(self.path, self.name + LOG_FILE_EXT)
+ self.file_handler = FileHandler(filename)
+ self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s')
+
+ # Listen to the root logger
+ self.logger = getLogger('yunohost')
+ self.logger.addHandler(self.file_handler)
+
+ def flush(self):
+ """
+ Write or rewrite the metadata file with all metadata known
+ """
+
+ filename = os.path.join(self.path, self.name + METADATA_FILE_EXT)
+ with open(filename, 'w') as outfile:
+ yaml.safe_dump(self.metadata, outfile, default_flow_style=False)
+
+ @property
+ def name(self):
+ """
+ Name of the operation
+ This name is used as filename, so don't use space
+ """
+ if self._name is not None:
+ return self._name
+
+ name = [self.started_at.strftime("%Y%m%d-%H%M%S")]
+ name += [self.operation]
+
+ if hasattr(self, "name_parameter_override"):
+ # This is for special cases where the operation is not really
+ # unitary. For instance, the regen conf cannot be logged "per
+ # service" because of the way it's built
+ name.append(self.name_parameter_override)
+ elif self.related_to:
+ # We use the name of the first related thing
+ name.append(self.related_to[0][1])
+
+ self._name = '-'.join(name)
+ return self._name
+
+ @property
+ def metadata(self):
+ """
+ Dictionnary of all metadata collected
+ """
+
+ data = {
+ 'started_at': self.started_at,
+ 'operation': self.operation,
+ }
+ if self.related_to is not None:
+ data['related_to'] = self.related_to
+ if self.ended_at is not None:
+ data['ended_at'] = self.ended_at
+ data['success'] = self._success
+ if self.error is not None:
+ data['error'] = self._error
+ # TODO: detect if 'extra' erase some key of 'data'
+ data.update(self.extra)
+ return data
+
+ def success(self):
+ """
+ Declare the success end of the unit operation
+ """
+ self.close()
+
+ def error(self, error):
+ """
+ Declare the failure of the unit operation
+ """
+ return self.close(error)
+
+ def close(self, error=None):
+ """
+ Close properly the unit operation
+ """
+ if self.ended_at is not None or self.started_at is None:
+ return
+ if error is not None and not isinstance(error, basestring):
+ error = str(error)
+ self.ended_at = datetime.now()
+ self._error = error
+ self._success = error is None
+ if self.logger is not None:
+ self.logger.removeHandler(self.file_handler)
+
+ is_api = msettings.get('interface') == 'api'
+ desc = _get_description_from_name(self.name)
+ if error is None:
+ if is_api:
+ msg = m18n.n('log_link_to_log', name=self.name, desc=desc)
+ else:
+ msg = m18n.n('log_help_to_get_log', name=self.name, desc=desc)
+ logger.debug(msg)
+ else:
+ if is_api:
+ msg = "" + m18n.n('log_link_to_failed_log',
+ name=self.name, desc=desc) + ""
+ else:
+ msg = m18n.n('log_help_to_get_failed_log', name=self.name,
+ desc=desc)
+ logger.info(msg)
+ self.flush()
+ return msg
+
+ def __del__(self):
+ """
+ Try to close the unit operation, if it's missing.
+ The missing of the message below could help to see an electrical
+ shortage.
+ """
+ self.error(m18n.n('log_operation_unit_unclosed_properly'))
+
+
+def _get_description_from_name(name):
+ """
+ Return the translated description from the filename
+ """
+
+ parts = name.split("-", 3)
+ try:
+ try:
+ datetime.strptime(" ".join(parts[:2]), "%Y%m%d %H%M%S")
+ except ValueError:
+ key = "log_" + parts[0]
+ args = parts[1:]
+ else:
+ key = "log_" + parts[2]
+ args = parts[3:]
+ return m18n.n(key, *args)
+ except IndexError:
+ return name
diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py
index d0fe224e9..fc10a4fbc 100644
--- a/src/yunohost/monitor.py
+++ b/src/yunohost/monitor.py
@@ -35,18 +35,20 @@ import errno
import os
import dns.resolver
import cPickle as pickle
-from datetime import datetime, timedelta
+from datetime import datetime
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
-from yunohost.domain import get_public_ip
+from yunohost.utils.network import get_public_ip
+from yunohost.domain import _get_maindomain
logger = getActionLogger('yunohost.monitor')
-glances_uri = 'http://127.0.0.1:61209'
-stats_path = '/var/lib/yunohost/stats'
-crontab_path = '/etc/cron.d/yunohost-monitor'
+GLANCES_URI = 'http://127.0.0.1:61209'
+STATS_PATH = '/var/lib/yunohost/stats'
+CRONTAB_PATH = '/etc/cron.d/yunohost-monitor'
def monitor_disk(units=None, mountpoint=None, human_readable=False):
@@ -87,13 +89,13 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False):
# Retrieve monitoring for unit(s)
for u in units:
if u == 'io':
- ## Define setter
+ # Define setter
if len(units) > 1:
def _set(dn, dvalue):
try:
result[dn][u] = dvalue
except KeyError:
- result[dn] = { u: dvalue }
+ result[dn] = {u: dvalue}
else:
def _set(dn, dvalue):
result[dn] = dvalue
@@ -111,13 +113,13 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False):
for dname in devices_names:
_set(dname, 'not-available')
elif u == 'filesystem':
- ## Define setter
+ # Define setter
if len(units) > 1:
def _set(dn, dvalue):
try:
result[dn][u] = dvalue
except KeyError:
- result[dn] = { u: dvalue }
+ result[dn] = {u: dvalue}
else:
def _set(dn, dvalue):
result[dn] = dvalue
@@ -162,6 +164,7 @@ def monitor_network(units=None, human_readable=False):
units = ['check', 'usage', 'infos']
# Get network devices and their addresses
+ # TODO / FIXME : use functions in utils/network.py to manage this
devices = {}
output = subprocess.check_output('ip addr show'.split())
for d in re.split('^(?:[0-9]+: )', output, flags=re.MULTILINE):
@@ -174,8 +177,7 @@ def monitor_network(units=None, human_readable=False):
for u in units:
if u == 'check':
result[u] = {}
- with open('/etc/yunohost/current_host', 'r') as f:
- domain = f.readline().rstrip()
+ domain = _get_maindomain()
cmd_check_smtp = os.system('/bin/nc -z -w1 yunohost.org 25')
if cmd_check_smtp == 0:
smtp_check = m18n.n('network_check_smtp_ok')
@@ -183,11 +185,11 @@ def monitor_network(units=None, human_readable=False):
smtp_check = m18n.n('network_check_smtp_ko')
try:
- answers = dns.resolver.query(domain,'MX')
+ answers = dns.resolver.query(domain, 'MX')
mx_check = {}
i = 0
for server in answers:
- mx_id = 'mx%s' %i
+ mx_id = 'mx%s' % i
mx_check[mx_id] = server
i = i + 1
except:
@@ -210,11 +212,9 @@ def monitor_network(units=None, human_readable=False):
else:
logger.debug('interface name %s was not found', iname)
elif u == 'infos':
- try:
- p_ipv4 = get_public_ip()
- except:
- p_ipv4 = 'unknown'
+ p_ipv4 = get_public_ip() or 'unknown'
+ # TODO / FIXME : use functions in utils/network.py to manage this
l_ip = 'unknown'
for name, addrs in devices.items():
if name == 'lo':
@@ -307,7 +307,7 @@ def monitor_update_stats(period):
stats = _retrieve_stats(period)
if not stats:
- stats = { 'disk': {}, 'network': {}, 'system': {}, 'timestamp': [] }
+ stats = {'disk': {}, 'network': {}, 'system': {}, 'timestamp': []}
monitor = None
# Get monitoring stats
@@ -357,7 +357,7 @@ def monitor_update_stats(period):
if 'usage' in stats['network'] and iname in stats['network']['usage']:
curr = stats['network']['usage'][iname]
net_usage[iname] = _append_to_stats(curr, values, 'time_since_update')
- stats['network'] = { 'usage': net_usage, 'infos': monitor['network']['infos'] }
+ stats['network'] = {'usage': net_usage, 'infos': monitor['network']['infos']}
# Append system stats
for unit, values in monitor['system'].items():
@@ -421,8 +421,8 @@ def monitor_enable(with_stats=False):
rules = ('*/5 * * * * root {cmd} day >> /dev/null\n'
'3 * * * * root {cmd} week >> /dev/null\n'
'6 */4 * * * root {cmd} month >> /dev/null').format(
- cmd='/usr/bin/yunohost --quiet monitor update-stats')
- with open(crontab_path, 'w') as f:
+ cmd='/usr/bin/yunohost --quiet monitor update-stats')
+ with open(CRONTAB_PATH, 'w') as f:
f.write(rules)
logger.success(m18n.n('monitor_enabled'))
@@ -447,7 +447,7 @@ def monitor_disable():
# Remove crontab
try:
- os.remove(crontab_path)
+ os.remove(CRONTAB_PATH)
except:
pass
@@ -460,7 +460,7 @@ def _get_glances_api():
"""
try:
- p = xmlrpclib.ServerProxy(glances_uri)
+ p = xmlrpclib.ServerProxy(GLANCES_URI)
p.system.methodHelp('getAll')
except (xmlrpclib.ProtocolError, IOError):
pass
@@ -530,7 +530,7 @@ def binary_to_human(n, customary=False):
symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
prefix = {}
for i, s in enumerate(symbols):
- prefix[s] = 1 << (i+1)*10
+ prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
@@ -552,9 +552,9 @@ def _retrieve_stats(period, date=None):
# Retrieve pickle file
if date is not None:
timestamp = calendar.timegm(date)
- pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period)
+ pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period)
else:
- pkl_file = '%s/%s.pkl' % (stats_path, period)
+ pkl_file = '%s/%s.pkl' % (STATS_PATH, period)
if not os.path.isfile(pkl_file):
return False
@@ -581,16 +581,16 @@ def _save_stats(stats, period, date=None):
# Set pickle file name
if date is not None:
timestamp = calendar.timegm(date)
- pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period)
+ pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period)
else:
- pkl_file = '%s/%s.pkl' % (stats_path, period)
- if not os.path.isdir(stats_path):
- os.makedirs(stats_path)
+ pkl_file = '%s/%s.pkl' % (STATS_PATH, period)
+ if not os.path.isdir(STATS_PATH):
+ os.makedirs(STATS_PATH)
# Limit stats
if date is None:
t = stats['timestamp']
- limit = { 'day': 86400, 'week': 604800, 'month': 2419200 }
+ limit = {'day': 86400, 'week': 604800, 'month': 2419200}
if (t[len(t) - 1] - t[0]) > limit[period]:
begin = t[len(t) - 1] - limit[period]
stats = _filter_stats(stats, begin)
@@ -612,7 +612,7 @@ def _monitor_all(period=None, since=None):
since -- Timestamp of the stats beginning
"""
- result = { 'disk': {}, 'network': {}, 'system': {} }
+ result = {'disk': {}, 'network': {}, 'system': {}}
# Real-time stats
if period == 'day' and since is None:
@@ -697,7 +697,7 @@ def _calculate_stats_mean(stats):
s[k] = _mean(v, t, ts)
elif isinstance(v, list):
try:
- nums = [ float(x * t[i]) for i, x in enumerate(v) ]
+ nums = [float(x * t[i]) for i, x in enumerate(v)]
except:
pass
else:
diff --git a/src/yunohost/service.py b/src/yunohost/service.py
index ab26dd2bc..66ae837a9 100644
--- a/src/yunohost/service.py
+++ b/src/yunohost/service.py
@@ -26,22 +26,26 @@
import os
import time
import yaml
-import glob
+import json
import subprocess
import errno
import shutil
import hashlib
-from difflib import unified_diff
+from difflib import unified_diff
+from datetime import datetime
+
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils import log, filesystem
-from yunohost.hook import hook_list, hook_callback
+from yunohost.hook import hook_callback
+from yunohost.log import is_unit_operation
-
-base_conf_path = '/home/yunohost.conf'
-backup_conf_dir = os.path.join(base_conf_path, 'backup')
-pending_conf_dir = os.path.join(base_conf_path, 'pending')
+BASE_CONF_PATH = '/home/yunohost.conf'
+BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
+PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, 'pending')
+MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
logger = log.getActionLogger('yunohost.service')
@@ -60,9 +64,9 @@ def service_add(name, status=None, log=None, runlevel=None):
services = _get_services()
if not status:
- services[name] = { 'status': 'service' }
+ services[name] = {'status': 'service'}
else:
- services[name] = { 'status': status }
+ services[name] = {'status': status}
if log is not None:
services[name]['log'] = log
@@ -73,6 +77,7 @@ def service_add(name, status=None, log=None, runlevel=None):
try:
_save_services(services)
except:
+ # we'll get a logger.warning with more details in _save_services
raise MoulinetteError(errno.EIO, m18n.n('service_add_failed', service=name))
logger.success(m18n.n('service_added', service=name))
@@ -96,6 +101,7 @@ def service_remove(name):
try:
_save_services(services)
except:
+ # we'll get a logger.warning with more details in _save_services
raise MoulinetteError(errno.EIO, m18n.n('service_remove_failed', service=name))
logger.success(m18n.n('service_removed', service=name))
@@ -111,14 +117,17 @@ def service_start(names):
"""
if isinstance(names, str):
names = [names]
+
for name in names:
if _run_service_command('start', name):
logger.success(m18n.n('service_started', service=name))
else:
if service_status(name)['status'] != 'running':
raise MoulinetteError(errno.EPERM,
- m18n.n('service_start_failed', service=name))
- logger.info(m18n.n('service_already_started', service=name))
+ m18n.n('service_start_failed',
+ service=name,
+ logs=_get_journalctl_logs(name)))
+ logger.debug(m18n.n('service_already_started', service=name))
def service_stop(names):
@@ -137,11 +146,13 @@ def service_stop(names):
else:
if service_status(name)['status'] != 'inactive':
raise MoulinetteError(errno.EPERM,
- m18n.n('service_stop_failed', service=name))
- logger.info(m18n.n('service_already_stopped', service=name))
+ m18n.n('service_stop_failed',
+ service=name,
+ logs=_get_journalctl_logs(name)))
+ logger.debug(m18n.n('service_already_stopped', service=name))
-
-def service_enable(names):
+@is_unit_operation()
+def service_enable(operation_logger, names):
"""
Enable one or more services
@@ -149,6 +160,7 @@ def service_enable(names):
names -- Services name to enable
"""
+ operation_logger.start()
if isinstance(names, str):
names = [names]
for name in names:
@@ -156,7 +168,9 @@ def service_enable(names):
logger.success(m18n.n('service_enabled', service=name))
else:
raise MoulinetteError(errno.EPERM,
- m18n.n('service_enable_failed', service=name))
+ m18n.n('service_enable_failed',
+ service=name,
+ logs=_get_journalctl_logs(name)))
def service_disable(names):
@@ -174,7 +188,9 @@ def service_disable(names):
logger.success(m18n.n('service_disabled', service=name))
else:
raise MoulinetteError(errno.EPERM,
- m18n.n('service_disable_failed', service=name))
+ m18n.n('service_disable_failed',
+ service=name,
+ logs=_get_journalctl_logs(name)))
def service_status(names=[]):
@@ -200,45 +216,91 @@ def service_status(names=[]):
raise MoulinetteError(errno.EINVAL,
m18n.n('service_unknown', service=name))
- status = None
- if 'status' not in services[name] or \
- services[name]['status'] == 'service':
- status = 'service %s status' % name
+ # this "service" isn't a service actually so we skip it
+ #
+ # the historical reason is because regenconf has been hacked into the
+ # service part of YunoHost will in some situation we need to regenconf
+ # for things that aren't services
+ # the hack was to add fake services...
+ # we need to extract regenconf from service at some point, also because
+ # some app would really like to use it
+ if "status" in services[name] and services[name]["status"] is None:
+ continue
+
+ status = _get_service_information_from_systemd(name)
+
+ # try to get status using alternative version if they exists
+ # this is for mariadb/mysql but is generic in case of
+ alternates = services[name].get("alternates", [])
+ while status is None and alternates:
+ status = _get_service_information_from_systemd(alternates.pop())
+
+ if status is None:
+ logger.error("Failed to get status information via dbus for service %s, systemctl didn't recognize this service ('NoSuchUnit')." % name)
+ result[name] = {
+ 'status': "unknown",
+ 'loaded': "unknown",
+ 'active': "unknown",
+ 'active_at': {
+ "timestamp": "unknown",
+ "human": "unknown",
+ },
+ 'description': "Error: failed to get information for this service, it doesn't exists for systemd",
+ 'service_file_path': "unknown",
+ }
+
else:
- status = str(services[name]['status'])
+ translation_key = "service_description_%s" % name
+ description = m18n.n(translation_key)
- runlevel = 5
- if 'runlevel' in services[name].keys():
- runlevel = int(services[name]['runlevel'])
+ # that mean that we don't have a translation for this string
+ # that's the only way to test for that for now
+ # if we don't have it, uses the one provided by systemd
+ if description == translation_key:
+ description = str(status.get("Description", ""))
- result[name] = { 'status': 'unknown', 'loaded': 'unknown' }
-
- # Retrieve service status
- try:
- ret = subprocess.check_output(status, stderr=subprocess.STDOUT,
- shell=True)
- except subprocess.CalledProcessError as e:
- if 'usage:' in e.output.lower():
- logger.warning(m18n.n('service_status_failed', service=name))
- else:
- result[name]['status'] = 'inactive'
- else:
- result[name]['status'] = 'running'
-
- # Retrieve service loading
- rc_path = glob.glob("/etc/rc%d.d/S[0-9][0-9]%s" % (runlevel, name))
- if len(rc_path) == 1 and os.path.islink(rc_path[0]):
- result[name]['loaded'] = 'enabled'
- elif os.path.isfile("/etc/init.d/%s" % name):
- result[name]['loaded'] = 'disabled'
- else:
- result[name]['loaded'] = 'not-found'
+ result[name] = {
+ 'status': str(status.get("SubState", "unknown")),
+ 'loaded': "enabled" if str(status.get("LoadState", "unknown")) == "loaded" else str(status.get("LoadState", "unknown")),
+ 'active': str(status.get("ActiveState", "unknown")),
+ 'active_at': {
+ "timestamp": str(status.get("ActiveEnterTimestamp", "unknown")),
+ "human": datetime.fromtimestamp(status["ActiveEnterTimestamp"] / 1000000).strftime("%F %X") if "ActiveEnterTimestamp" in status else "unknown",
+ },
+ 'description': description,
+ 'service_file_path': str(status.get("FragmentPath", "unknown")),
+ }
if len(names) == 1:
return result[names[0]]
return result
+def _get_service_information_from_systemd(service):
+ "this is the equivalent of 'systemctl status $service'"
+ import dbus
+ from dbus.exceptions import DBusException
+
+ d = dbus.SystemBus()
+
+ systemd = d.get_object('org.freedesktop.systemd1','/org/freedesktop/systemd1')
+ manager = dbus.Interface(systemd, 'org.freedesktop.systemd1.Manager')
+
+ try:
+ service_path = manager.GetUnit(service + ".service")
+ except DBusException as exception:
+ if exception.get_dbus_name() == 'org.freedesktop.systemd1.NoSuchUnit':
+ return None
+ raise
+
+ service_proxy = d.get_object('org.freedesktop.systemd1', service_path)
+
+ # unit_proxy = dbus.Interface(service_proxy, 'org.freedesktop.systemd1.Unit',)
+ properties_interface = dbus.Interface(service_proxy, 'org.freedesktop.DBus.Properties')
+
+ return properties_interface.GetAll('org.freedesktop.systemd1.Unit')
+
+
def service_log(name, number=50):
"""
Log every log files of a service
@@ -253,25 +315,38 @@ def service_log(name, number=50):
if name not in services.keys():
raise MoulinetteError(errno.EINVAL, m18n.n('service_unknown', service=name))
- if 'log' in services[name]:
- log_list = services[name]['log']
- result = {}
- if not isinstance(log_list, list):
- log_list = [log_list]
-
- for log_path in log_list:
- if os.path.isdir(log_path):
- for log in [ f for f in os.listdir(log_path) if os.path.isfile(os.path.join(log_path, f)) and f[-4:] == '.log' ]:
- result[os.path.join(log_path, log)] = _tail(os.path.join(log_path, log), int(number))
- else:
- result[log_path] = _tail(log_path, int(number))
- else:
+ if 'log' not in services[name]:
raise MoulinetteError(errno.EPERM, m18n.n('service_no_log', service=name))
+ log_list = services[name]['log']
+
+ if not isinstance(log_list, list):
+ log_list = [log_list]
+
+ result = {}
+
+ for log_path in log_list:
+ # log is a file, read it
+ if not os.path.isdir(log_path):
+ result[log_path] = _tail(log_path, int(number)) if os.path.exists(log_path) else []
+ continue
+
+ for log_file in os.listdir(log_path):
+ log_file_path = os.path.join(log_path, log_file)
+ # not a file : skip
+ if not os.path.isfile(log_file_path):
+ continue
+
+ if not log_file.endswith(".log"):
+ continue
+
+ result[log_file_path] = _tail(log_file_path, int(number)) if os.path.exists(log_file_path) else []
+
return result
-def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
+@is_unit_operation([('names', 'service')])
+def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
"""
Regenerate the configuration file(s) for a service
@@ -289,42 +364,59 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Return the list of pending conf
if list_pending:
pending_conf = _get_pending_conf(names)
- if with_diff:
- for service, conf_files in pending_conf.items():
- for system_path, pending_path in conf_files.items():
- pending_conf[service][system_path] = {
- 'pending_conf': pending_path,
- 'diff': _get_files_diff(
- system_path, pending_path, True),
- }
+
+ if not with_diff:
+ return pending_conf
+
+ for service, conf_files in pending_conf.items():
+ for system_path, pending_path in conf_files.items():
+
+ pending_conf[service][system_path] = {
+ 'pending_conf': pending_path,
+ 'diff': _get_files_diff(
+ system_path, pending_path, True),
+ }
+
return pending_conf
- # Clean pending conf directory
- if os.path.isdir(pending_conf_dir):
+ if not dry_run:
+ operation_logger.related_to = [('service', x) for x in names]
if not names:
- shutil.rmtree(pending_conf_dir, ignore_errors=True)
+ operation_logger.name_parameter_override = 'all'
+ elif len(names) != 1:
+ operation_logger.name_parameter_override = str(len(operation_logger.related_to))+'_services'
+ operation_logger.start()
+
+ # Clean pending conf directory
+ if os.path.isdir(PENDING_CONF_DIR):
+ if not names:
+ shutil.rmtree(PENDING_CONF_DIR, ignore_errors=True)
else:
for name in names:
- shutil.rmtree(os.path.join(pending_conf_dir, name),
+ shutil.rmtree(os.path.join(PENDING_CONF_DIR, name),
ignore_errors=True)
else:
- filesystem.mkdir(pending_conf_dir, 0755, True)
+ filesystem.mkdir(PENDING_CONF_DIR, 0755, True)
# Format common hooks arguments
common_args = [1 if force else 0, 1 if dry_run else 0]
# Execute hooks for pre-regen
- pre_args = ['pre',] + common_args
+ pre_args = ['pre', ] + common_args
+
def _pre_call(name, priority, path, args):
# create the pending conf directory for the service
- service_pending_path = os.path.join(pending_conf_dir, name)
- filesystem.mkdir(service_pending_path, 0755, True, uid='admin')
+ service_pending_path = os.path.join(PENDING_CONF_DIR, name)
+ filesystem.mkdir(service_pending_path, 0755, True, uid='root')
+
# return the arguments to pass to the script
- return pre_args + [service_pending_path,]
+ return pre_args + [service_pending_path, ]
+
pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)
# Update the services name
names = pre_result['succeed'].keys()
+
if not names:
raise MoulinetteError(errno.EIO,
m18n.n('service_regenconf_failed',
@@ -333,11 +425,16 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Set the processing method
_regen = _process_regen_conf if not dry_run else lambda *a, **k: True
+ operation_logger.related_to = []
+
# Iterate over services and process pending conf
for service, conf_files in _get_pending_conf(names).items():
- logger.info(m18n.n(
- 'service_regenconf_pending_applying' if not dry_run else \
- 'service_regenconf_dry_pending_applying',
+ if not dry_run:
+ operation_logger.related_to.append(('service', service))
+
+ logger.debug(m18n.n(
+ 'service_regenconf_pending_applying' if not dry_run else
+ 'service_regenconf_dry_pending_applying',
service=service))
conf_hashes = _get_conf_hashes(service)
@@ -378,10 +475,11 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
regenerated = _regen(
system_path, pending_path, save=False)
else:
- logger.warning(m18n.n(
+ logger.info(m18n.n(
'service_conf_file_manually_removed',
conf=system_path))
conf_status = 'removed'
+
# -> system conf is not managed yet
elif not saved_hash:
logger.debug("> system conf is not managed yet")
@@ -389,16 +487,23 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
logger.debug("> no changes to system conf has been made")
conf_status = 'managed'
regenerated = True
- elif force and to_remove:
+ elif not to_remove:
+ # If the conf exist but is not managed yet, and is not to be removed,
+ # we assume that it is safe to regen it, since the file is backuped
+ # anyway (by default in _regen), as long as we warn the user
+ # appropriately.
+ logger.info(m18n.n('service_conf_new_managed_file',
+ conf=system_path, service=service))
+ regenerated = _regen(system_path, pending_path)
+ conf_status = 'new'
+ elif force:
regenerated = _regen(system_path)
conf_status = 'force-removed'
- elif force:
- regenerated = _regen(system_path, pending_path)
- conf_status = 'force-updated'
else:
- logger.warning(m18n.n('service_conf_file_not_managed',
- conf=system_path))
+ logger.info(m18n.n('service_conf_file_kept_back',
+ conf=system_path, service=service))
conf_status = 'unmanaged'
+
# -> system conf has not been manually modified
elif system_hash == saved_hash:
if to_remove:
@@ -411,6 +516,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
logger.debug("> system conf is already up-to-date")
os.remove(pending_path)
continue
+
else:
logger.debug("> system conf has been manually modified")
if system_hash == new_hash:
@@ -440,13 +546,14 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Check for service conf changes
if not succeed_regen and not failed_regen:
- logger.info(m18n.n('service_conf_up_to_date', service=service))
+ logger.debug(m18n.n('service_conf_up_to_date', service=service))
continue
elif not failed_regen:
logger.success(m18n.n(
- 'service_conf_updated' if not dry_run else \
- 'service_conf_would_be_updated',
+ 'service_conf_updated' if not dry_run else
+ 'service_conf_would_be_updated',
service=service))
+
if succeed_regen and not dry_run:
_update_conf_hashes(service, conf_hashes)
@@ -461,16 +568,20 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
return result
# Execute hooks for post-regen
- post_args = ['post',] + common_args
+ post_args = ['post', ] + common_args
+
def _pre_call(name, priority, path, args):
# append coma-separated applied changes for the service
if name in result and result[name]['applied']:
regen_conf_files = ','.join(result[name]['applied'].keys())
else:
regen_conf_files = ''
- return post_args + [regen_conf_files,]
+ return post_args + [regen_conf_files, ]
+
hook_callback('conf_regen', names, pre_callback=_pre_call)
+ operation_logger.success()
+
return result
@@ -483,27 +594,79 @@ def _run_service_command(action, service):
service -- Service name
"""
- if service not in _get_services().keys():
+ services = _get_services()
+ if service not in services.keys():
raise MoulinetteError(errno.EINVAL, m18n.n('service_unknown', service=service))
- cmd = None
- if action in ['start', 'stop', 'restart', 'reload']:
- cmd = 'service %s %s' % (service, action)
- elif action in ['enable', 'disable']:
- arg = 'defaults' if action == 'enable' else 'remove'
- cmd = 'update-rc.d %s %s' % (service, arg)
- else:
- raise ValueError("Unknown action '%s'" % action)
+ possible_actions = ['start', 'stop', 'restart', 'reload', 'enable', 'disable']
+ if action not in possible_actions:
+ raise ValueError("Unknown action '%s', available actions are: %s" % (action, ", ".join(possible_actions)))
+
+ cmd = 'systemctl %s %s' % (action, service)
+
+ need_lock = services[service].get('need_lock', False) \
+ and action in ['start', 'stop', 'restart', 'reload']
try:
- ret = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
+ # Launch the command
+ logger.debug("Running '%s'" % cmd)
+ p = subprocess.Popen(cmd.split(), stderr=subprocess.STDOUT)
+ # If this command needs a lock (because the service uses yunohost
+ # commands inside), find the PID and add a lock for it
+ if need_lock:
+ PID = _give_lock(action, service, p)
+ # Wait for the command to complete
+ p.communicate()
+
except subprocess.CalledProcessError as e:
# TODO: Log output?
logger.warning(m18n.n('service_cmd_exec_failed', command=' '.join(e.cmd)))
return False
+
+ finally:
+ # Remove the lock if one was given
+ if need_lock and PID != 0:
+ _remove_lock(PID)
+
return True
+def _give_lock(action, service, p):
+
+ # Depending of the action, systemctl calls the PID differently :/
+ if action == "start" or action == "restart":
+ systemctl_PID_name = "MainPID"
+ else:
+ systemctl_PID_name = "ControlPID"
+
+ cmd_get_son_PID ="systemctl show %s -p %s" % (service, systemctl_PID_name)
+ son_PID = 0
+ # As long as we did not found the PID and that the command is still running
+ while son_PID == 0 and p.poll() == None:
+ # Call systemctl to get the PID
+ # Output of the command is e.g. ControlPID=1234
+ son_PID = subprocess.check_output(cmd_get_son_PID.split()) \
+ .strip().split("=")[1]
+ son_PID = int(son_PID)
+ time.sleep(1)
+
+ # If we found a PID
+ if son_PID != 0:
+ # Append the PID to the lock file
+ logger.debug("Giving a lock to PID %s for service %s !"
+ % (str(son_PID), service))
+ filesystem.append_to_file(MOULINETTE_LOCK, "\n%s" % str(son_PID))
+
+ return son_PID
+
+def _remove_lock(PID_to_remove):
+ # FIXME ironically not concurrency safe because it's not atomic...
+
+ PIDs = filesystem.read_file(MOULINETTE_LOCK).split("\n")
+ PIDs_to_keep = [ PID for PID in PIDs if int(PID) != PID_to_remove ]
+ filesystem.write_to_file(MOULINETTE_LOCK, '\n'.join(PIDs_to_keep))
+
+
def _get_services():
"""
Get a dict of managed services with their parameters
@@ -515,6 +678,12 @@ def _get_services():
except:
return {}
else:
+ # some services are marked as None to remove them from YunoHost
+ # filter this
+ for key, value in services.items():
+ if value is None:
+ del services[key]
+
return services
@@ -526,38 +695,87 @@ def _save_services(services):
services -- A dict of managed services with their parameters
"""
- # TODO: Save to custom services.yml
- with open('/etc/yunohost/services.yml', 'w') as f:
- yaml.safe_dump(services, f, default_flow_style=False)
+ try:
+ with open('/etc/yunohost/services.yml', 'w') as f:
+ yaml.safe_dump(services, f, default_flow_style=False)
+ except Exception as e:
+ logger.warning('Error while saving services, exception: %s', e, exc_info=1)
+ raise
-def _tail(file, n, offset=None):
+def _tail(file, n):
"""
Reads a n lines from f with an offset of offset lines. The return
value is a tuple in the form ``(lines, has_more)`` where `has_more` is
an indicator that is `True` if there are more lines in the file.
+ This function works even with splitted logs (gz compression, log rotate...)
"""
avg_line_length = 74
- to_read = n + (offset or 0)
+ to_read = n
try:
- with open(file, 'r') as f:
- while 1:
+ if file.endswith(".gz"):
+ import gzip
+ f = gzip.open(file)
+ lines = f.read().splitlines()
+ else:
+ f = open(file)
+ pos = 1
+ lines = []
+ while len(lines) < to_read and pos > 0:
try:
f.seek(-(avg_line_length * to_read), 2)
except IOError:
# woops. apparently file is smaller than what we want
# to step back, go to the beginning instead
f.seek(0)
+
pos = f.tell()
lines = f.read().splitlines()
- if len(lines) >= to_read or pos == 0:
- return lines[-to_read:offset and -offset or None]
+
+ if len(lines) >= to_read:
+ return lines[-to_read:]
+
avg_line_length *= 1.3
+ f.close()
- except IOError: return []
+ except IOError as e:
+ logger.warning("Error while tailing file '%s': %s", file, e, exc_info=1)
+ return []
+ if len(lines) < to_read:
+ previous_log_file = _find_previous_log_file(file)
+ if previous_log_file is not None:
+ lines = _tail(previous_log_file, to_read - len(lines)) + lines
+
+ return lines
+
+
+def _find_previous_log_file(file):
+ """
+ Find the previous log file
+ """
+ import re
+
+ splitext = os.path.splitext(file)
+ if splitext[1] == '.gz':
+ file = splitext[0]
+ splitext = os.path.splitext(file)
+ ext = splitext[1]
+ i = re.findall(r'\.(\d+)', ext)
+ i = int(i[0]) + 1 if len(i) > 0 else 1
+
+ previous_file = file if i == 1 else splitext[0]
+ previous_file = previous_file + '.%d' % (i)
+ if os.path.exists(previous_file):
+ return previous_file
+
+ previous_file = previous_file + ".gz"
+ if os.path.exists(previous_file):
+ return previous_file
+
+ return None
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
"""Compare two files and return the differences
@@ -567,36 +785,50 @@ def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
header can also be removed if skip_header is True.
"""
- contents = [[], []]
- for i, path in enumerate((orig_file, new_file)):
- try:
- with open(path, 'r') as f:
- contents[i] = f.readlines()
- except IOError:
- pass
+
+ if os.path.exists(orig_file):
+ with open(orig_file, 'r') as orig_file:
+ orig_file = orig_file.readlines()
+ else:
+ orig_file = []
+
+ if os.path.exists(new_file):
+ with open(new_file, 'r') as new_file:
+ new_file = new_file.readlines()
+ else:
+ new_file = []
# Compare files and format output
- diff = unified_diff(contents[0], contents[1])
+ diff = unified_diff(orig_file, new_file)
+
if skip_header:
- for i in range(2):
- try:
- next(diff)
- except:
- break
+ try:
+ next(diff)
+ next(diff)
+ except:
+ pass
+
if as_string:
- result = ''.join(line for line in diff)
- return result.rstrip()
+ return ''.join(diff).rstrip()
+
return diff
def _calculate_hash(path):
"""Calculate the MD5 hash of a file"""
+
+ if not os.path.exists(path):
+ return None
+
hasher = hashlib.md5()
+
try:
with open(path, 'rb') as f:
hasher.update(f.read())
return hasher.hexdigest()
- except IOError:
+
+ except IOError as e:
+ logger.warning("Error while calculating file '%s' hash: %s", path, e, exc_info=1)
return None
@@ -612,37 +844,52 @@ def _get_pending_conf(services=[]):
"""
result = {}
- if not os.path.isdir(pending_conf_dir):
+
+ if not os.path.isdir(PENDING_CONF_DIR):
return result
+
if not services:
- services = os.listdir(pending_conf_dir)
+ services = os.listdir(PENDING_CONF_DIR)
+
for name in services:
- service_pending_path = os.path.join(pending_conf_dir, name)
+ service_pending_path = os.path.join(PENDING_CONF_DIR, name)
+
if not os.path.isdir(service_pending_path):
continue
+
path_index = len(service_pending_path)
service_conf = {}
+
for root, dirs, files in os.walk(service_pending_path):
for filename in files:
pending_path = os.path.join(root, filename)
service_conf[pending_path[path_index:]] = pending_path
+
if service_conf:
result[name] = service_conf
else:
# remove empty directory
shutil.rmtree(service_pending_path, ignore_errors=True)
+
return result
def _get_conf_hashes(service):
"""Get the registered conf hashes for a service"""
- try:
- return _get_services()[service]['conffiles']
- except:
- logger.debug("unable to retrieve conf hashes for %s",
- service, exc_info=1)
+
+ services = _get_services()
+
+ if service not in services:
+ logger.debug("Service %s is not in services.yml yet.", service)
return {}
+ elif services[service] is None or 'conffiles' not in services[service]:
+ logger.debug("No configuration files for service %s.", service)
+ return {}
+
+ else:
+ return services[service]['conffiles']
+
def _update_conf_hashes(service, hashes):
"""Update the registered conf hashes for a service"""
@@ -650,6 +897,11 @@ def _update_conf_hashes(service, hashes):
service, hashes)
services = _get_services()
service_conf = services.get(service, {})
+
+ # Handle the case where services[service] is set to null in the yaml
+ if service_conf is None:
+ service_conf = {}
+
service_conf['conffiles'] = hashes
services[service] = service_conf
_save_services(services)
@@ -664,34 +916,44 @@ def _process_regen_conf(system_conf, new_conf=None, save=True):
"""
if save:
- backup_path = os.path.join(backup_conf_dir, '{0}-{1}'.format(
+ backup_path = os.path.join(BACKUP_CONF_DIR, '{0}-{1}'.format(
system_conf.lstrip('/'), time.strftime("%Y%m%d.%H%M%S")))
backup_dir = os.path.dirname(backup_path)
+
if not os.path.isdir(backup_dir):
filesystem.mkdir(backup_dir, 0755, True)
+
shutil.copy2(system_conf, backup_path)
- logger.info(m18n.n('service_conf_file_backed_up',
+ logger.debug(m18n.n('service_conf_file_backed_up',
conf=system_conf, backup=backup_path))
+
try:
if not new_conf:
os.remove(system_conf)
- logger.info(m18n.n('service_conf_file_removed',
+ logger.debug(m18n.n('service_conf_file_removed',
conf=system_conf))
else:
system_dir = os.path.dirname(system_conf)
+
if not os.path.isdir(system_dir):
filesystem.mkdir(system_dir, 0755, True)
+
shutil.copyfile(new_conf, system_conf)
- logger.info(m18n.n('service_conf_file_updated',
- conf=system_conf))
- except:
+ logger.debug(m18n.n('service_conf_file_updated',
+ conf=system_conf))
+ except Exception as e:
+ logger.warning("Exception while trying to regenerate conf '%s': %s", system_conf, e, exc_info=1)
if not new_conf and os.path.exists(system_conf):
logger.warning(m18n.n('service_conf_file_remove_failed',
conf=system_conf),
exc_info=1)
return False
+
elif new_conf:
try:
+ # From documentation:
+ # Raise an exception if an os.stat() call on either pathname fails.
+ # (os.stats returns a series of information from a file like type, size...)
copy_succeed = os.path.samefile(system_conf, new_conf)
except:
copy_succeed = False
@@ -701,4 +963,45 @@ def _process_regen_conf(system_conf, new_conf=None, save=True):
conf=system_conf, new=new_conf),
exc_info=1)
return False
+
return True
+
+
+def manually_modified_files():
+
+ # We do this to have --quiet, i.e. don't throw a whole bunch of logs
+ # just to fetch this...
+ # Might be able to optimize this by looking at what service_regenconf does
+ # and only do the part that checks file hashes...
+ cmd = "yunohost service regen-conf --dry-run --output-as json --quiet"
+ j = json.loads(subprocess.check_output(cmd.split()))
+
+ # j is something like :
+ # {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}}
+
+ output = []
+ for app, actions in j.items():
+ for action, files in actions.items():
+ for filename, infos in files.items():
+ if infos["status"] == "modified":
+ output.append(filename)
+
+ return output
+
+
+def _get_journalctl_logs(service):
+ try:
+ return subprocess.check_output("journalctl -xn -u %s" % service, shell=True)
+ except:
+ import traceback
+ return "error while get services logs from journalctl:\n%s" % traceback.format_exc()
+
+
+def manually_modified_files_compared_to_debian_default():
+
+ # from https://serverfault.com/a/90401
+ r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
+ | awk 'OFS=\" \"{print $2,$1}' \
+ | md5sum -c 2>/dev/null \
+ | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
+ return r.strip().split("\n")
diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py
new file mode 100644
index 000000000..aba6e32b3
--- /dev/null
+++ b/src/yunohost/settings.py
@@ -0,0 +1,237 @@
+import os
+import json
+import errno
+
+from datetime import datetime
+from collections import OrderedDict
+
+from moulinette import m18n
+from moulinette.core import MoulinetteError
+from moulinette.utils.log import getActionLogger
+
+logger = getActionLogger('yunohost.settings')
+
+SETTINGS_PATH = "/etc/yunohost/settings.json"
+SETTINGS_PATH_OTHER_LOCATION = "/etc/yunohost/settings-%s.json"
+
+# a settings entry is in the form of:
+# namespace.subnamespace.name: {type, value, default, description, [choices]}
+# choices is only for enum
+# the keyname can have as many subnamespace as needed but should have at least
+# one level of namespace
+
+# description is implied from the translated strings
+# the key is "global_settings_setting_%s" % key.replace(".", "_")
+
+# type can be:
+# * bool
+# * int
+# * string
+# * enum (in form a python list)
+
+# we don't store the value in default options
+DEFAULTS = OrderedDict([
+ ("example.bool", {"type": "bool", "default": True}),
+ ("example.int", {"type": "int", "default": 42}),
+ ("example.string", {"type": "string", "default": "yolo swag"}),
+ ("example.enum", {"type": "enum", "default": "a", "choices": ["a", "b", "c"]}),
+])
+
+
+def settings_get(key, full=False):
+ """
+ Get an entry value in the settings
+
+ Keyword argument:
+ key -- Settings key
+
+ """
+ settings = _get_settings()
+
+ if key not in settings:
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_key_doesnt_exists', settings_key=key))
+
+ if full:
+ return settings[key]
+
+ return settings[key]['value']
+
+
+def settings_list():
+ """
+ List all entries of the settings
+
+ """
+ return _get_settings()
+
+
+def settings_set(key, value):
+ """
+ Set an entry value in the settings
+
+ Keyword argument:
+ key -- Settings key
+ value -- New value
+
+ """
+ settings = _get_settings()
+
+ if key not in settings:
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_key_doesnt_exists', settings_key=key))
+
+ key_type = settings[key]["type"]
+
+ if key_type == "bool":
+ if not isinstance(value, bool):
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_bad_type_for_setting', setting=key,
+ received_type=type(value).__name__, expected_type=key_type))
+ elif key_type == "int":
+ if not isinstance(value, int) or isinstance(value, bool):
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_bad_type_for_setting', setting=key,
+ received_type=type(value).__name__, expected_type=key_type))
+ elif key_type == "string":
+ if not isinstance(value, basestring):
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_bad_type_for_setting', setting=key,
+ received_type=type(value).__name__, expected_type=key_type))
+ elif key_type == "enum":
+ if value not in settings[key]["choices"]:
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_bad_choice_for_enum', setting=key,
+ received_type=type(value).__name__,
+ expected_type=", ".join(settings[key]["choices"])))
+ else:
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_unknown_type', setting=key,
+ unknown_type=key_type))
+
+ settings[key]["value"] = value
+
+ _save_settings(settings)
+
+
+def settings_reset(key):
+ """
+ Set an entry value to its default one
+
+ Keyword argument:
+ key -- Settings key
+
+ """
+ settings = _get_settings()
+
+ if key not in settings:
+ raise MoulinetteError(errno.EINVAL, m18n.n(
+ 'global_settings_key_doesnt_exists', settings_key=key))
+
+ settings[key]["value"] = settings[key]["default"]
+ _save_settings(settings)
+
+
+def settings_reset_all():
+ """
+ Reset all settings to their default value
+
+ Keyword argument:
+ yes -- Yes I'm sure I want to do that
+
+ """
+ settings = _get_settings()
+
+ # For now on, we backup the previous settings in case of but we don't have
+ # any mecanism to take advantage of those backups. It could be a nice
+ # addition but we'll see if this is a common need.
+ # Another solution would be to use etckeeper and integrate those
+ # modification inside of it and take advantage of its git history
+ old_settings_backup_path = SETTINGS_PATH_OTHER_LOCATION % datetime.now().strftime("%F_%X")
+ _save_settings(settings, location=old_settings_backup_path)
+
+ for value in settings.values():
+ value["value"] = value["default"]
+
+ _save_settings(settings)
+
+ return {
+ "old_settings_backup_path": old_settings_backup_path,
+ "message": m18n.n("global_settings_reset_success", path=old_settings_backup_path)
+ }
+
+
+def _get_settings():
+ settings = {}
+
+ for key, value in DEFAULTS.copy().items():
+ settings[key] = value
+ settings[key]["value"] = value["default"]
+ settings[key]["description"] = m18n.n("global_settings_setting_%s" % key.replace(".", "_"))
+
+ if not os.path.exists(SETTINGS_PATH):
+ return settings
+
+ # we have a very strict policy on only allowing settings that we know in
+ # the OrderedDict DEFAULTS
+ # For various reason, while reading the local settings we might encounter
+ # settings that aren't in DEFAULTS, those can come from settings key that
+ # we have removed, errors or the user trying to modify
+ # /etc/yunohost/settings.json
+ # To avoid to simply overwrite them, we store them in
+ # /etc/yunohost/settings-unknown.json in case of
+ unknown_settings = {}
+ unknown_settings_path = SETTINGS_PATH_OTHER_LOCATION % "unknown"
+
+ if os.path.exists(unknown_settings_path):
+ try:
+ unknown_settings = json.load(open(unknown_settings_path, "r"))
+ except Exception as e:
+ logger.warning("Error while loading unknown settings %s" % e)
+
+ try:
+ with open(SETTINGS_PATH) as settings_fd:
+ local_settings = json.load(settings_fd)
+
+ for key, value in local_settings.items():
+ if key in settings:
+ settings[key] = value
+ settings[key]["description"] = m18n.n("global_settings_setting_%s" % key.replace(".", "_"))
+ else:
+ logger.warning(m18n.n('global_settings_unknown_setting_from_settings_file',
+ setting_key=key))
+ unknown_settings[key] = value
+ except Exception as e:
+ raise MoulinetteError(errno.EIO, m18n.n('global_settings_cant_open_settings', reason=e),
+ exc_info=1)
+
+ if unknown_settings:
+ try:
+ _save_settings(unknown_settings, location=unknown_settings_path)
+ except Exception as e:
+ logger.warning("Failed to save unknown settings (because %s), aborting." % e)
+
+ return settings
+
+
+def _save_settings(settings, location=SETTINGS_PATH):
+ settings_without_description = {}
+ for key, value in settings.items():
+ settings_without_description[key] = value
+ if "description" in value:
+ del settings_without_description[key]["description"]
+
+ try:
+ result = json.dumps(settings_without_description, indent=4)
+ except Exception as e:
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('global_settings_cant_serialize_settings', reason=e),
+ exc_info=1)
+
+ try:
+ with open(location, "w") as settings_fd:
+ settings_fd.write(result)
+ except Exception as e:
+ raise MoulinetteError(errno.EIO,
+ m18n.n('global_settings_cant_write_settings', reason=e),
+ exc_info=1)
diff --git a/src/yunohost/ssh.py b/src/yunohost/ssh.py
new file mode 100644
index 000000000..5ddebfc2f
--- /dev/null
+++ b/src/yunohost/ssh.py
@@ -0,0 +1,203 @@
+# encoding: utf-8
+
+import re
+import os
+import errno
+import pwd
+import subprocess
+
+from moulinette import m18n
+from moulinette.core import MoulinetteError
+from moulinette.utils.filesystem import read_file, write_to_file, chown, chmod, mkdir
+
+SSHD_CONFIG_PATH = "/etc/ssh/sshd_config"
+
+
+def user_ssh_allow(auth, username):
+ """
+ Allow YunoHost user connect as ssh.
+
+ Keyword argument:
+ username -- User username
+ """
+ # TODO it would be good to support different kind of shells
+
+ if not _get_user_for_ssh(auth, username):
+ raise MoulinetteError(errno.EINVAL, m18n.n('user_unknown', user=username))
+
+ auth.update('uid=%s,ou=users' % username, {'loginShell': '/bin/bash'})
+
+ # Somehow this is needed otherwise the PAM thing doesn't forget about the
+ # old loginShell value ?
+ subprocess.call(['nscd', '-i', 'passwd'])
+
+
+def user_ssh_disallow(auth, username):
+ """
+ Disallow YunoHost user connect as ssh.
+
+ Keyword argument:
+ username -- User username
+ """
+ # TODO it would be good to support different kind of shells
+
+ if not _get_user_for_ssh(auth, username):
+ raise MoulinetteError(errno.EINVAL, m18n.n('user_unknown', user=username))
+
+ auth.update('uid=%s,ou=users' % username, {'loginShell': '/bin/false'})
+
+ # Somehow this is needed otherwise the PAM thing doesn't forget about the
+ # old loginShell value ?
+ subprocess.call(['nscd', '-i', 'passwd'])
+
+
+def user_ssh_list_keys(auth, username):
+ user = _get_user_for_ssh(auth, username, ["homeDirectory"])
+ if not user:
+ raise Exception("User with username '%s' doesn't exists" % username)
+
+ authorized_keys_file = os.path.join(user["homeDirectory"][0], ".ssh", "authorized_keys")
+
+ if not os.path.exists(authorized_keys_file):
+ return {"keys": []}
+
+ keys = []
+ last_comment = ""
+ for line in read_file(authorized_keys_file).split("\n"):
+ # empty line
+ if not line.strip():
+ continue
+
+ if line.lstrip().startswith("#"):
+ last_comment = line.lstrip().lstrip("#").strip()
+ continue
+
+ # assuming a key per non empty line
+ key = line.strip()
+ keys.append({
+ "key": key,
+ "name": last_comment,
+ })
+
+ last_comment = ""
+
+ return {"keys": keys}
+
+
+def user_ssh_add_key(auth, username, key, comment):
+ user = _get_user_for_ssh(auth, username, ["homeDirectory", "uid"])
+ if not user:
+ raise Exception("User with username '%s' doesn't exists" % username)
+
+ authorized_keys_file = os.path.join(user["homeDirectory"][0], ".ssh", "authorized_keys")
+
+ if not os.path.exists(authorized_keys_file):
+ # ensure ".ssh" exists
+ mkdir(os.path.join(user["homeDirectory"][0], ".ssh"),
+ force=True, parents=True, uid=user["uid"][0])
+
+ # create empty file to set good permissions
+ write_to_file(authorized_keys_file, "")
+ chown(authorized_keys_file, uid=user["uid"][0])
+ chmod(authorized_keys_file, 0600)
+
+ authorized_keys_content = read_file(authorized_keys_file)
+
+ authorized_keys_content += "\n"
+ authorized_keys_content += "\n"
+
+ if comment and comment.strip():
+ if not comment.lstrip().startswith("#"):
+ comment = "# " + comment
+ authorized_keys_content += comment.replace("\n", " ").strip()
+ authorized_keys_content += "\n"
+
+ authorized_keys_content += key.strip()
+ authorized_keys_content += "\n"
+
+ write_to_file(authorized_keys_file, authorized_keys_content)
+
+
+def user_ssh_remove_key(auth, username, key):
+ user = _get_user_for_ssh(auth, username, ["homeDirectory", "uid"])
+ if not user:
+ raise Exception("User with username '%s' doesn't exists" % username)
+
+ authorized_keys_file = os.path.join(user["homeDirectory"][0], ".ssh", "authorized_keys")
+
+ if not os.path.exists(authorized_keys_file):
+ raise Exception("this key doesn't exists ({} dosesn't exists)".format(authorized_keys_file))
+
+ authorized_keys_content = read_file(authorized_keys_file)
+
+ if key not in authorized_keys_content:
+ raise Exception("Key '{}' is not present in authorized_keys".format(key))
+
+ # don't delete the previous comment because we can't verify if it's legit
+
+ # this regex approach failed for some reasons and I don't know why :(
+ # authorized_keys_content = re.sub("{} *\n?".format(key),
+ # "",
+ # authorized_keys_content,
+ # flags=re.MULTILINE)
+
+ authorized_keys_content = authorized_keys_content.replace(key, "")
+
+ write_to_file(authorized_keys_file, authorized_keys_content)
+
+#
+# Helpers
+#
+
+
+def _get_user_for_ssh(auth, username, attrs=None):
+ def ssh_root_login_status(auth):
+ # XXX temporary placed here for when the ssh_root commands are integrated
+ # extracted from https://github.com/YunoHost/yunohost/pull/345
+ # XXX should we support all the options?
+ # this is the content of "man sshd_config"
+ # PermitRootLogin
+ # Specifies whether root can log in using ssh(1). The argument must be
+ # “yes”, “without-password”, “forced-commands-only”, or “no”. The
+ # default is “yes”.
+ sshd_config_content = read_file(SSHD_CONFIG_PATH)
+
+ if re.search("^ *PermitRootLogin +(no|forced-commands-only) *$",
+ sshd_config_content, re.MULTILINE):
+ return {"PermitRootLogin": False}
+
+ return {"PermitRootLogin": True}
+
+ if username == "root":
+ root_unix = pwd.getpwnam("root")
+ return {
+ 'username': 'root',
+ 'fullname': '',
+ 'mail': '',
+ 'ssh_allowed': ssh_root_login_status(auth)["PermitRootLogin"],
+ 'shell': root_unix.pw_shell,
+ 'home_path': root_unix.pw_dir,
+ }
+
+ if username == "admin":
+ admin_unix = pwd.getpwnam("admin")
+ return {
+ 'username': 'admin',
+ 'fullname': '',
+ 'mail': '',
+ 'ssh_allowed': admin_unix.pw_shell.strip() != "/bin/false",
+ 'shell': admin_unix.pw_shell,
+ 'home_path': admin_unix.pw_dir,
+ }
+
+ # TODO escape input using https://www.python-ldap.org/doc/html/ldap-filter.html
+ user = auth.search('ou=users,dc=yunohost,dc=org',
+ '(&(objectclass=person)(uid=%s))' % username,
+ attrs)
+
+ assert len(user) in (0, 1)
+
+ if not user:
+ return None
+
+ return user[0]
diff --git a/src/yunohost/tests/__init__.py b/src/yunohost/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py
new file mode 100644
index 000000000..6958ae679
--- /dev/null
+++ b/src/yunohost/tests/conftest.py
@@ -0,0 +1,96 @@
+import sys
+import moulinette
+
+sys.path.append("..")
+
+
+def pytest_addoption(parser):
+ parser.addoption("--yunodebug", action="store_true", default=False)
+
+###############################################################################
+# Tweak translator to raise exceptions if string keys are not defined #
+###############################################################################
+
+
+old_translate = moulinette.core.Translator.translate
+def new_translate(self, key, *args, **kwargs):
+
+ if key not in self._translations[self.default_locale].keys():
+ raise KeyError("Unable to retrieve key %s for default locale !" % key)
+
+ return old_translate(self, key, *args, **kwargs)
+moulinette.core.Translator.translate = new_translate
+
+def new_m18nn(self, key, *args, **kwargs):
+ return self._namespaces[self._current_namespace].translate(key, *args, **kwargs)
+
+moulinette.core.Moulinette18n.n = new_m18nn
+
+###############################################################################
+# Init the moulinette to have the cli loggers stuff #
+###############################################################################
+
+
+def pytest_cmdline_main(config):
+ """Configure logging and initialize the moulinette"""
+ # Define loggers handlers
+ handlers = set(['tty'])
+ root_handlers = set(handlers)
+
+ # Define loggers level
+ level = 'INFO'
+ if config.option.yunodebug:
+ tty_level = 'DEBUG'
+ else:
+ tty_level = 'SUCCESS'
+
+ # Custom logging configuration
+ logging = {
+ 'version': 1,
+ 'disable_existing_loggers': True,
+ 'formatters': {
+ 'tty-debug': {
+ 'format': '%(relativeCreated)-4d %(fmessage)s'
+ },
+ 'precise': {
+ 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s'
+ },
+ },
+ 'filters': {
+ 'action': {
+ '()': 'moulinette.utils.log.ActionFilter',
+ },
+ },
+ 'handlers': {
+ 'tty': {
+ 'level': tty_level,
+ 'class': 'moulinette.interfaces.cli.TTYHandler',
+ 'formatter': '',
+ },
+ },
+ 'loggers': {
+ 'yunohost': {
+ 'level': level,
+ 'handlers': handlers,
+ 'propagate': False,
+ },
+ 'moulinette': {
+ 'level': level,
+ 'handlers': [],
+ 'propagate': True,
+ },
+ 'moulinette.interface': {
+ 'level': level,
+ 'handlers': handlers,
+ 'propagate': False,
+ },
+ },
+ 'root': {
+ 'level': level,
+ 'handlers': root_handlers,
+ },
+ }
+
+ # Initialize moulinette
+ moulinette.init(logging_config=logging, _from_source=False)
+ moulinette.m18n.load_namespace('yunohost')
diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py
new file mode 100644
index 000000000..6b7141f4a
--- /dev/null
+++ b/src/yunohost/tests/test_appslist.py
@@ -0,0 +1,389 @@
+import os
+import pytest
+import requests
+import requests_mock
+import glob
+import time
+
+from moulinette.core import MoulinetteError
+
+from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist
+
+URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json"
+REPO_PATH = '/var/cache/yunohost/repo'
+APPSLISTS_JSON = '/etc/yunohost/appslists.json'
+
+
+def setup_function(function):
+
+ # Clear all appslist
+ files = glob.glob(REPO_PATH+"/*")
+ for f in files:
+ os.remove(f)
+
+ # Clear appslist crons
+ files = glob.glob("/etc/cron.d/yunohost-applist-*")
+ for f in files:
+ os.remove(f)
+
+ if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
+ os.remove("/etc/cron.daily/yunohost-fetch-appslists")
+
+ if os.path.exists(APPSLISTS_JSON):
+ os.remove(APPSLISTS_JSON)
+
+
+def teardown_function(function):
+ pass
+
+
+def cron_job_is_there():
+ r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists")
+ return r == 0
+
+
+###############################################################################
+# Test listing of appslists and registering of appslists #
+###############################################################################
+
+
+def test_appslist_list_empty():
+ """
+ Calling app_listlists() with no registered list should return empty dict
+ """
+
+ assert app_listlists() == {}
+
+
+def test_appslist_list_register():
+ """
+ Register a new list
+ """
+
+ # Assume we're starting with an empty app list
+ assert app_listlists() == {}
+
+ # Register a new dummy list
+ _register_new_appslist("https://lol.com/appslist.json", "dummy")
+
+ appslist_dict = app_listlists()
+ assert "dummy" in appslist_dict.keys()
+ assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json"
+
+ assert cron_job_is_there()
+
+
+def test_appslist_list_register_conflict_name():
+ """
+ Attempt to register a new list with conflicting name
+ """
+
+ _register_new_appslist("https://lol.com/appslist.json", "dummy")
+ with pytest.raises(MoulinetteError):
+ _register_new_appslist("https://lol.com/appslist2.json", "dummy")
+
+ appslist_dict = app_listlists()
+
+ assert "dummy" in appslist_dict.keys()
+ assert "dummy2" not in appslist_dict.keys()
+
+
+def test_appslist_list_register_conflict_url():
+ """
+ Attempt to register a new list with conflicting url
+ """
+
+ _register_new_appslist("https://lol.com/appslist.json", "dummy")
+ with pytest.raises(MoulinetteError):
+ _register_new_appslist("https://lol.com/appslist.json", "plopette")
+
+ appslist_dict = app_listlists()
+
+ assert "dummy" in appslist_dict.keys()
+ assert "plopette" not in appslist_dict.keys()
+
+
+###############################################################################
+# Test fetching of appslists #
+###############################################################################
+
+
+def test_appslist_fetch():
+ """
+ Do a fetchlist and test the .json got updated.
+ """
+ assert app_listlists() == {}
+
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+ with requests_mock.Mocker() as m:
+
+ # Mock the server response with a valid (well, empty, yep) json
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
+
+ official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+ app_fetchlist()
+ new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+
+ assert new_official_lastUpdate > official_lastUpdate
+
+
+def test_appslist_fetch_single_appslist():
+ """
+ Register several lists but only fetch one. Check only one got updated.
+ """
+
+ assert app_listlists() == {}
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+ _register_new_appslist("https://lol.com/appslist.json", "dummy")
+
+ time.sleep(1)
+
+ with requests_mock.Mocker() as m:
+
+ # Mock the server response with a valid (well, empty, yep) json
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
+
+ official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+ dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
+ app_fetchlist(name="yunohost")
+ new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+ new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
+
+ assert new_official_lastUpdate > official_lastUpdate
+ assert new_dummy_lastUpdate == dummy_lastUpdate
+
+
+def test_appslist_fetch_unknownlist():
+ """
+ Attempt to fetch an unknown list
+ """
+
+ assert app_listlists() == {}
+
+ with pytest.raises(MoulinetteError):
+ app_fetchlist(name="swag")
+
+
+def test_appslist_fetch_url_but_no_name():
+ """
+ Do a fetchlist with url given, but no name given
+ """
+
+ with pytest.raises(MoulinetteError):
+ app_fetchlist(url=URL_OFFICIAL_APP_LIST)
+
+
+def test_appslist_fetch_badurl():
+ """
+ Do a fetchlist with a bad url
+ """
+
+ app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop")
+
+
+def test_appslist_fetch_badfile():
+ """
+ Do a fetchlist and mock a response with a bad json
+ """
+ assert app_listlists() == {}
+
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+ with requests_mock.Mocker() as m:
+
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }')
+
+ app_fetchlist()
+
+
+def test_appslist_fetch_404():
+ """
+ Do a fetchlist and mock a 404 response
+ """
+ assert app_listlists() == {}
+
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+ with requests_mock.Mocker() as m:
+
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404)
+
+ app_fetchlist()
+
+
+def test_appslist_fetch_sslerror():
+ """
+ Do a fetchlist and mock an SSL error
+ """
+ assert app_listlists() == {}
+
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+ with requests_mock.Mocker() as m:
+
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST,
+ exc=requests.exceptions.SSLError)
+
+ app_fetchlist()
+
+
+def test_appslist_fetch_timeout():
+ """
+ Do a fetchlist and mock a timeout
+ """
+ assert app_listlists() == {}
+
+ _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+ with requests_mock.Mocker() as m:
+
+ m.register_uri("GET", URL_OFFICIAL_APP_LIST,
+ exc=requests.exceptions.ConnectTimeout)
+
+ app_fetchlist()
+
+
+###############################################################################
+# Test remove of appslist #
+###############################################################################
+
+
+def test_appslist_remove():
+ """
+ Register a new appslist, then remove it
+ """
+
+ # Assume we're starting with an empty app list
+ assert app_listlists() == {}
+
+ # Register a new dummy list
+ _register_new_appslist("https://lol.com/appslist.json", "dummy")
+ app_removelist("dummy")
+
+ # Should end up with no list registered
+ assert app_listlists() == {}
+
+
+def test_appslist_remove_unknown():
+ """
+ Attempt to remove an unknown list
+ """
+
+ with pytest.raises(MoulinetteError):
+ app_removelist("dummy")
+
+
+###############################################################################
+# Test migration from legacy appslist system #
+###############################################################################
+
+
+def add_legacy_cron(name, url):
+ with open("/etc/cron.d/yunohost-applist-%s" % name, "w") as f:
+ f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
+
+
+def test_appslist_check_using_legacy_system_testFalse():
+ """
+ If no legacy cron job is there, the check should return False
+ """
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ assert _using_legacy_appslist_system() is False
+
+
+def test_appslist_check_using_legacy_system_testTrue():
+ """
+ If there's a legacy cron job, the check should return True
+ """
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+ assert _using_legacy_appslist_system() is True
+
+
+def test_appslist_system_migration():
+ """
+ Test that legacy cron jobs get migrated correctly when calling app_listlists
+ """
+
+ # Start with no legacy cron, no appslist registered
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ assert app_listlists() == {}
+ assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+ # Add a few legacy crons
+ add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+ add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json")
+
+ # Migrate
+ assert _using_legacy_appslist_system() is True
+ _migrate_appslist_system()
+ assert _using_legacy_appslist_system() is False
+
+ # No legacy cron job should remain
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+
+ # Check they are in app_listlists anyway
+ appslist_dict = app_listlists()
+ assert "yunohost" in appslist_dict.keys()
+ assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json"
+ assert "dummy" in appslist_dict.keys()
+ assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json"
+
+ assert cron_job_is_there()
+
+
+def test_appslist_system_migration_badcron():
+ """
+ Test the migration on a bad legacy cron (no url found inside cron job)
+ """
+
+ # Start with no legacy cron, no appslist registered
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ assert app_listlists() == {}
+ assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+ # Add a "bad" legacy cron
+ add_legacy_cron("wtflist", "ftp://the.fuck.is.this")
+
+ # Migrate
+ assert _using_legacy_appslist_system() is True
+ _migrate_appslist_system()
+ assert _using_legacy_appslist_system() is False
+
+ # No legacy cron should remain, but it should be backuped in /etc/yunohost
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp")
+
+ # Appslist should still be empty
+ assert app_listlists() == {}
+
+
+def test_appslist_system_migration_conflict():
+ """
+ Test migration of conflicting cron job (in terms of url)
+ """
+
+ # Start with no legacy cron, no appslist registered
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+ assert app_listlists() == {}
+ assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+ # Add a few legacy crons
+ add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+ add_legacy_cron("dummy", "https://app.yunohost.org/official.json")
+
+ # Migrate
+ assert _using_legacy_appslist_system() is True
+ _migrate_appslist_system()
+ assert _using_legacy_appslist_system() is False
+
+ # No legacy cron job should remain
+ assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
+
+ # Only one among "dummy" and "yunohost" should be listed
+ appslist_dict = app_listlists()
+ assert (len(appslist_dict.keys()) == 1)
+ assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys())
+
+ assert cron_job_is_there()
diff --git a/src/yunohost/tests/test_appurl.py b/src/yunohost/tests/test_appurl.py
new file mode 100644
index 000000000..dc1dbc29b
--- /dev/null
+++ b/src/yunohost/tests/test_appurl.py
@@ -0,0 +1,67 @@
+import pytest
+
+from moulinette.core import MoulinetteError, init_authenticator
+
+from yunohost.app import app_install, app_remove
+from yunohost.domain import _get_maindomain, domain_url_available, _normalize_domain_path
+
+# Instantiate LDAP Authenticator
+auth_identifier = ('ldap', 'ldap-anonymous')
+auth_parameters = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'}
+auth = init_authenticator(auth_identifier, auth_parameters)
+
+
+# Get main domain
+maindomain = _get_maindomain()
+
+
+def setup_function(function):
+
+ try:
+ app_remove(auth, "register_url_app")
+ except:
+ pass
+
+def teardown_function(function):
+
+ try:
+ app_remove(auth, "register_url_app")
+ except:
+ pass
+
+
+def test_normalize_domain_path():
+
+ assert _normalize_domain_path("https://yolo.swag/", "macnuggets") == ("yolo.swag", "/macnuggets")
+ assert _normalize_domain_path("http://yolo.swag", "/macnuggets/") == ("yolo.swag", "/macnuggets")
+ assert _normalize_domain_path("yolo.swag/", "macnuggets/") == ("yolo.swag", "/macnuggets")
+
+
+def test_urlavailable():
+
+ # Except the maindomain/macnuggets to be available
+ assert domain_url_available(auth, maindomain, "/macnuggets")
+
+ # We don't know the domain yolo.swag
+ with pytest.raises(MoulinetteError):
+ assert domain_url_available(auth, "yolo.swag", "/macnuggets")
+
+
+def test_registerurl():
+
+ app_install(auth, "./tests/apps/register_url_app_ynh",
+ args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"))
+
+ assert not domain_url_available(auth, maindomain, "/urlregisterapp")
+
+ # Try installing at same location
+ with pytest.raises(MoulinetteError):
+ app_install(auth, "./tests/apps/register_url_app_ynh",
+ args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"))
+
+
+def test_registerurl_baddomain():
+
+ with pytest.raises(MoulinetteError):
+ app_install(auth, "./tests/apps/register_url_app_ynh",
+ args="domain=%s&path=%s" % ("yolo.swag", "/urlregisterapp"))
diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py
new file mode 100644
index 000000000..1071c1642
--- /dev/null
+++ b/src/yunohost/tests/test_backuprestore.py
@@ -0,0 +1,583 @@
+import pytest
+import time
+import requests
+import os
+import shutil
+import subprocess
+from mock import ANY
+
+from moulinette import m18n
+from moulinette.core import init_authenticator
+from yunohost.app import app_install, app_remove, app_ssowatconf
+from yunohost.app import _is_installed
+from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete
+from yunohost.domain import _get_maindomain
+from moulinette.core import MoulinetteError
+
+# Get main domain
+maindomain = _get_maindomain()
+
+# Instantiate LDAP Authenticator
+AUTH_IDENTIFIER = ('ldap', 'ldap-anonymous')
+AUTH_PARAMETERS = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'}
+auth = None
+
+def setup_function(function):
+
+ print ""
+
+ global auth
+ auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS)
+
+ assert backup_test_dependencies_are_met()
+
+ clean_tmp_backup_directory()
+ reset_ssowat_conf()
+ delete_all_backups()
+ uninstall_test_apps_if_needed()
+
+ assert len(backup_list()["archives"]) == 0
+
+ markers = function.__dict__.keys()
+
+ if "with_wordpress_archive_from_2p4" in markers:
+ add_archive_wordpress_from_2p4()
+ assert len(backup_list()["archives"]) == 1
+
+ if "with_backup_legacy_app_installed" in markers:
+ assert not app_is_installed("backup_legacy_app")
+ install_app("backup_legacy_app_ynh", "/yolo")
+ assert app_is_installed("backup_legacy_app")
+
+ if "with_backup_recommended_app_installed" in markers:
+ assert not app_is_installed("backup_recommended_app")
+ install_app("backup_recommended_app_ynh", "/yolo",
+ "&helper_to_test=ynh_restore_file")
+ assert app_is_installed("backup_recommended_app")
+
+ if "with_backup_recommended_app_installed_with_ynh_restore" in markers:
+ assert not app_is_installed("backup_recommended_app")
+ install_app("backup_recommended_app_ynh", "/yolo",
+ "&helper_to_test=ynh_restore")
+ assert app_is_installed("backup_recommended_app")
+
+ if "with_system_archive_from_2p4" in markers:
+ add_archive_system_from_2p4()
+ assert len(backup_list()["archives"]) == 1
+
+
+def teardown_function(function):
+
+ print ""
+ global auth
+ auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS)
+
+ assert tmp_backup_directory_is_empty()
+
+ reset_ssowat_conf()
+ delete_all_backups()
+ uninstall_test_apps_if_needed()
+
+ markers = function.__dict__.keys()
+
+ if "clean_opt_dir" in markers:
+ shutil.rmtree("/opt/test_backup_output_directory")
+
+
+###############################################################################
+# Helpers #
+###############################################################################
+
+def app_is_installed(app):
+
+ # These are files we know should be installed by the app
+ app_files = []
+ app_files.append("/etc/nginx/conf.d/%s.d/%s.conf" % (maindomain, app))
+ app_files.append("/var/www/%s/index.html" % app)
+ app_files.append("/etc/importantfile")
+
+ return _is_installed(app) and all(os.path.exists(f) for f in app_files)
+
+
+def backup_test_dependencies_are_met():
+
+ # Dummy test apps (or backup archives)
+ assert os.path.exists("./tests/apps/backup_wordpress_from_2p4")
+ assert os.path.exists("./tests/apps/backup_legacy_app_ynh")
+ assert os.path.exists("./tests/apps/backup_recommended_app_ynh")
+
+ return True
+
+def tmp_backup_directory_is_empty():
+
+ if not os.path.exists("/home/yunohost.backup/tmp/"):
+ return True
+ else:
+ return len(os.listdir('/home/yunohost.backup/tmp/')) == 0
+
+def clean_tmp_backup_directory():
+
+ if tmp_backup_directory_is_empty():
+ return
+
+ mount_lines = subprocess.check_output("mount").split("\n")
+
+ points_to_umount = [ line.split(" ")[2]
+ for line in mount_lines
+ if len(line) >= 3
+ and line.split(" ")[2].startswith("/home/yunohost.backup/tmp") ]
+
+ for point in reversed(points_to_umount):
+ os.system("umount %s" % point)
+
+ for f in os.listdir('/home/yunohost.backup/tmp/'):
+ shutil.rmtree("/home/yunohost.backup/tmp/%s" % f)
+
+ shutil.rmtree("/home/yunohost.backup/tmp/")
+
+def reset_ssowat_conf():
+
+ # Make sure we have a ssowat
+ os.system("mkdir -p /etc/ssowat/")
+ app_ssowatconf(auth)
+
+
+def delete_all_backups():
+
+ for archive in backup_list()["archives"]:
+ backup_delete(archive)
+
+
+def uninstall_test_apps_if_needed():
+
+ if _is_installed("backup_legacy_app"):
+ app_remove(auth, "backup_legacy_app")
+
+ if _is_installed("backup_recommended_app"):
+ app_remove(auth, "backup_recommended_app")
+
+ if _is_installed("wordpress"):
+ app_remove(auth, "wordpress")
+
+
+def install_app(app, path, additionnal_args=""):
+
+ app_install(auth, "./tests/apps/%s" % app,
+ args="domain=%s&path=%s%s" % (maindomain, path,
+ additionnal_args))
+
+
+def add_archive_wordpress_from_2p4():
+
+ os.system("mkdir -p /home/yunohost.backup/archives")
+
+ os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.info.json \
+ /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json")
+
+ os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.tar.gz \
+ /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz")
+
+
+def add_archive_system_from_2p4():
+
+ os.system("mkdir -p /home/yunohost.backup/archives")
+
+ os.system("cp ./tests/apps/backup_system_from_2p4/backup.info.json \
+ /home/yunohost.backup/archives/backup_system_from_2p4.info.json")
+
+ os.system("cp ./tests/apps/backup_system_from_2p4/backup.tar.gz \
+ /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz")
+
+###############################################################################
+# System backup #
+###############################################################################
+
+def test_backup_only_ldap():
+
+ # Create the backup
+ backup_create(ignore_system=False, ignore_apps=True, system=["conf_ldap"])
+
+ archives = backup_list()["archives"]
+ assert len(archives) == 1
+
+ archives_info = backup_info(archives[0], with_details=True)
+ assert archives_info["apps"] == {}
+ assert len(archives_info["system"].keys()) == 1
+ assert "conf_ldap" in archives_info["system"].keys()
+
+
+def test_backup_system_part_that_does_not_exists(mocker):
+
+ mocker.spy(m18n, "n")
+
+ # Create the backup
+ with pytest.raises(MoulinetteError):
+ backup_create(ignore_system=False, ignore_apps=True, system=["yolol"])
+
+ m18n.n.assert_any_call('backup_hook_unknown', hook="yolol")
+ m18n.n.assert_any_call('backup_nothings_done')
+
+###############################################################################
+# System backup and restore #
+###############################################################################
+
+def test_backup_and_restore_all_sys():
+
+ # Create the backup
+ backup_create(ignore_system=False, ignore_apps=True)
+
+ archives = backup_list()["archives"]
+ assert len(archives) == 1
+
+ archives_info = backup_info(archives[0], with_details=True)
+ assert archives_info["apps"] == {}
+ assert (len(archives_info["system"].keys()) ==
+ len(os.listdir("/usr/share/yunohost/hooks/backup/")))
+
+ # Remove ssowat conf
+ assert os.path.exists("/etc/ssowat/conf.json")
+ os.system("rm -rf /etc/ssowat/")
+ assert not os.path.exists("/etc/ssowat/conf.json")
+
+ # Restore the backup
+ backup_restore(auth, name=archives[0], force=True,
+ ignore_system=False, ignore_apps=True)
+
+ # Check ssowat conf is back
+ assert os.path.exists("/etc/ssowat/conf.json")
+
+
+###############################################################################
+# System restore from 2.4 #
+###############################################################################
+
+@pytest.mark.with_system_archive_from_2p4
+def test_restore_system_from_Ynh2p4(monkeypatch, mocker):
+
+ # Backup current system
+ backup_create(ignore_system=False, ignore_apps=True)
+ archives = backup_list()["archives"]
+ assert len(archives) == 2
+
+ # Restore system archive from 2.4
+ try:
+ backup_restore(auth, name=backup_list()["archives"][1],
+ ignore_system=False,
+ ignore_apps=True,
+ force=True)
+ finally:
+ # Restore system as it was
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=False,
+ ignore_apps=True,
+ force=True)
+
+###############################################################################
+# App backup #
+###############################################################################
+
+@pytest.mark.with_backup_recommended_app_installed
+def test_backup_script_failure_handling(monkeypatch, mocker):
+
+ def custom_hook_exec(name, *args, **kwargs):
+
+ if os.path.basename(name).startswith("backup_"):
+ raise Exception
+ else:
+ return True
+
+ # Create a backup of this app and simulate a crash (patching the backup
+ # call with monkeypatch). We also patch m18n to check later it's been called
+ # with the expected error message key
+ monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
+ mocker.spy(m18n, "n")
+
+ with pytest.raises(MoulinetteError):
+ backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"])
+
+ m18n.n.assert_any_call('backup_app_failed', app='backup_recommended_app')
+
+@pytest.mark.with_backup_recommended_app_installed
+def test_backup_not_enough_free_space(monkeypatch, mocker):
+
+ def custom_disk_usage(path):
+ return 99999999999999999
+
+ def custom_free_space_in_directory(dirpath):
+ return 0
+
+ monkeypatch.setattr("yunohost.backup.disk_usage", custom_disk_usage)
+ monkeypatch.setattr("yunohost.backup.free_space_in_directory",
+ custom_free_space_in_directory)
+
+ mocker.spy(m18n, "n")
+
+ with pytest.raises(MoulinetteError):
+ backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"])
+
+ m18n.n.assert_any_call('not_enough_disk_space', path=ANY)
+
+
+def test_backup_app_not_installed(mocker):
+
+ assert not _is_installed("wordpress")
+
+ mocker.spy(m18n, "n")
+
+ with pytest.raises(MoulinetteError):
+ backup_create(ignore_system=True, ignore_apps=False, apps=["wordpress"])
+
+ m18n.n.assert_any_call("unbackup_app", app="wordpress")
+ m18n.n.assert_any_call('backup_nothings_done')
+
+
+@pytest.mark.with_backup_recommended_app_installed
+def test_backup_app_with_no_backup_script(mocker):
+
+ backup_script = "/etc/yunohost/apps/backup_recommended_app/scripts/backup"
+ os.system("rm %s" % backup_script)
+ assert not os.path.exists(backup_script)
+
+ mocker.spy(m18n, "n")
+
+ with pytest.raises(MoulinetteError):
+ backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"])
+
+ m18n.n.assert_any_call("backup_with_no_backup_script_for_app", app="backup_recommended_app")
+ m18n.n.assert_any_call('backup_nothings_done')
+
+
+@pytest.mark.with_backup_recommended_app_installed
+def test_backup_app_with_no_restore_script(mocker):
+
+ restore_script = "/etc/yunohost/apps/backup_recommended_app/scripts/restore"
+ os.system("rm %s" % restore_script)
+ assert not os.path.exists(restore_script)
+
+ mocker.spy(m18n, "n")
+
+ # Backuping an app with no restore script will only display a warning to the
+ # user...
+
+ backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"])
+
+ m18n.n.assert_any_call("backup_with_no_restore_script_for_app", app="backup_recommended_app")
+
+
+@pytest.mark.clean_opt_dir
+def test_backup_with_different_output_directory():
+
+ # Create the backup
+ backup_create(ignore_system=False, ignore_apps=True, system=["conf_ssh"],
+ output_directory="/opt/test_backup_output_directory",
+ name="backup")
+
+ assert os.path.exists("/opt/test_backup_output_directory/backup.tar.gz")
+
+ archives = backup_list()["archives"]
+ assert len(archives) == 1
+
+ archives_info = backup_info(archives[0], with_details=True)
+ assert archives_info["apps"] == {}
+ assert len(archives_info["system"].keys()) == 1
+ assert "conf_ssh" in archives_info["system"].keys()
+
+@pytest.mark.clean_opt_dir
+def test_backup_with_no_compress():
+ # Create the backup
+ backup_create(ignore_system=False, ignore_apps=True, system=["conf_nginx"],
+ output_directory="/opt/test_backup_output_directory",
+ no_compress=True,
+ name="backup")
+
+ assert os.path.exists("/opt/test_backup_output_directory/info.json")
+
+
+###############################################################################
+# App restore #
+###############################################################################
+
+@pytest.mark.with_wordpress_archive_from_2p4
+def test_restore_app_wordpress_from_Ynh2p4():
+
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["wordpress"])
+
+
+@pytest.mark.with_wordpress_archive_from_2p4
+def test_restore_app_script_failure_handling(monkeypatch, mocker):
+
+ def custom_hook_exec(name, *args, **kwargs):
+ if os.path.basename(name).startswith("restore"):
+ monkeypatch.undo()
+ raise Exception
+
+ monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
+ mocker.spy(m18n, "n")
+
+ assert not _is_installed("wordpress")
+
+ with pytest.raises(MoulinetteError):
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["wordpress"])
+
+ m18n.n.assert_any_call('restore_app_failed', app='wordpress')
+ m18n.n.assert_any_call('restore_nothings_done')
+ assert not _is_installed("wordpress")
+
+
+@pytest.mark.with_wordpress_archive_from_2p4
+def test_restore_app_not_enough_free_space(monkeypatch, mocker):
+
+ def custom_free_space_in_directory(dirpath):
+ return 0
+
+ monkeypatch.setattr("yunohost.backup.free_space_in_directory",
+ custom_free_space_in_directory)
+ mocker.spy(m18n, "n")
+
+ assert not _is_installed("wordpress")
+
+ with pytest.raises(MoulinetteError):
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["wordpress"])
+
+ m18n.n.assert_any_call('restore_not_enough_disk_space',
+ free_space=0,
+ margin=ANY,
+ needed_space=ANY)
+ assert not _is_installed("wordpress")
+
+
+@pytest.mark.with_wordpress_archive_from_2p4
+def test_restore_app_not_in_backup(mocker):
+
+ assert not _is_installed("wordpress")
+ assert not _is_installed("yoloswag")
+
+ mocker.spy(m18n, "n")
+
+ with pytest.raises(MoulinetteError):
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["yoloswag"])
+
+ m18n.n.assert_any_call('backup_archive_app_not_found', app="yoloswag")
+ assert not _is_installed("wordpress")
+ assert not _is_installed("yoloswag")
+
+
+@pytest.mark.with_wordpress_archive_from_2p4
+def test_restore_app_already_installed(mocker):
+
+ assert not _is_installed("wordpress")
+
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["wordpress"])
+
+ assert _is_installed("wordpress")
+
+ mocker.spy(m18n, "n")
+ with pytest.raises(MoulinetteError):
+ backup_restore(auth, name=backup_list()["archives"][0],
+ ignore_system=True,
+ ignore_apps=False,
+ apps=["wordpress"])
+
+ m18n.n.assert_any_call('restore_already_installed_app', app="wordpress")
+ m18n.n.assert_any_call('restore_nothings_done')
+
+ assert _is_installed("wordpress")
+
+
+@pytest.mark.with_backup_legacy_app_installed
+def test_backup_and_restore_legacy_app():
+
+ _test_backup_and_restore_app("backup_legacy_app")
+
+
+@pytest.mark.with_backup_recommended_app_installed
+def test_backup_and_restore_recommended_app():
+
+ _test_backup_and_restore_app("backup_recommended_app")
+
+
+@pytest.mark.with_backup_recommended_app_installed_with_ynh_restore
+def test_backup_and_restore_with_ynh_restore():
+
+ _test_backup_and_restore_app("backup_recommended_app")
+
+
+def _test_backup_and_restore_app(app):
+
+ # Create a backup of this app
+ backup_create(ignore_system=True, ignore_apps=False, apps=[app])
+
+ archives = backup_list()["archives"]
+ assert len(archives) == 1
+
+ archives_info = backup_info(archives[0], with_details=True)
+ assert archives_info["system"] == {}
+ assert len(archives_info["apps"].keys()) == 1
+ assert app in archives_info["apps"].keys()
+
+ # Uninstall the app
+ app_remove(auth, app)
+ assert not app_is_installed(app)
+
+ # Restore the app
+ backup_restore(auth, name=archives[0], ignore_system=True,
+ ignore_apps=False, apps=[app])
+
+ assert app_is_installed(app)
+
+###############################################################################
+# Some edge cases #
+###############################################################################
+
+def test_restore_archive_with_no_json(mocker):
+
+ # Create a backup with no info.json associated
+ os.system("touch /tmp/afile")
+ os.system("tar -czvf /home/yunohost.backup/archives/badbackup.tar.gz /tmp/afile")
+
+ assert "badbackup" in backup_list()["archives"]
+
+ mocker.spy(m18n, "n")
+ with pytest.raises(MoulinetteError):
+ backup_restore(auth, name="badbackup", force=True,
+ ignore_system=False, ignore_apps=False)
+ m18n.n.assert_any_call('backup_invalid_archive')
+
+
+def test_backup_binds_are_readonly(monkeypatch):
+
+ def custom_mount_and_backup(self, backup_manager):
+ self.manager = backup_manager
+ self._organize_files()
+
+ confssh = os.path.join(self.work_dir, "conf/ssh")
+ output = subprocess.check_output("touch %s/test 2>&1 || true" % confssh,
+ shell=True)
+
+ assert "Read-only file system" in output
+
+ if self._recursive_umount(self.work_dir) > 0:
+ raise Exception("Backup cleaning failed !")
+
+ self.clean()
+
+ monkeypatch.setattr("yunohost.backup.BackupMethod.mount_and_backup",
+ custom_mount_and_backup)
+
+ # Create the backup
+ backup_create(ignore_system=False, ignore_apps=True)
diff --git a/src/yunohost/tests/test_changeurl.py b/src/yunohost/tests/test_changeurl.py
new file mode 100644
index 000000000..737b68a6d
--- /dev/null
+++ b/src/yunohost/tests/test_changeurl.py
@@ -0,0 +1,61 @@
+import pytest
+import time
+import requests
+
+from moulinette.core import init_authenticator
+from yunohost.app import app_install, app_change_url, app_remove, app_map
+from yunohost.domain import _get_maindomain
+
+from moulinette.core import MoulinetteError
+
+# Instantiate LDAP Authenticator
+AUTH_IDENTIFIER = ('ldap', 'ldap-anonymous')
+AUTH_PARAMETERS = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'}
+
+auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS)
+
+# Get main domain
+maindomain = _get_maindomain()
+
+
+def setup_function(function):
+ pass
+
+
+def teardown_function(function):
+ app_remove(auth, "change_url_app")
+
+
+def install_changeurl_app(path):
+ app_install(auth, "./tests/apps/change_url_app_ynh",
+ args="domain=%s&path=%s" % (maindomain, path))
+
+
+def check_changeurl_app(path):
+ appmap = app_map(raw=True)
+
+ assert path + "/" in appmap[maindomain].keys()
+
+ assert appmap[maindomain][path + "/"]["id"] == "change_url_app"
+
+ r = requests.get("https://%s%s/" % (maindomain, path), verify=False)
+ assert r.status_code == 200
+
+
+def test_appchangeurl():
+ install_changeurl_app("/changeurl")
+ check_changeurl_app("/changeurl")
+
+ app_change_url(auth, "change_url_app", maindomain, "/newchangeurl")
+
+ # For some reason the nginx reload can take some time to propagate ...?
+ time.sleep(2)
+
+ check_changeurl_app("/newchangeurl")
+
+def test_appchangeurl_sameurl():
+ install_changeurl_app("/changeurl")
+ check_changeurl_app("/changeurl")
+
+ with pytest.raises(MoulinetteError):
+ app_change_url(auth, "change_url_app", maindomain, "changeurl")
diff --git a/src/yunohost/tests/test_settings.py b/src/yunohost/tests/test_settings.py
new file mode 100644
index 000000000..746f5a9d4
--- /dev/null
+++ b/src/yunohost/tests/test_settings.py
@@ -0,0 +1,166 @@
+import os
+import json
+import pytest
+
+from moulinette.core import MoulinetteError
+
+from yunohost.settings import settings_get, settings_list, _get_settings, \
+ settings_set, settings_reset, settings_reset_all, \
+ SETTINGS_PATH_OTHER_LOCATION, SETTINGS_PATH
+
+
+def setup_function(function):
+ os.system("mv /etc/yunohost/settings.json /etc/yunohost/settings.json.saved")
+
+
+def teardown_function(function):
+ os.system("mv /etc/yunohost/settings.json.saved /etc/yunohost/settings.json")
+
+
+def test_settings_get_bool():
+ assert settings_get("example.bool") == True
+
+def test_settings_get_full_bool():
+ assert settings_get("example.bool", True) == {"type": "bool", "value": True, "default": True, "description": "Example boolean option"}
+
+
+def test_settings_get_int():
+ assert settings_get("example.int") == 42
+
+def test_settings_get_full_int():
+ assert settings_get("example.int", True) == {"type": "int", "value": 42, "default": 42, "description": "Example int option"}
+
+
+def test_settings_get_string():
+ assert settings_get("example.string") == "yolo swag"
+
+def test_settings_get_full_string():
+ assert settings_get("example.string", True) == {"type": "string", "value": "yolo swag", "default": "yolo swag", "description": "Example string option"}
+
+
+def test_settings_get_enum():
+ assert settings_get("example.enum") == "a"
+
+def test_settings_get_full_enum():
+ assert settings_get("example.enum", True) == {"type": "enum", "value": "a", "default": "a", "description": "Example enum option", "choices": ["a", "b", "c"]}
+
+
+def test_settings_get_doesnt_exists():
+ with pytest.raises(MoulinetteError):
+ settings_get("doesnt.exists")
+
+
+def test_settings_list():
+ assert settings_list() == _get_settings()
+
+
+def test_settings_set():
+ settings_set("example.bool", False)
+ assert settings_get("example.bool") == False
+
+
+def test_settings_set_int():
+ settings_set("example.int", 21)
+ assert settings_get("example.int") == 21
+
+
+def test_settings_set_enum():
+ settings_set("example.enum", "c")
+ assert settings_get("example.enum") == "c"
+
+
+def test_settings_set_doesexit():
+ with pytest.raises(MoulinetteError):
+ settings_set("doesnt.exist", True)
+
+
+def test_settings_set_bad_type_bool():
+ with pytest.raises(MoulinetteError):
+ settings_set("example.bool", 42)
+ with pytest.raises(MoulinetteError):
+ settings_set("example.bool", "pouet")
+
+
+def test_settings_set_bad_type_int():
+ with pytest.raises(MoulinetteError):
+ settings_set("example.int", True)
+ with pytest.raises(MoulinetteError):
+ settings_set("example.int", "pouet")
+
+
+def test_settings_set_bad_type_string():
+ with pytest.raises(MoulinetteError):
+ settings_set("example.string", True)
+ with pytest.raises(MoulinetteError):
+ settings_set("example.string", 42)
+
+
+def test_settings_set_bad_value_enum():
+ with pytest.raises(MoulinetteError):
+ settings_set("example.enum", True)
+ with pytest.raises(MoulinetteError):
+ settings_set("example.enum", "e")
+ with pytest.raises(MoulinetteError):
+ settings_set("example.enum", 42)
+ with pytest.raises(MoulinetteError):
+ settings_set("example.enum", "pouet")
+
+
+def test_settings_list_modified():
+ settings_set("example.int", 21)
+ assert settings_list()["example.int"] == {'default': 42, 'description': 'Example int option', 'type': 'int', 'value': 21}
+
+
+def test_reset():
+ settings_set("example.int", 21)
+ assert settings_get("example.int") == 21
+ settings_reset("example.int")
+ assert settings_get("example.int") == settings_get("example.int", True)["default"]
+
+
+def test_settings_reset_doesexit():
+ with pytest.raises(MoulinetteError):
+ settings_reset("doesnt.exist")
+
+
+def test_reset_all():
+ settings_before = settings_list()
+ settings_set("example.bool", False)
+ settings_set("example.int", 21)
+ settings_set("example.string", "pif paf pouf")
+ settings_set("example.enum", "c")
+ assert settings_before != settings_list()
+ settings_reset_all()
+ if settings_before != settings_list():
+ for i in settings_before:
+ assert settings_before[i] == settings_list()[i]
+
+
+def test_reset_all_backup():
+ settings_before = settings_list()
+ settings_set("example.bool", False)
+ settings_set("example.int", 21)
+ settings_set("example.string", "pif paf pouf")
+ settings_set("example.enum", "c")
+ settings_after_modification = settings_list()
+ assert settings_before != settings_after_modification
+ old_settings_backup_path = settings_reset_all()["old_settings_backup_path"]
+
+ for i in settings_after_modification:
+ del settings_after_modification[i]["description"]
+
+ assert settings_after_modification == json.load(open(old_settings_backup_path, "r"))
+
+
+
+def test_unknown_keys():
+ unknown_settings_path = SETTINGS_PATH_OTHER_LOCATION % "unknown"
+ unknown_setting = {
+ "unkown_key": {"value": 42, "default": 31, "type": "int"},
+ }
+ open(SETTINGS_PATH, "w").write(json.dumps(unknown_setting))
+
+ # stimulate a write
+ settings_reset_all()
+
+ assert unknown_setting == json.load(open(unknown_settings_path, "r"))
diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py
index f78e32363..f9ee14994 100644
--- a/src/yunohost/tools.py
+++ b/src/yunohost/tools.py
@@ -23,51 +23,72 @@
Specific tools
"""
-import os
-import sys
-import yaml
import re
-import getpass
-import requests
+import os
+import yaml
import json
import errno
import logging
+import subprocess
+import pwd
+import socket
+from xmlrpclib import Fault
+from importlib import import_module
from collections import OrderedDict
import apt
import apt.progress
+from moulinette import msettings, msignals, m18n
from moulinette.core import MoulinetteError, init_authenticator
from moulinette.utils.log import getActionLogger
-from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list
-from yunohost.domain import domain_add, domain_list, get_public_ip
-from yunohost.dyndns import dyndns_subscribe
-from yunohost.firewall import firewall_upnp, firewall_reload
-from yunohost.service import service_status, service_regen_conf, service_log
-from yunohost.monitor import monitor_disk, monitor_network, monitor_system
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_json, write_to_json
+from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
+from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain
+from yunohost.dyndns import _dyndns_available, _dyndns_provides
+from yunohost.firewall import firewall_upnp
+from yunohost.service import service_status, service_regen_conf, service_log, service_start, service_enable
+from yunohost.monitor import monitor_disk, monitor_system
from yunohost.utils.packages import ynh_packages_version
+from yunohost.utils.network import get_public_ip
+from yunohost.log import is_unit_operation, OperationLogger
-apps_setting_path= '/etc/yunohost/apps/'
+# FIXME this is a duplicate from apps.py
+APPS_SETTING_PATH = '/etc/yunohost/apps/'
+MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json"
logger = getActionLogger('yunohost.tools')
-def tools_ldapinit(auth):
+def tools_ldapinit():
"""
YunoHost LDAP initialization
"""
+
+ # Instantiate LDAP Authenticator
+ auth = init_authenticator(('ldap', 'default'),
+ {'uri': "ldap://localhost:389",
+ 'base_dn': "dc=yunohost,dc=org",
+ 'user_rdn': "cn=admin"})
+ auth.authenticate('yunohost')
+
with open('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') as f:
ldap_map = yaml.load(f)
for rdn, attr_dict in ldap_map['parents'].items():
- try: auth.add(rdn, attr_dict)
- except: pass
+ try:
+ auth.add(rdn, attr_dict)
+ except Exception as e:
+ logger.warn("Error when trying to inject '%s' -> '%s' into ldap: %s" % (rdn, attr_dict, e))
for rdn, attr_dict in ldap_map['children'].items():
- try: auth.add(rdn, attr_dict)
- except: pass
+ try:
+ auth.add(rdn, attr_dict)
+ except Exception as e:
+ logger.warn("Error when trying to inject '%s' -> '%s' into ldap: %s" % (rdn, attr_dict, e))
admin_dict = {
'cn': 'admin',
@@ -83,7 +104,18 @@ def tools_ldapinit(auth):
auth.update('cn=admin', admin_dict)
+ # Force nscd to refresh cache to take admin creation into account
+ subprocess.call(['nscd', '-i', 'passwd'])
+
+ # Check admin actually exists now
+ try:
+ pwd.getpwnam("admin")
+ except KeyError:
+ logger.error(m18n.n('ldap_init_failed_to_create_admin'))
+ raise MoulinetteError(errno.EINVAL, m18n.n('installation_failed'))
+
logger.success(m18n.n('ldap_initialized'))
+ return auth
def tools_adminpw(auth, new_password):
@@ -94,8 +126,11 @@ def tools_adminpw(auth, new_password):
new_password
"""
+ from yunohost.user import _hash_user_password
try:
- auth.con.passwd_s('cn=admin,dc=yunohost,dc=org', None, new_password)
+ auth.update("cn=admin", {
+ "userPassword": _hash_user_password(new_password),
+ })
except:
logger.exception('unable to change admin password')
raise MoulinetteError(errno.EPERM,
@@ -104,103 +139,174 @@ def tools_adminpw(auth, new_password):
logger.success(m18n.n('admin_password_changed'))
-def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
+@is_unit_operation()
+def tools_maindomain(operation_logger, auth, new_domain=None):
"""
- Main domain change tool
+ Check the current main domain, or change it
Keyword argument:
- new_domain
- old_domain
+ new_domain -- The new domain to be set as the main domain
"""
- if not old_domain:
- with open('/etc/yunohost/current_host', 'r') as f:
- old_domain = f.readline().rstrip()
-
- if not new_domain:
- return { 'current_main_domain': old_domain }
+ # If no new domain specified, we return the current main domain
if not new_domain:
- raise MoulinetteError(errno.EINVAL, m18n.n('new_domain_required'))
+ return {'current_main_domain': _get_maindomain()}
+
+ # Check domain exists
if new_domain not in domain_list(auth)['domains']:
- domain_add(auth, new_domain)
+ raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))
- os.system('rm /etc/ssl/private/yunohost_key.pem')
- os.system('rm /etc/ssl/certs/yunohost_crt.pem')
+ operation_logger.related_to.append(('domain', new_domain))
+ operation_logger.start()
- command_list = [
- 'ln -s /etc/yunohost/certs/%s/key.pem /etc/ssl/private/yunohost_key.pem' % new_domain,
- 'ln -s /etc/yunohost/certs/%s/crt.pem /etc/ssl/certs/yunohost_crt.pem' % new_domain,
- 'echo %s > /etc/yunohost/current_host' % new_domain,
- ]
+ # Apply changes to ssl certs
+ ssl_key = "/etc/ssl/private/yunohost_key.pem"
+ ssl_crt = "/etc/ssl/private/yunohost_crt.pem"
+ new_ssl_key = "/etc/yunohost/certs/%s/key.pem" % new_domain
+ new_ssl_crt = "/etc/yunohost/certs/%s/crt.pem" % new_domain
- for command in command_list:
- if os.system(command) != 0:
- raise MoulinetteError(errno.EPERM,
- m18n.n('maindomain_change_failed'))
+ try:
+ if os.path.exists(ssl_key) or os.path.lexists(ssl_key):
+ os.remove(ssl_key)
+ if os.path.exists(ssl_crt) or os.path.lexists(ssl_crt):
+ os.remove(ssl_crt)
- if dyndns and len(new_domain.split('.')) >= 3:
- try:
- r = requests.get('https://dyndns.yunohost.org/domains')
- except requests.ConnectionError:
- pass
- else:
- dyndomains = json.loads(r.text)
- dyndomain = '.'.join(new_domain.split('.')[1:])
- if dyndomain in dyndomains:
- dyndns_subscribe(domain=new_domain)
+ os.symlink(new_ssl_key, ssl_key)
+ os.symlink(new_ssl_crt, ssl_crt)
+ _set_maindomain(new_domain)
+ except Exception as e:
+ logger.warning("%s" % e, exc_info=1)
+ raise MoulinetteError(errno.EPERM, m18n.n('maindomain_change_failed'))
+
+ _set_hostname(new_domain)
+
+ # Generate SSOwat configuration file
+ app_ssowatconf(auth)
+
+ # Regen configurations
try:
with open('/etc/yunohost/installed', 'r') as f:
service_regen_conf()
- except IOError: pass
+ except IOError:
+ pass
logger.success(m18n.n('maindomain_changed'))
-def tools_postinstall(domain, password, ignore_dyndns=False):
+def _set_hostname(hostname, pretty_hostname=None):
+ """
+ Change the machine hostname using hostnamectl
+ """
+
+ if _is_inside_container():
+ logger.warning("You are inside a container and hostname cannot easily be changed")
+ return
+
+ if not pretty_hostname:
+ pretty_hostname = "(YunoHost/%s)" % hostname
+
+ # First clear nsswitch cache for hosts to make sure hostname is resolved...
+ subprocess.call(['nscd', '-i', 'hosts'])
+
+ # Then call hostnamectl
+ commands = [
+ "sudo hostnamectl --static set-hostname".split() + [hostname],
+ "sudo hostnamectl --transient set-hostname".split() + [hostname],
+ "sudo hostnamectl --pretty set-hostname".split() + [pretty_hostname]
+ ]
+
+ for command in commands:
+ p = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+
+ out, _ = p.communicate()
+
+ if p.returncode != 0:
+ logger.warning(command)
+ logger.warning(out)
+ raise MoulinetteError(errno.EIO, m18n.n('domain_hostname_failed'))
+ else:
+ logger.debug(out)
+
+
+def _is_inside_container():
+ """
+ Check if we're inside a container (i.e. LXC)
+
+ Returns True or False
+ """
+
+ # See https://www.2daygeek.com/check-linux-system-physical-virtual-machine-virtualization-technology/
+ p = subprocess.Popen("sudo systemd-detect-virt".split(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+
+ out, _ = p.communicate()
+ container = ['lxc','lxd','docker']
+ return out.split()[0] in container
+
+
+@is_unit_operation()
+def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False):
"""
YunoHost post-install
Keyword argument:
domain -- YunoHost main domain
- ignore_dyndns -- Do not subscribe domain to a DynDNS service
+ ignore_dyndns -- Do not subscribe domain to a DynDNS service (only
+ needed for nohost.me, noho.st domains)
password -- YunoHost admin password
"""
- dyndns = not ignore_dyndns
+ dyndns_provider = "dyndns.yunohost.org"
# Do some checks at first
if os.path.isfile('/etc/yunohost/installed'):
raise MoulinetteError(errno.EPERM,
m18n.n('yunohost_already_installed'))
- if len(domain.split('.')) >= 3 and not ignore_dyndns:
- try:
- r = requests.get('https://dyndns.yunohost.org/domains')
- except requests.ConnectionError:
- pass
- else:
- dyndomains = json.loads(r.text)
- dyndomain = '.'.join(domain.split('.')[1:])
- if dyndomain in dyndomains:
- if requests.get('https://dyndns.yunohost.org/test/%s' % domain).status_code == 200:
- dyndns = True
- else:
- raise MoulinetteError(errno.EEXIST,
- m18n.n('dyndns_unavailable'))
+ if not ignore_dyndns:
+ # Check if yunohost dyndns can handle the given domain
+ # (i.e. is it a .nohost.me ? a .noho.st ?)
+ try:
+ is_nohostme_or_nohost = _dyndns_provides(dyndns_provider, domain)
+ # If an exception is thrown, most likely we don't have internet
+ # connectivity or something. Assume that this domain isn't manageable
+ # and inform the user that we could not contact the dyndns host server.
+ except:
+ logger.warning(m18n.n('dyndns_provider_unreachable',
+ provider=dyndns_provider))
+ is_nohostme_or_nohost = False
+
+ # If this is a nohost.me/noho.st, actually check for availability
+ if is_nohostme_or_nohost:
+ # (Except if the user explicitly said he/she doesn't care about dyndns)
+ if ignore_dyndns:
+ dyndns = False
+ # Check if the domain is available...
+ elif _dyndns_available(dyndns_provider, domain):
+ dyndns = True
+ # If not, abort the postinstall
+ else:
+ raise MoulinetteError(errno.EEXIST,
+ m18n.n('dyndns_unavailable',
+ domain=domain))
+ else:
+ dyndns = False
+ else:
+ dyndns = False
+
+ operation_logger.start()
logger.info(m18n.n('yunohost_installing'))
- # Instantiate LDAP Authenticator
- auth = init_authenticator(('ldap', 'default'),
- {'uri': "ldap://localhost:389",
- 'base_dn': "dc=yunohost,dc=org",
- 'user_rdn': "cn=admin" })
- auth.authenticate('yunohost')
+ service_regen_conf(['nslcd', 'nsswitch'], force=True)
# Initialize LDAP for YunoHost
# TODO: Improve this part by integrate ldapinit into conf_regen hook
- tools_ldapinit(auth)
+ auth = tools_ldapinit()
# Create required folders
folders_to_create = [
@@ -212,57 +318,75 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
]
for folder in folders_to_create:
- try: os.listdir(folder)
- except OSError: os.makedirs(folder)
+ try:
+ os.listdir(folder)
+ except OSError:
+ os.makedirs(folder)
# Change folders permissions
os.system('chmod 755 /home/yunohost.app')
# Set hostname to avoid amavis bug
- if os.system('hostname -d') != 0:
+ if os.system('hostname -d >/dev/null') != 0:
os.system('hostname yunohost.yunohost.org')
# Add a temporary SSOwat rule to redirect SSO to admin page
try:
with open('/etc/ssowat/conf.json.persistent') as json_conf:
ssowat_conf = json.loads(str(json_conf.read()))
+ except ValueError as e:
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('ssowat_persistent_conf_read_error', error=str(e)))
except IOError:
ssowat_conf = {}
if 'redirected_urls' not in ssowat_conf:
ssowat_conf['redirected_urls'] = {}
- ssowat_conf['redirected_urls']['/'] = domain +'/yunohost/admin'
+ ssowat_conf['redirected_urls']['/'] = domain + '/yunohost/admin'
- with open('/etc/ssowat/conf.json.persistent', 'w+') as f:
- json.dump(ssowat_conf, f, sort_keys=True, indent=4)
+ try:
+ with open('/etc/ssowat/conf.json.persistent', 'w+') as f:
+ json.dump(ssowat_conf, f, sort_keys=True, indent=4)
+ except IOError as e:
+ raise MoulinetteError(errno.EPERM,
+ m18n.n('ssowat_persistent_conf_write_error', error=str(e)))
os.system('chmod 644 /etc/ssowat/conf.json.persistent')
# Create SSL CA
service_regen_conf(['ssl'], force=True)
ssl_dir = '/usr/share/yunohost/yunohost-config/ssl/yunoCA'
- command_list = [
+ commands = [
'echo "01" > %s/serial' % ssl_dir,
- 'rm %s/index.txt' % ssl_dir,
- 'touch %s/index.txt' % ssl_dir,
+ 'rm %s/index.txt' % ssl_dir,
+ 'touch %s/index.txt' % ssl_dir,
'cp %s/openssl.cnf %s/openssl.ca.cnf' % (ssl_dir, ssl_dir),
- 'sed -i "s/yunohost.org/%s/g" %s/openssl.ca.cnf ' % (domain, ssl_dir),
+ 'sed -i s/yunohost.org/%s/g %s/openssl.ca.cnf ' % (domain, ssl_dir),
'openssl req -x509 -new -config %s/openssl.ca.cnf -days 3650 -out %s/ca/cacert.pem -keyout %s/ca/cakey.pem -nodes -batch' % (ssl_dir, ssl_dir, ssl_dir),
'cp %s/ca/cacert.pem /etc/ssl/certs/ca-yunohost_crt.pem' % ssl_dir,
'update-ca-certificates'
]
- for command in command_list:
- if os.system(command) != 0:
+ for command in commands:
+ p = subprocess.Popen(
+ command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+ out, _ = p.communicate()
+
+ if p.returncode != 0:
+ logger.warning(out)
raise MoulinetteError(errno.EPERM,
m18n.n('yunohost_ca_creation_failed'))
+ else:
+ logger.debug(out)
+
+ logger.success(m18n.n('yunohost_ca_creation_success'))
# New domain config
- tools_maindomain(auth, old_domain='yunohost.org', new_domain=domain, dyndns=dyndns)
-
- # Generate SSOwat configuration file
- app_ssowatconf(auth)
+ service_regen_conf(['nsswitch'], force=True)
+ domain_add(auth, domain, dyndns)
+ tools_maindomain(auth, domain)
# Change LDAP admin password
tools_adminpw(auth, password)
@@ -270,16 +394,29 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
# Enable UPnP silently and reload firewall
firewall_upnp('enable', no_refresh=True)
+ # Setup the default official app list with cron job
+ try:
+ app_fetchlist(name="yunohost",
+ url="https://app.yunohost.org/official.json")
+ except Exception as e:
+ logger.warning(str(e))
+
+ _install_appslist_fetch_cron()
+
+ # Init migrations (skip them, no need to run them on a fresh system)
+ tools_migrations_migrate(skip=True, auto=True)
+
os.system('touch /etc/yunohost/installed')
# Enable and start YunoHost firewall at boot time
- os.system('update-rc.d yunohost-firewall enable')
- os.system('service yunohost-firewall start')
+ service_enable("yunohost-firewall")
+ service_start("yunohost-firewall")
service_regen_conf(force=True)
-
logger.success(m18n.n('yunohost_configured'))
+ logger.warning(m18n.n('recommend_to_add_first_user'))
+
def tools_update(ignore_apps=False, ignore_packages=False):
"""
@@ -290,15 +427,15 @@ def tools_update(ignore_apps=False, ignore_packages=False):
ignore_packages -- Ignore apt cache update and changelog
"""
+ # "packages" will list upgradable packages
packages = []
if not ignore_packages:
cache = apt.Cache()
# Update APT cache
- logger.info(m18n.n('updating_apt_cache'))
+ logger.debug(m18n.n('updating_apt_cache'))
if not cache.update():
raise MoulinetteError(errno.EPERM, m18n.n('update_cache_failed'))
- logger.info(m18n.n('done'))
cache.open(None)
cache.upgrade(True)
@@ -310,45 +447,36 @@ def tools_update(ignore_apps=False, ignore_packages=False):
'fullname': pkg.fullname,
'changelog': pkg.get_changelog()
})
+ logger.debug(m18n.n('done'))
+ # "apps" will list upgradable packages
apps = []
if not ignore_apps:
try:
app_fetchlist()
except MoulinetteError:
+ # FIXME : silent exception !?
pass
- app_list = os.listdir(apps_setting_path)
- if len(app_list) > 0:
- for app_id in app_list:
- if '__' in app_id:
- original_app_id = app_id[:app_id.index('__')]
- else:
- original_app_id = app_id
- current_app_dict = app_info(app_id, raw=True)
- new_app_dict = app_info(original_app_id, raw=True)
+ app_list_installed = os.listdir(APPS_SETTING_PATH)
+ for app_id in app_list_installed:
- # Custom app
- if new_app_dict is None or 'lastUpdate' not in new_app_dict or 'git' not in new_app_dict:
- continue
+ app_dict = app_info(app_id, raw=True)
- if (new_app_dict['lastUpdate'] > current_app_dict['lastUpdate']) \
- or ('update_time' not in current_app_dict['settings'] \
- and (new_app_dict['lastUpdate'] > current_app_dict['settings']['install_time'])) \
- or ('update_time' in current_app_dict['settings'] \
- and (new_app_dict['lastUpdate'] > current_app_dict['settings']['update_time'])):
- apps.append({
- 'id': app_id,
- 'label': current_app_dict['settings']['label']
- })
+ if app_dict["upgradable"] == "yes":
+ apps.append({
+ 'id': app_id,
+ 'label': app_dict['settings']['label']
+ })
if len(apps) == 0 and len(packages) == 0:
logger.info(m18n.n('packages_no_upgrade'))
- return { 'packages': packages, 'apps': apps }
+ return {'packages': packages, 'apps': apps}
-def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
+@is_unit_operation()
+def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
@@ -378,15 +506,18 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
critical_upgrades.add(pkg.name)
# Temporarily keep package ...
pkg.mark_keep()
+
# ... and set a hourly cron up to upgrade critical packages
if critical_upgrades:
logger.info(m18n.n('packages_upgrade_critical_later',
- packages=', '.join(critical_upgrades)))
+ packages=', '.join(critical_upgrades)))
with open('/etc/cron.d/yunohost-upgrade', 'w+') as f:
f.write('00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install %s -y && rm -f /etc/cron.d/yunohost-upgrade\n' % ' '.join(critical_upgrades))
if cache.get_changes():
logger.info(m18n.n('upgrading_packages'))
+
+ operation_logger.start()
try:
# Apply APT changes
# TODO: Logs output for the API
@@ -394,27 +525,30 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
apt.progress.base.InstallProgress())
except Exception as e:
failure = True
- logging.warning('unable to upgrade packages: %s' % str(e))
+ logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
+ operation_logger.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
+ operation_logger.success()
else:
logger.info(m18n.n('packages_no_upgrade'))
+
if not ignore_apps:
try:
app_upgrade(auth)
except Exception as e:
failure = True
- logging.warning('unable to upgrade apps: %s' % str(e))
- logger.error(m18n.n('app_upgrade_failed'))
+ logger.warning('unable to upgrade apps: %s' % str(e))
+ logger.error(m18n.n('app_upgrade_some_app_failed'))
if not failure:
logger.success(m18n.n('system_upgraded'))
# Return API logs if it is an API call
if is_api:
- return { "log": service_log('yunohost-api', number="100").values()[0] }
+ return {"log": service_log('yunohost-api', number="100").values()[0]}
def tools_diagnosis(auth, private=False):
@@ -422,7 +556,7 @@ def tools_diagnosis(auth, private=False):
Return global info about current yunohost instance to help debugging
"""
- diagnosis = OrderedDict();
+ diagnosis = OrderedDict()
# Debian release
try:
@@ -445,20 +579,25 @@ def tools_diagnosis(auth, private=False):
# Packages version
diagnosis['packages'] = ynh_packages_version()
+ diagnosis["backports"] = check_output("dpkg -l |awk '/^ii/ && $3 ~ /bpo[6-8]/ {print $2}'").split()
+
# Server basic monitoring
diagnosis['system'] = OrderedDict()
try:
disks = monitor_disk(units=['filesystem'], human_readable=True)
- except MoulinetteError as e:
+ except (MoulinetteError, Fault) as e:
logger.warning(m18n.n('diagnosis_monitor_disk_error', error=format(e)), exc_info=1)
else:
diagnosis['system']['disks'] = {}
for disk in disks:
- diagnosis['system']['disks'][disk] = 'Mounted on %s, %s (%s free)' % (
- disks[disk]['mnt_point'],
- disks[disk]['size'],
- disks[disk]['avail']
- )
+ if isinstance(disks[disk], str):
+ diagnosis['system']['disks'][disk] = disks[disk]
+ else:
+ diagnosis['system']['disks'][disk] = 'Mounted on %s, %s (%s free)' % (
+ disks[disk]['mnt_point'],
+ disks[disk]['size'],
+ disks[disk]['avail']
+ )
try:
system = monitor_system(units=['cpu', 'memory'], human_readable=True)
@@ -466,13 +605,22 @@ def tools_diagnosis(auth, private=False):
logger.warning(m18n.n('diagnosis_monitor_system_error', error=format(e)), exc_info=1)
else:
diagnosis['system']['memory'] = {
- 'ram' : '%s (%s free)' % (system['memory']['ram']['total'], system['memory']['ram']['free']),
- 'swap' : '%s (%s free)' % (system['memory']['swap']['total'], system['memory']['swap']['free']),
+ 'ram': '%s (%s free)' % (system['memory']['ram']['total'], system['memory']['ram']['free']),
+ 'swap': '%s (%s free)' % (system['memory']['swap']['total'], system['memory']['swap']['free']),
}
+ # nginx -t
+ try:
+ diagnosis['nginx'] = check_output("nginx -t").strip().split("\n")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ logger.warning("Unable to check 'nginx -t', exception: %s" % e)
+
# Services status
services = service_status()
diagnosis['services'] = {}
+
for service in services:
diagnosis['services'][service] = "%s (%s)" % (services[service]['status'], services[service]['loaded'])
@@ -490,18 +638,411 @@ def tools_diagnosis(auth, private=False):
# Private data
if private:
diagnosis['private'] = OrderedDict()
+
# Public IP
diagnosis['private']['public_ip'] = {}
- try:
- diagnosis['private']['public_ip']['IPv4'] = get_public_ip(4)
- except MoulinetteError as e:
- pass
- try:
- diagnosis['private']['public_ip']['IPv6'] = get_public_ip(6)
- except MoulinetteError as e:
- pass
+ diagnosis['private']['public_ip']['IPv4'] = get_public_ip(4)
+ diagnosis['private']['public_ip']['IPv6'] = get_public_ip(6)
# Domains
diagnosis['private']['domains'] = domain_list(auth)['domains']
+ diagnosis['private']['regen_conf'] = service_regen_conf(with_diff=True, dry_run=True)
+
+ try:
+ diagnosis['security'] = {
+ "CVE-2017-5754": {
+ "name": "meltdown",
+ "vulnerable": _check_if_vulnerable_to_meltdown(),
+ }
+ }
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ logger.warning("Unable to check for meltdown vulnerability: %s" % e)
+
return diagnosis
+
+
+def _check_if_vulnerable_to_meltdown():
+ # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
+
+ # script taken from https://github.com/speed47/spectre-meltdown-checker
+ # script commit id is store directly in the script
+ file_dir = os.path.split(__file__)[0]
+ SCRIPT_PATH = os.path.join(file_dir, "./vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh")
+
+ # '--variant 3' corresponds to Meltdown
+ # example output from the script:
+ # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
+ try:
+ call = subprocess.Popen("bash %s --batch json --variant 3" %
+ SCRIPT_PATH, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+
+ output, _ = call.communicate()
+ assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
+
+ CVEs = json.loads(output)
+ assert len(CVEs) == 1
+ assert CVEs[0]["NAME"] == "MELTDOWN"
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e)
+ raise Exception("Command output for failed meltdown check: '%s'" % output)
+
+ return CVEs[0]["VULNERABLE"]
+
+
+def tools_port_available(port):
+ """
+ Check availability of a local port
+
+ Keyword argument:
+ port -- Port to check
+
+ """
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(1)
+ s.connect(("localhost", int(port)))
+ s.close()
+ except socket.error:
+ return True
+ else:
+ return False
+
+
+@is_unit_operation()
+def tools_shutdown(operation_logger, force=False):
+ shutdown = force
+ if not shutdown:
+ try:
+ # Ask confirmation for server shutdown
+ i = msignals.prompt(m18n.n('server_shutdown_confirm', answers='y/N'))
+ except NotImplemented:
+ pass
+ else:
+ if i.lower() == 'y' or i.lower() == 'yes':
+ shutdown = True
+
+ if shutdown:
+ operation_logger.start()
+ logger.warn(m18n.n('server_shutdown'))
+ subprocess.check_call(['systemctl', 'poweroff'])
+
+
+@is_unit_operation()
+def tools_reboot(operation_logger, force=False):
+ reboot = force
+ if not reboot:
+ try:
+ # Ask confirmation for restoring
+ i = msignals.prompt(m18n.n('server_reboot_confirm', answers='y/N'))
+ except NotImplemented:
+ pass
+ else:
+ if i.lower() == 'y' or i.lower() == 'yes':
+ reboot = True
+ if reboot:
+ operation_logger.start()
+ logger.warn(m18n.n('server_reboot'))
+ subprocess.check_call(['systemctl', 'reboot'])
+
+
+def tools_migrations_list(pending=False, done=False):
+ """
+ List existing migrations
+ """
+
+ # Check for option conflict
+ if pending and done:
+ raise MoulinetteError(errno.EINVAL, m18n.n("migrations_list_conflict_pending_done"))
+
+ # Get all migrations
+ migrations = _get_migrations_list()
+
+ # If asked, filter pending or done migrations
+ if pending or done:
+ last_migration = tools_migrations_state()["last_run_migration"]
+ last_migration = last_migration["number"] if last_migration else -1
+ if done:
+ migrations = [m for m in migrations if m.number <= last_migration]
+ if pending:
+ migrations = [m for m in migrations if m.number > last_migration]
+
+ # Reduce to dictionnaries
+ migrations = [{ "id": migration.id,
+ "number": migration.number,
+ "name": migration.name,
+ "mode": migration.mode,
+ "description": migration.description,
+ "disclaimer": migration.disclaimer } for migration in migrations ]
+
+ return {"migrations": migrations}
+
+
+def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclaimer=False):
+ """
+ Perform migrations
+ """
+
+ # state is a datastructure that represents the last run migration
+ # it has this form:
+ # {
+ # "last_run_migration": {
+ # "number": "00xx",
+ # "name": "some name",
+ # }
+ # }
+ state = tools_migrations_state()
+
+ last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0
+
+ # load all migrations
+ migrations = _get_migrations_list()
+ migrations = sorted(migrations, key=lambda x: x.number)
+
+ if not migrations:
+ logger.info(m18n.n('migrations_no_migrations_to_run'))
+ return
+
+ all_migration_numbers = [x.number for x in migrations]
+
+ if target is None:
+ target = migrations[-1].number
+
+ # validate input, target must be "0" or a valid number
+ elif target != 0 and target not in all_migration_numbers:
+ raise MoulinetteError(errno.EINVAL, m18n.n('migrations_bad_value_for_target', ", ".join(map(str, all_migration_numbers))))
+
+ logger.debug(m18n.n('migrations_current_target', target))
+
+ # no new migrations to run
+ if target == last_run_migration_number:
+ logger.warn(m18n.n('migrations_no_migrations_to_run'))
+ return
+
+ logger.debug(m18n.n('migrations_show_last_migration', last_run_migration_number))
+
+ # we need to run missing migrations
+ if last_run_migration_number < target:
+ logger.debug(m18n.n('migrations_forward'))
+ # drop all already run migrations
+ migrations = filter(lambda x: target >= x.number > last_run_migration_number, migrations)
+ mode = "forward"
+
+ # we need to go backward on already run migrations
+ elif last_run_migration_number > target:
+ logger.debug(m18n.n('migrations_backward'))
+ # drop all not already run migrations
+ migrations = filter(lambda x: target < x.number <= last_run_migration_number, migrations)
+ mode = "backward"
+
+ else: # can't happen, this case is handle before
+ raise Exception()
+
+ # If we are migrating in "automatic mode" (i.e. from debian
+ # configure during an upgrade of the package) but we are asked to run
+ # migrations is to be ran manually by the user
+ manual_migrations = [m for m in migrations if m.mode == "manual"]
+ if not skip and auto and manual_migrations:
+ for m in manual_migrations:
+ logger.warn(m18n.n('migrations_to_be_ran_manually',
+ number=m.number,
+ name=m.name))
+ return
+
+ # If some migrations have disclaimers, require the --accept-disclaimer
+ # option
+ migrations_with_disclaimer = [m for m in migrations if m.disclaimer]
+ if not skip and not accept_disclaimer and migrations_with_disclaimer:
+ for m in migrations_with_disclaimer:
+ logger.warn(m18n.n('migrations_need_to_accept_disclaimer',
+ number=m.number,
+ name=m.name,
+ disclaimer=m.disclaimer))
+ return
+
+ # effectively run selected migrations
+ for migration in migrations:
+
+ # Start register change on system
+ operation_logger= OperationLogger('tools_migrations_migrate_' + mode)
+ operation_logger.start()
+
+ if not skip:
+
+ logger.warn(m18n.n('migrations_show_currently_running_migration',
+ number=migration.number, name=migration.name))
+
+ try:
+ migration.operation_logger = operation_logger
+ if mode == "forward":
+ migration.migrate()
+ elif mode == "backward":
+ migration.backward()
+ else: # can't happen
+ raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode)
+ except Exception as e:
+ # migration failed, let's stop here but still update state because
+ # we managed to run the previous ones
+ msg = m18n.n('migrations_migration_has_failed',
+ exception=e,
+ number=migration.number,
+ name=migration.name)
+ logger.error(msg, exc_info=1)
+ operation_logger.error(msg)
+ break
+
+ else: # if skip
+ logger.warn(m18n.n('migrations_skip_migration',
+ number=migration.number,
+ name=migration.name))
+
+ # update the state to include the latest run migration
+ state["last_run_migration"] = {
+ "number": migration.number,
+ "name": migration.name
+ }
+
+ operation_logger.success()
+
+ # special case where we want to go back from the start
+ if target == 0:
+ state["last_run_migration"] = None
+
+ write_to_json(MIGRATIONS_STATE_PATH, state)
+
+
+def tools_migrations_state():
+ """
+ Show current migration state
+ """
+ if not os.path.exists(MIGRATIONS_STATE_PATH):
+ return {"last_run_migration": None}
+
+ return read_json(MIGRATIONS_STATE_PATH)
+
+
+def tools_shell(auth, command=None):
+ """
+ Launch an (i)python shell in the YunoHost context.
+
+ This is entirely aim for development.
+ """
+
+ if command:
+ exec(command)
+ return
+
+ logger.warn("The \033[1;34mauth\033[0m is available in this context")
+ try:
+ from IPython import embed
+ embed()
+ except ImportError:
+ logger.warn("You don't have IPython installed, consider installing it as it is way better than the standard shell.")
+ logger.warn("Falling back on the standard shell.")
+
+ import readline # will allow Up/Down/History in the console
+ readline # to please pyflakes
+ import code
+ vars = globals().copy()
+ vars.update(locals())
+ shell = code.InteractiveConsole(vars)
+ shell.interact()
+
+
+def _get_migrations_list():
+ migrations = []
+
+ try:
+ import data_migrations
+ except ImportError:
+ # not data migrations present, return empty list
+ return migrations
+
+ migrations_path = data_migrations.__path__[0]
+
+ if not os.path.exists(migrations_path):
+ logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path))
+ return migrations
+
+ for migration_file in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
+ migrations.append(_load_migration(migration_file))
+
+ return sorted(migrations, key=lambda m: m.id)
+
+
+def _get_migration_by_name(migration_name):
+ """
+ Low-level / "private" function to find a migration by its name
+ """
+
+ try:
+ import data_migrations
+ except ImportError:
+ raise AssertionError("Unable to find migration with name %s" % migration_name)
+
+ migrations_path = data_migrations.__path__[0]
+ migrations_found = filter(lambda x: re.match("^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path))
+
+ assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name
+
+ return _load_migration(migrations_found[0])
+
+
+def _load_migration(migration_file):
+
+ migration_id = migration_file[:-len(".py")]
+
+ number, name = migration_id.split("_", 1)
+
+ logger.debug(m18n.n('migrations_loading_migration',
+ number=number, name=name))
+
+ try:
+ # this is python builtin method to import a module using a name, we
+ # use that to import the migration as a python object so we'll be
+ # able to run it in the next loop
+ module = import_module("yunohost.data_migrations.{}".format(migration_id))
+ return module.MyMigration(migration_id)
+ except Exception:
+ import traceback
+ traceback.print_exc()
+
+ raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration',
+ number=number, name=name))
+
+
+class Migration(object):
+
+ # Those are to be implemented by daughter classes
+
+ mode = "auto"
+
+ def forward(self):
+ raise NotImplementedError()
+
+ def backward(self):
+ pass
+
+ @property
+ def disclaimer(self):
+ return None
+
+ # The followings shouldn't be overriden
+
+ def migrate(self):
+ self.forward()
+
+ def __init__(self, id_):
+ self.id = id_
+ self.number = int(self.id.split("_", 1)[0])
+ self.name = self.id.split("_", 1)[1]
+
+ @property
+ def description(self):
+ return m18n.n("migration_description_%s" % self.id)
diff --git a/src/yunohost/user.py b/src/yunohost/user.py
index 8e2bf4d63..7c5b847a2 100644
--- a/src/yunohost/user.py
+++ b/src/yunohost/user.py
@@ -24,28 +24,31 @@
Manage users
"""
import os
+import re
+import pwd
+import json
+import errno
import crypt
import random
import string
-import json
-import errno
import subprocess
-import math
-import re
import cracklib
+from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
+from yunohost.service import service_status
+from yunohost.log import is_unit_operation
logger = getActionLogger('yunohost.user')
def _check_password(password):
try:
- cracklib.VeryFascistCheck(password)
+ cracklib.VeryFascistCheck(password)
except ValueError as e:
- raise MoulinetteError(errno.EINVAL, m18n.n('password_too_weak') + " : " + str(e) )
+ raise MoulinetteError(errno.EINVAL, m18n.n('password_too_weak') + " : " + str(e) )
-def user_list(auth, fields=None, filter=None, limit=None, offset=None):
+def user_list(auth, fields=None):
"""
List users
@@ -56,21 +59,19 @@ def user_list(auth, fields=None, filter=None, limit=None, offset=None):
fields -- fields to fetch
"""
- user_attrs = { 'uid': 'username',
- 'cn': 'fullname',
- 'mail': 'mail',
- 'maildrop': 'mail-forward',
- 'mailuserquota': 'mailbox-quota' }
- attrs = [ 'uid' ]
+ user_attrs = {
+ 'uid': 'username',
+ 'cn': 'fullname',
+ 'mail': 'mail',
+ 'maildrop': 'mail-forward',
+ 'loginShell': 'shell',
+ 'homeDirectory': 'home_path',
+ 'mailuserquota': 'mailbox-quota'
+ }
+
+ attrs = ['uid']
users = {}
- # Set default arguments values
- if offset is None:
- offset = 0
- if limit is None:
- limit = 1000
- if filter is None:
- filter = '(&(objectclass=person)(!(uid=root))(!(uid=nobody)))'
if fields:
keys = user_attrs.keys()
for attr in fields:
@@ -80,25 +81,33 @@ def user_list(auth, fields=None, filter=None, limit=None, offset=None):
raise MoulinetteError(errno.EINVAL,
m18n.n('field_invalid', attr))
else:
- attrs = [ 'uid', 'cn', 'mail', 'mailuserquota' ]
+ attrs = ['uid', 'cn', 'mail', 'mailuserquota', 'loginShell']
- result = auth.search('ou=users,dc=yunohost,dc=org', filter, attrs)
+ result = auth.search('ou=users,dc=yunohost,dc=org',
+ '(&(objectclass=person)(!(uid=root))(!(uid=nobody)))',
+ attrs)
- if len(result) > offset and limit > 0:
- for user in result[offset:offset+limit]:
- entry = {}
- for attr, values in user.items():
- try:
- entry[user_attrs[attr]] = values[0]
- except:
- pass
- uid = entry[user_attrs['uid']]
- users[uid] = entry
- return { 'users' : users }
+ for user in result:
+ entry = {}
+ for attr, values in user.items():
+ if values:
+ if attr == "loginShell":
+ if values[0].strip() == "/bin/false":
+ entry["ssh_allowed"] = False
+ else:
+ entry["ssh_allowed"] = True
+
+ entry[user_attrs[attr]] = values[0]
+
+ uid = entry[user_attrs['uid']]
+ users[uid] = entry
+
+ return {'users': users}
-def user_create(auth, username, firstname, lastname, mail, password,
- mailbox_quota=0):
+@is_unit_operation([('username', 'user')])
+def user_create(operation_logger, auth, username, firstname, lastname, mail, password,
+ mailbox_quota="0"):
"""
Create user
@@ -111,8 +120,7 @@ def user_create(auth, username, firstname, lastname, mail, password,
mailbox_quota -- Mailbox size quota
"""
- import pwd
- from yunohost.domain import domain_list
+ from yunohost.domain import domain_list, _get_maindomain
from yunohost.hook import hook_callback
from yunohost.app import app_ssowatconf
@@ -121,89 +129,89 @@ def user_create(auth, username, firstname, lastname, mail, password,
# Validate uniqueness of username and mail in LDAP
auth.validate_uniqueness({
- 'uid' : username,
- 'mail' : mail
+ 'uid': username,
+ 'mail': mail
})
# Validate uniqueness of username in system users
- try:
- pwd.getpwnam(username)
- except KeyError:
- pass
- else:
+ all_existing_usernames = {x.pw_name for x in pwd.getpwall()}
+ if username in all_existing_usernames:
raise MoulinetteError(errno.EEXIST, m18n.n('system_username_exists'))
# Check that the mail domain exists
- if mail[mail.find('@')+1:] not in domain_list(auth)['domains']:
+ if mail.split("@")[1] not in domain_list(auth)['domains']:
raise MoulinetteError(errno.EINVAL,
m18n.n('mail_domain_unknown',
- domain=mail[mail.find('@')+1:]))
+ domain=mail.split("@")[1]))
+
+ operation_logger.start()
# Get random UID/GID
- uid_check = gid_check = 0
- while uid_check == 0 and gid_check == 0:
+ all_uid = {x.pw_uid for x in pwd.getpwall()}
+ all_gid = {x.pw_gid for x in pwd.getpwall()}
+
+ uid_guid_found = False
+ while not uid_guid_found:
uid = str(random.randint(200, 99999))
- uid_check = os.system("getent passwd %s" % uid)
- gid_check = os.system("getent group %s" % uid)
+ uid_guid_found = uid not in all_uid and uid not in all_gid
# Adapt values for LDAP
fullname = '%s %s' % (firstname, lastname)
- rdn = 'uid=%s,ou=users' % username
- char_set = string.ascii_uppercase + string.digits
- salt = ''.join(random.sample(char_set,8))
- salt = '$1$' + salt + '$'
- user_pwd = '{CRYPT}' + crypt.crypt(str(password), salt)
attr_dict = {
- 'objectClass' : ['mailAccount', 'inetOrgPerson', 'posixAccount'],
- 'givenName' : firstname,
- 'sn' : lastname,
- 'displayName' : fullname,
- 'cn' : fullname,
- 'uid' : username,
- 'mail' : mail,
- 'maildrop' : username,
- 'mailuserquota' : mailbox_quota,
- 'userPassword' : user_pwd,
- 'gidNumber' : uid,
- 'uidNumber' : uid,
- 'homeDirectory' : '/home/' + username,
- 'loginShell' : '/bin/false'
+ 'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount'],
+ 'givenName': firstname,
+ 'sn': lastname,
+ 'displayName': fullname,
+ 'cn': fullname,
+ 'uid': username,
+ 'mail': mail,
+ 'maildrop': username,
+ 'mailuserquota': mailbox_quota,
+ 'userPassword': _hash_user_password(password),
+ 'gidNumber': uid,
+ 'uidNumber': uid,
+ 'homeDirectory': '/home/' + username,
+ 'loginShell': '/bin/false'
}
# If it is the first user, add some aliases
if not auth.search(base='ou=users,dc=yunohost,dc=org', filter='uid=*'):
- with open('/etc/yunohost/current_host') as f:
- main_domain = f.readline().rstrip()
+ main_domain = _get_maindomain()
aliases = [
- 'root@'+ main_domain,
- 'admin@'+ main_domain,
- 'webmaster@'+ main_domain,
- 'postmaster@'+ main_domain,
+ 'root@' + main_domain,
+ 'admin@' + main_domain,
+ 'webmaster@' + main_domain,
+ 'postmaster@' + main_domain,
]
- attr_dict['mail'] = [ attr_dict['mail'] ] + aliases
+ attr_dict['mail'] = [attr_dict['mail']] + aliases
# If exists, remove the redirection from the SSO
try:
with open('/etc/ssowat/conf.json.persistent') as json_conf:
ssowat_conf = json.loads(str(json_conf.read()))
+ except ValueError as e:
+ raise MoulinetteError(errno.EINVAL,
+ m18n.n('ssowat_persistent_conf_read_error', error=e.strerror))
+ except IOError:
+ ssowat_conf = {}
- if 'redirected_urls' in ssowat_conf and '/' in ssowat_conf['redirected_urls']:
- del ssowat_conf['redirected_urls']['/']
+ if 'redirected_urls' in ssowat_conf and '/' in ssowat_conf['redirected_urls']:
+ del ssowat_conf['redirected_urls']['/']
+ try:
+ with open('/etc/ssowat/conf.json.persistent', 'w+') as f:
+ json.dump(ssowat_conf, f, sort_keys=True, indent=4)
+ except IOError as e:
+ raise MoulinetteError(errno.EPERM,
+ m18n.n('ssowat_persistent_conf_write_error', error=e.strerror))
- with open('/etc/ssowat/conf.json.persistent', 'w+') as f:
- json.dump(ssowat_conf, f, sort_keys=True, indent=4)
-
- except IOError: pass
-
-
- if auth.add(rdn, attr_dict):
+ if auth.add('uid=%s,ou=users' % username, attr_dict):
# Invalidate passwd to take user creation into account
subprocess.call(['nscd', '-i', 'passwd'])
# Update SFTP user group
memberlist = auth.search(filter='cn=sftpusers', attrs=['memberUid'])[0]['memberUid']
memberlist.append(username)
- if auth.update('cn=sftpusers,ou=groups', { 'memberUid': memberlist }):
+ if auth.update('cn=sftpusers,ou=groups', {'memberUid': memberlist}):
try:
# Attempt to create user home folder
subprocess.check_call(
@@ -213,17 +221,18 @@ def user_create(auth, username, firstname, lastname, mail, password,
logger.warning(m18n.n('user_home_creation_failed'),
exc_info=1)
app_ssowatconf(auth)
- #TODO: Send a welcome mail to user
+ # TODO: Send a welcome mail to user
logger.success(m18n.n('user_created'))
hook_callback('post_user_create',
args=[username, mail, password, firstname, lastname])
- return { 'fullname' : fullname, 'username' : username, 'mail' : mail }
+ return {'fullname': fullname, 'username': username, 'mail': mail}
raise MoulinetteError(169, m18n.n('user_creation_failed'))
-def user_delete(auth, username, purge=False):
+@is_unit_operation([('username', 'user')])
+def user_delete(operation_logger, auth, username, purge=False):
"""
Delete user
@@ -235,15 +244,18 @@ def user_delete(auth, username, purge=False):
from yunohost.app import app_ssowatconf
from yunohost.hook import hook_callback
+ operation_logger.start()
if auth.remove('uid=%s,ou=users' % username):
# Invalidate passwd to take user deletion into account
subprocess.call(['nscd', '-i', 'passwd'])
# Update SFTP user group
memberlist = auth.search(filter='cn=sftpusers', attrs=['memberUid'])[0]['memberUid']
- try: memberlist.remove(username)
- except: pass
- if auth.update('cn=sftpusers,ou=groups', { 'memberUid': memberlist }):
+ try:
+ memberlist.remove(username)
+ except:
+ pass
+ if auth.update('cn=sftpusers,ou=groups', {'memberUid': memberlist}):
if purge:
subprocess.call(['rm', '-rf', '/home/{0}'.format(username)])
else:
@@ -256,7 +268,8 @@ def user_delete(auth, username, purge=False):
logger.success(m18n.n('user_deleted'))
-def user_update(auth, username, firstname=None, lastname=None, mail=None,
+@is_unit_operation([('username', 'user')], exclude=['auth', 'change_password'])
+def user_update(operation_logger, auth, username, firstname=None, lastname=None, mail=None,
change_password=None, add_mailforward=None, remove_mailforward=None,
add_mailalias=None, remove_mailalias=None, mailbox_quota=None):
"""
@@ -289,11 +302,11 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
# Get modifications from arguments
if firstname:
- new_attr_dict['givenName'] = firstname # TODO: Validate
+ new_attr_dict['givenName'] = firstname # TODO: Validate
new_attr_dict['cn'] = new_attr_dict['displayName'] = firstname + ' ' + user['sn'][0]
if lastname:
- new_attr_dict['sn'] = lastname # TODO: Validate
+ new_attr_dict['sn'] = lastname # TODO: Validate
new_attr_dict['cn'] = new_attr_dict['displayName'] = user['givenName'][0] + ' ' + lastname
if lastname and firstname:
@@ -303,35 +316,32 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
# Ensure sufficiently complex password
_check_password(change_password)
- char_set = string.ascii_uppercase + string.digits
- salt = ''.join(random.sample(char_set,8))
- salt = '$1$' + salt + '$'
- new_attr_dict['userPassword'] = '{CRYPT}' + crypt.crypt(str(change_password), salt)
+ new_attr_dict['userPassword'] = _hash_user_password(change_password)
if mail:
- auth.validate_uniqueness({ 'mail': mail })
- if mail[mail.find('@')+1:] not in domains:
+ auth.validate_uniqueness({'mail': mail})
+ if mail[mail.find('@') + 1:] not in domains:
raise MoulinetteError(errno.EINVAL,
m18n.n('mail_domain_unknown',
- domain=mail[mail.find('@')+1:]))
+ domain=mail[mail.find('@') + 1:]))
del user['mail'][0]
new_attr_dict['mail'] = [mail] + user['mail']
if add_mailalias:
if not isinstance(add_mailalias, list):
- add_mailalias = [ add_mailalias ]
+ add_mailalias = [add_mailalias]
for mail in add_mailalias:
- auth.validate_uniqueness({ 'mail': mail })
- if mail[mail.find('@')+1:] not in domains:
+ auth.validate_uniqueness({'mail': mail})
+ if mail[mail.find('@') + 1:] not in domains:
raise MoulinetteError(errno.EINVAL,
m18n.n('mail_domain_unknown',
- domain=mail[mail.find('@')+1:]))
+ domain=mail[mail.find('@') + 1:]))
user['mail'].append(mail)
new_attr_dict['mail'] = user['mail']
if remove_mailalias:
if not isinstance(remove_mailalias, list):
- remove_mailalias = [ remove_mailalias ]
+ remove_mailalias = [remove_mailalias]
for mail in remove_mailalias:
if len(user['mail']) > 1 and mail in user['mail'][1:]:
user['mail'].remove(mail)
@@ -342,7 +352,7 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
if add_mailforward:
if not isinstance(add_mailforward, list):
- add_mailforward = [ add_mailforward ]
+ add_mailforward = [add_mailforward]
for mail in add_mailforward:
if mail in user['maildrop'][1:]:
continue
@@ -351,7 +361,7 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
if remove_mailforward:
if not isinstance(remove_mailforward, list):
- remove_mailforward = [ remove_mailforward ]
+ remove_mailforward = [remove_mailforward]
for mail in remove_mailforward:
if len(user['maildrop']) > 1 and mail in user['maildrop'][1:]:
user['maildrop'].remove(mail)
@@ -363,12 +373,14 @@ def user_update(auth, username, firstname=None, lastname=None, mail=None,
if mailbox_quota is not None:
new_attr_dict['mailuserquota'] = mailbox_quota
+ operation_logger.start()
+
if auth.update('uid=%s,ou=users' % username, new_attr_dict):
- logger.success(m18n.n('user_updated'))
- app_ssowatconf(auth)
- return user_info(auth, username)
+ logger.success(m18n.n('user_updated'))
+ app_ssowatconf(auth)
+ return user_info(auth, username)
else:
- raise MoulinetteError(169, m18n.n('user_update_failed'))
+ raise MoulinetteError(169, m18n.n('user_update_failed'))
def user_info(auth, username):
@@ -384,9 +396,9 @@ def user_info(auth, username):
]
if len(username.split('@')) is 2:
- filter = 'mail='+ username
+ filter = 'mail=' + username
else:
- filter = 'uid='+ username
+ filter = 'uid=' + username
result = auth.search('ou=users,dc=yunohost,dc=org', filter, user_attrs)
@@ -410,28 +422,107 @@ def user_info(auth, username):
result_dict['mail-forward'] = user['maildrop'][1:]
if 'mailuserquota' in user:
- if user['mailuserquota'][0] != '0':
- cmd = 'doveadm -f flow quota get -u %s' % user['uid'][0]
- userquota = subprocess.check_output(cmd,stderr=subprocess.STDOUT,
- shell=True)
- quotavalue = re.findall(r'\d+', userquota)
- result = '%s (%s%s)' % ( _convertSize(eval(quotavalue[0])),
- quotavalue[2], '%')
- result_dict['mailbox-quota'] = {
- 'limit' : user['mailuserquota'][0],
- 'use' : result
- }
+ userquota = user['mailuserquota'][0]
+
+ if isinstance(userquota, int):
+ userquota = str(userquota)
+
+ # Test if userquota is '0' or '0M' ( quota pattern is ^(\d+[bkMGT])|0$ )
+ is_limited = not re.match('0[bkMGT]?', userquota)
+ storage_use = '?'
+
+ if service_status("dovecot")["status"] != "running":
+ logger.warning(m18n.n('mailbox_used_space_dovecot_down'))
else:
- result_dict['mailbox-quota'] = m18n.n('unlimit')
-
+ cmd = 'doveadm -f flow quota get -u %s' % user['uid'][0]
+ cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
+ shell=True)
+ # Exemple of return value for cmd:
+ # """Quota name=User quota Type=STORAGE Value=0 Limit=- %=0
+ # Quota name=User quota Type=MESSAGE Value=0 Limit=- %=0"""
+ has_value = re.search(r'Value=(\d+)', cmd_result)
+
+ if has_value:
+ storage_use = int(has_value.group(1))
+ storage_use = _convertSize(storage_use)
+
+ if is_limited:
+ has_percent = re.search(r'%=(\d+)', cmd_result)
+
+ if has_percent:
+ percentage = int(has_percent.group(1))
+ storage_use += ' (%s%%)' % percentage
+
+ result_dict['mailbox-quota'] = {
+ 'limit': userquota if is_limited else m18n.n('unlimit'),
+ 'use': storage_use
+ }
+
if result:
return result_dict
else:
raise MoulinetteError(167, m18n.n('user_info_failed'))
+#
+# SSH subcategory
+#
+#
+import yunohost.ssh
+
+def user_ssh_allow(auth, username):
+ return yunohost.ssh.user_ssh_allow(auth, username)
+
+def user_ssh_disallow(auth, username):
+ return yunohost.ssh.user_ssh_disallow(auth, username)
+
+def user_ssh_list_keys(auth, username):
+ return yunohost.ssh.user_ssh_list_keys(auth, username)
+
+def user_ssh_add_key(auth, username, key, comment):
+ return yunohost.ssh.user_ssh_add_key(auth, username, key, comment)
+
+def user_ssh_remove_key(auth, username, key):
+ return yunohost.ssh.user_ssh_remove_key(auth, username, key)
+
+#
+# End SSH subcategory
+#
+
def _convertSize(num, suffix=''):
- for unit in ['K','M','G','T','P','E','Z']:
+ for unit in ['K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
+
+
+def _hash_user_password(password):
+ """
+ This function computes and return a salted hash for the password in input.
+ This implementation is inspired from [1].
+
+ The hash follows SHA-512 scheme from Linux/glibc.
+ Hence the {CRYPT} and $6$ prefixes
+ - {CRYPT} means it relies on the OS' crypt lib
+ - $6$ corresponds to SHA-512, the strongest hash available on the system
+
+ The salt is generated using random.SystemRandom(). It is the crypto-secure
+ pseudo-random number generator according to the python doc [2] (c.f. the
+ red square). It internally relies on /dev/urandom
+
+ The salt is made of 16 characters from the set [./a-zA-Z0-9]. This is the
+ max sized allowed for salts according to [3]
+
+ [1] https://www.redpill-linpro.com/techblog/2016/08/16/ldap-password-hash.html
+ [2] https://docs.python.org/2/library/random.html
+ [3] https://www.safaribooksonline.com/library/view/practical-unix-and/0596003234/ch04s03.html
+ """
+
+ char_set = string.ascii_uppercase + string.ascii_lowercase + string.digits + "./"
+ salt = ''.join([random.SystemRandom().choice(char_set) for x in range(16)])
+
+ salt = '$6$' + salt + '$'
+ return '{CRYPT}' + crypt.crypt(str(password), salt)
+
+
+
diff --git a/src/yunohost/utils/filesystem.py b/src/yunohost/utils/filesystem.py
new file mode 100644
index 000000000..3f026f980
--- /dev/null
+++ b/src/yunohost/utils/filesystem.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+ Copyright (C) 2018 YUNOHOST.ORG
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+import os
+
+def free_space_in_directory(dirpath):
+ stat = os.statvfs(dirpath)
+ return stat.f_frsize * stat.f_bavail
+
+def space_used_by_directory(dirpath):
+ stat = os.statvfs(dirpath)
+ return stat.f_frsize * stat.f_blocks
diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py
new file mode 100644
index 000000000..a9602ff56
--- /dev/null
+++ b/src/yunohost/utils/network.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+ Copyright (C) 2017 YUNOHOST.ORG
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+import logging
+import re
+import subprocess
+from moulinette.utils.network import download_text
+
+logger = logging.getLogger('yunohost.utils.network')
+
+
+def get_public_ip(protocol=4):
+ """Retrieve the public IP address from ip.yunohost.org"""
+
+ if protocol == 4:
+ url = 'https://ip.yunohost.org'
+ elif protocol == 6:
+ url = 'https://ip6.yunohost.org'
+ else:
+ raise ValueError("invalid protocol version")
+
+ try:
+ return download_text(url, timeout=30).strip()
+ except Exception as e:
+ logger.debug("Could not get public IPv%s : %s" % (str(protocol), str(e)))
+ return None
+
+
+def get_network_interfaces():
+
+ # Get network devices and their addresses (raw infos from 'ip addr')
+ devices_raw = {}
+ output = subprocess.check_output('ip addr show'.split())
+ for d in re.split('^(?:[0-9]+: )', output, flags=re.MULTILINE):
+ # Extract device name (1) and its addresses (2)
+ m = re.match('([^\s@]+)(?:@[\S]+)?: (.*)', d, flags=re.DOTALL)
+ if m:
+ devices_raw[m.group(1)] = m.group(2)
+
+ # Parse relevant informations for each of them
+ devices = {name: _extract_inet(addrs) for name, addrs in devices_raw.items() if name != "lo"}
+
+ return devices
+
+
+def get_gateway():
+
+ output = subprocess.check_output('ip route show'.split())
+ m = re.search('default via (.*) dev ([a-z]+[0-9]?)', output)
+ if not m:
+ return None
+
+ addr = _extract_inet(m.group(1), True)
+ return addr.popitem()[1] if len(addr) == 1 else None
+
+
+###############################################################################
+
+
+def _extract_inet(string, skip_netmask=False, skip_loopback=True):
+ """
+ Extract IP addresses (v4 and/or v6) from a string limited to one
+ address by protocol
+
+ Keyword argument:
+ string -- String to search in
+ skip_netmask -- True to skip subnet mask extraction
+ skip_loopback -- False to include addresses reserved for the
+ loopback interface
+
+ Returns:
+ A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6'
+
+ """
+ ip4_pattern = '((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}'
+ ip6_pattern = '(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)'
+ ip4_pattern += '/[0-9]{1,2})' if not skip_netmask else ')'
+ ip6_pattern += '/[0-9]{1,3})' if not skip_netmask else ')'
+ result = {}
+
+ for m in re.finditer(ip4_pattern, string):
+ addr = m.group(1)
+ if skip_loopback and addr.startswith('127.'):
+ continue
+
+ # Limit to only one result
+ result['ipv4'] = addr
+ break
+
+ for m in re.finditer(ip6_pattern, string):
+ addr = m.group(1)
+ if skip_loopback and addr == '::1':
+ continue
+
+ # Limit to only one result
+ result['ipv6'] = addr
+ break
+
+ return result
diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py
index 5be2103e5..3917ef563 100644
--- a/src/yunohost/utils/packages.py
+++ b/src/yunohost/utils/packages.py
@@ -25,6 +25,8 @@ from collections import OrderedDict
import apt
from apt_pkg import version_compare
+from moulinette import m18n
+
logger = logging.getLogger('yunohost.utils.packages')
@@ -404,6 +406,7 @@ def get_installed_version(*pkgnames, **kwargs):
# Retrieve options
as_dict = kwargs.get('as_dict', False)
strict = kwargs.get('strict', False)
+ with_repo = kwargs.get('with_repo', False)
for pkgname in pkgnames:
try:
@@ -412,18 +415,38 @@ def get_installed_version(*pkgnames, **kwargs):
if strict:
raise UnknownPackage(pkgname)
logger.warning(m18n.n('package_unknown', pkgname=pkgname))
+ continue
+
try:
version = pkg.installed.version
except AttributeError:
if strict:
raise UninstalledPackage(pkgname)
version = None
- versions[pkgname] = version
+
+ try:
+ # stable, testing, unstable
+ repo = pkg.installed.origins[0].component
+ except AttributeError:
+ if strict:
+ raise UninstalledPackage(pkgname)
+ repo = ""
+
+ if with_repo:
+ versions[pkgname] = {
+ "version": version,
+ # when we don't have component it's because it's from a local
+ # install or from an image (like in vagrant)
+ "repo": repo if repo else "local",
+ }
+ else:
+ versions[pkgname] = version
if len(pkgnames) == 1 and not as_dict:
return versions[pkgnames[0]]
return versions
+
def meets_version_specifier(pkgname, specifier):
"""Check if a package installed version meets specifier"""
spec = SpecifierSet(specifier)
@@ -433,7 +456,11 @@ def meets_version_specifier(pkgname, specifier):
# YunoHost related methods ---------------------------------------------------
def ynh_packages_version(*args, **kwargs):
+ # from cli the received arguments are:
+ # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {}
+ # they don't seem to serve any purpose
"""Return the version of each YunoHost package"""
return get_installed_version(
'yunohost', 'yunohost-admin', 'moulinette', 'ssowat',
+ with_repo=True
)
diff --git a/src/yunohost/utils/yunopaste.py b/src/yunohost/utils/yunopaste.py
new file mode 100644
index 000000000..2b53062d1
--- /dev/null
+++ b/src/yunohost/utils/yunopaste.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+import errno
+
+from moulinette.core import MoulinetteError
+
+def yunopaste(data):
+
+ paste_server = "https://paste.yunohost.org"
+
+ try:
+ r = requests.post("%s/documents" % paste_server, data=data, timeout=30)
+ except Exception as e:
+ raise MoulinetteError(errno.EIO,
+ "Something wrong happened while trying to paste data on paste.yunohost.org : %s" % str(e))
+
+ if r.status_code != 200:
+ raise MoulinetteError(errno.EIO,
+ "Something wrong happened while trying to paste data on paste.yunohost.org : %s, %s" % (r.status_code, r.text))
+
+ try:
+ url = json.loads(r.text)["key"]
+ except:
+ raise MoulinetteError(errno.EIO,
+ "Uhoh, couldn't parse the answer from paste.yunohost.org : %s" % r.text)
+
+ return "%s/raw/%s" % (paste_server, url)
diff --git a/src/yunohost/vendor/__init__.py b/src/yunohost/vendor/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/yunohost/vendor/acme_tiny/__init__.py b/src/yunohost/vendor/acme_tiny/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/yunohost/vendor/acme_tiny/acme_tiny.py b/src/yunohost/vendor/acme_tiny/acme_tiny.py
new file mode 100644
index 000000000..f36aef877
--- /dev/null
+++ b/src/yunohost/vendor/acme_tiny/acme_tiny.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+import argparse, subprocess, json, os, sys, base64, binascii, time, hashlib, re, copy, textwrap, logging
+try:
+ from urllib.request import urlopen # Python 3
+except ImportError:
+ from urllib2 import urlopen # Python 2
+
+#DEFAULT_CA = "https://acme-staging.api.letsencrypt.org"
+DEFAULT_CA = "https://acme-v01.api.letsencrypt.org"
+
+LOGGER = logging.getLogger(__name__)
+LOGGER.addHandler(logging.StreamHandler())
+LOGGER.setLevel(logging.INFO)
+
+def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=False):
+ # helper function base64 encode for jose spec
+ def _b64(b):
+ return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "")
+
+ # parse account key to get public key
+ log.info("Parsing account key...")
+ proc = subprocess.Popen(["openssl", "rsa", "-in", account_key, "-noout", "-text"],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ raise IOError("OpenSSL Error: {0}".format(err))
+ pub_hex, pub_exp = re.search(
+ r"modulus:\n\s+00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)",
+ out.decode('utf8'), re.MULTILINE|re.DOTALL).groups()
+ pub_exp = "{0:x}".format(int(pub_exp))
+ pub_exp = "0{0}".format(pub_exp) if len(pub_exp) % 2 else pub_exp
+ header = {
+ "alg": "RS256",
+ "jwk": {
+ "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))),
+ "kty": "RSA",
+ "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))),
+ },
+ }
+ accountkey_json = json.dumps(header['jwk'], sort_keys=True, separators=(',', ':'))
+ thumbprint = _b64(hashlib.sha256(accountkey_json.encode('utf8')).digest())
+
+ # helper function make signed requests
+ def _send_signed_request(url, payload):
+ payload64 = _b64(json.dumps(payload).encode('utf8'))
+ protected = copy.deepcopy(header)
+ protected["nonce"] = urlopen(CA + "/directory").headers['Replay-Nonce']
+ protected64 = _b64(json.dumps(protected).encode('utf8'))
+ proc = subprocess.Popen(["openssl", "dgst", "-sha256", "-sign", account_key],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate("{0}.{1}".format(protected64, payload64).encode('utf8'))
+ if proc.returncode != 0:
+ raise IOError("OpenSSL Error: {0}".format(err))
+ data = json.dumps({
+ "header": header, "protected": protected64,
+ "payload": payload64, "signature": _b64(out),
+ })
+ try:
+ resp = urlopen(url, data.encode('utf8'))
+ return resp.getcode(), resp.read()
+ except IOError as e:
+ return getattr(e, "code", None), getattr(e, "read", e.__str__)()
+
+ # find domains
+ log.info("Parsing CSR...")
+ proc = subprocess.Popen(["openssl", "req", "-in", csr, "-noout", "-text"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ raise IOError("Error loading {0}: {1}".format(csr, err))
+ domains = set([])
+ common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8'))
+ if common_name is not None:
+ domains.add(common_name.group(1))
+ subject_alt_names = re.search(r"X509v3 Subject Alternative Name: \n +([^\n]+)\n", out.decode('utf8'), re.MULTILINE|re.DOTALL)
+ if subject_alt_names is not None:
+ for san in subject_alt_names.group(1).split(", "):
+ if san.startswith("DNS:"):
+ domains.add(san[4:])
+
+ # get the certificate domains and expiration
+ log.info("Registering account...")
+ code, result = _send_signed_request(CA + "/acme/new-reg", {
+ "resource": "new-reg",
+ "agreement": json.loads(urlopen(CA + "/directory").read().decode('utf8'))['meta']['terms-of-service'],
+ })
+ if code == 201:
+ log.info("Registered!")
+ elif code == 409:
+ log.info("Already registered!")
+ else:
+ raise ValueError("Error registering: {0} {1}".format(code, result))
+
+ # verify each domain
+ for domain in domains:
+ log.info("Verifying {0}...".format(domain))
+
+ # get new challenge
+ code, result = _send_signed_request(CA + "/acme/new-authz", {
+ "resource": "new-authz",
+ "identifier": {"type": "dns", "value": domain},
+ })
+ if code != 201:
+ raise ValueError("Error requesting challenges: {0} {1}".format(code, result))
+
+ # make the challenge file
+ challenge = [c for c in json.loads(result.decode('utf8'))['challenges'] if c['type'] == "http-01"][0]
+ token = re.sub(r"[^A-Za-z0-9_\-]", "_", challenge['token'])
+ keyauthorization = "{0}.{1}".format(token, thumbprint)
+ wellknown_path = os.path.join(acme_dir, token)
+ with open(wellknown_path, "w") as wellknown_file:
+ wellknown_file.write(keyauthorization)
+
+ if not no_checks: # sometime the local g
+ # check that the file is in place
+ wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token)
+ try:
+ resp = urlopen(wellknown_url)
+ resp_data = resp.read().decode('utf8').strip()
+ assert resp_data == keyauthorization
+ except (IOError, AssertionError):
+ os.remove(wellknown_path)
+ raise ValueError("Wrote file to {0}, but couldn't download {1}".format(
+ wellknown_path, wellknown_url))
+
+ # notify challenge are met
+ code, result = _send_signed_request(challenge['uri'], {
+ "resource": "challenge",
+ "keyAuthorization": keyauthorization,
+ })
+ if code != 202:
+ raise ValueError("Error triggering challenge: {0} {1}".format(code, result))
+
+ # wait for challenge to be verified
+ while True:
+ try:
+ resp = urlopen(challenge['uri'])
+ challenge_status = json.loads(resp.read().decode('utf8'))
+ except IOError as e:
+ raise ValueError("Error checking challenge: {0} {1}".format(
+ e.code, json.loads(e.read().decode('utf8'))))
+ if challenge_status['status'] == "pending":
+ time.sleep(2)
+ elif challenge_status['status'] == "valid":
+ log.info("{0} verified!".format(domain))
+ os.remove(wellknown_path)
+ break
+ else:
+ raise ValueError("{0} challenge did not pass: {1}".format(
+ domain, challenge_status))
+
+ # get the new certificate
+ log.info("Signing certificate...")
+ proc = subprocess.Popen(["openssl", "req", "-in", csr, "-outform", "DER"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ csr_der, err = proc.communicate()
+ code, result = _send_signed_request(CA + "/acme/new-cert", {
+ "resource": "new-cert",
+ "csr": _b64(csr_der),
+ })
+ if code != 201:
+ raise ValueError("Error signing certificate: {0} {1}".format(code, result))
+
+ # return signed certificate!
+ log.info("Certificate signed!")
+ return """-----BEGIN CERTIFICATE-----\n{0}\n-----END CERTIFICATE-----\n""".format(
+ "\n".join(textwrap.wrap(base64.b64encode(result).decode('utf8'), 64)))
+
+def main(argv):
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.dedent("""\
+ This script automates the process of getting a signed TLS certificate from
+ Let's Encrypt using the ACME protocol. It will need to be run on your server
+ and have access to your private account key, so PLEASE READ THROUGH IT! It's
+ only ~200 lines, so it won't take long.
+
+ ===Example Usage===
+ python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > signed.crt
+ ===================
+
+ ===Example Crontab Renewal (once per month)===
+ 0 0 1 * * python /path/to/acme_tiny.py --account-key /path/to/account.key --csr /path/to/domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > /path/to/signed.crt 2>> /var/log/acme_tiny.log
+ ==============================================
+ """)
+ )
+ parser.add_argument("--account-key", required=True, help="path to your Let's Encrypt account private key")
+ parser.add_argument("--csr", required=True, help="path to your certificate signing request")
+ parser.add_argument("--acme-dir", required=True, help="path to the .well-known/acme-challenge/ directory")
+ parser.add_argument("--quiet", action="store_const", const=logging.ERROR, help="suppress output except for errors")
+ parser.add_argument("--ca", default=DEFAULT_CA, help="certificate authority, default is Let's Encrypt")
+
+ args = parser.parse_args(argv)
+ LOGGER.setLevel(args.quiet or LOGGER.level)
+ signed_crt = get_crt(args.account_key, args.csr, args.acme_dir, log=LOGGER, CA=args.ca)
+ sys.stdout.write(signed_crt)
+
+if __name__ == "__main__": # pragma: no cover
+ main(sys.argv[1:])
diff --git a/src/yunohost/vendor/spectre-meltdown-checker/LICENSE b/src/yunohost/vendor/spectre-meltdown-checker/LICENSE
new file mode 100644
index 000000000..94a9ed024
--- /dev/null
+++ b/src/yunohost/vendor/spectre-meltdown-checker/LICENSE
@@ -0,0 +1,674 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ Copyright (C)
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+.
diff --git a/src/yunohost/vendor/spectre-meltdown-checker/README.md b/src/yunohost/vendor/spectre-meltdown-checker/README.md
new file mode 100644
index 000000000..4a9c71828
--- /dev/null
+++ b/src/yunohost/vendor/spectre-meltdown-checker/README.md
@@ -0,0 +1,88 @@
+Spectre & Meltdown Checker
+==========================
+
+A shell script to tell if your system is vulnerable against the 3 "speculative execution" CVEs that were made public early 2018.
+
+Supported operating systems:
+- Linux (all versions, flavors and distros)
+- BSD (FreeBSD, NetBSD, DragonFlyBSD)
+
+Supported architectures:
+- x86 (32 bits)
+- amd64/x86_64 (64 bits)
+- ARM and ARM64
+- other architectures will work, but mitigations (if they exist) might not always be detected
+
+For Linux systems, the script will detect mitigations, including backported non-vanilla patches, regardless of the advertised kernel version number and the distribution (such as Debian, Ubuntu, CentOS, RHEL, Fedora, openSUSE, Arch, ...), it also works if you've compiled your own kernel.
+
+For BSD systems, the detection will work as long as the BSD you're using supports `cpuctl` and `linprocfs` (this is not the case of OpenBSD for example).
+
+## Easy way to run the script
+
+- Get the latest version of the script using `curl` *or* `wget`
+
+```bash
+curl -L https://meltdown.ovh -o spectre-meltdown-checker.sh
+wget https://meltdown.ovh -O spectre-meltdown-checker.sh
+```
+
+- Inspect the script. You never blindly run scripts you downloaded from the Internet, do you?
+
+```bash
+vim spectre-meltdown-checker.sh
+```
+
+- When you're ready, run the script as root
+
+```bash
+chmod +x spectre-meltdown-checker.sh
+sudo ./spectre-meltdown-checker.sh
+```
+
+## Example of script output
+
+- Intel Haswell CPU running under Ubuntu 16.04 LTS
+
+
+
+- AMD Ryzen running under OpenSUSE Tumbleweed
+
+
+
+- Batch mode (JSON flavor)
+
+
+
+## Quick summary of the CVEs
+
+**CVE-2017-5753** bounds check bypass (Spectre Variant 1)
+
+ - Impact: Kernel & all software
+ - Mitigation: recompile software *and* kernel with a modified compiler that introduces the LFENCE opcode at the proper positions in the resulting code
+ - Performance impact of the mitigation: negligible
+
+**CVE-2017-5715** branch target injection (Spectre Variant 2)
+
+ - Impact: Kernel
+ - Mitigation 1: new opcode via microcode update that should be used by up to date compilers to protect the BTB (by flushing indirect branch predictors)
+ - Mitigation 2: introducing "retpoline" into compilers, and recompile software/OS with it
+ - Performance impact of the mitigation: high for mitigation 1, medium for mitigation 2, depending on your CPU
+
+**CVE-2017-5754** rogue data cache load (Meltdown)
+
+ - Impact: Kernel
+ - Mitigation: updated kernel (with PTI/KPTI patches), updating the kernel is enough
+ - Performance impact of the mitigation: low to medium
+
+## Disclaimer
+
+This tool does its best to determine whether your system is immune (or has proper mitigations in place) for the collectively named "speculative execution" vulnerabilities. It doesn't attempt to run any kind of exploit, and can't guarantee that your system is secure, but rather helps you verifying whether your system has the known correct mitigations in place.
+However, some mitigations could also exist in your kernel that this script doesn't know (yet) how to detect, or it might falsely detect mitigations that in the end don't work as expected (for example, on backported or modified kernels).
+
+Your system exposure also depends on your CPU. As of now, AMD and ARM processors are marked as immune to some or all of these vulnerabilities (except some specific ARM models). All Intel processors manufactured since circa 1995 are thought to be vulnerable, except some specific/old models, such as some early Atoms. Whatever processor one uses, one might seek more information from the manufacturer of that processor and/or of the device in which it runs.
+
+The nature of the discovered vulnerabilities being quite new, the landscape of vulnerable processors can be expected to change over time, which is why this script makes the assumption that all CPUs are vulnerable, except if the manufacturer explicitly stated otherwise in a verifiable public announcement.
+
+Please also note that for Spectre vulnerabilities, all software can possibly be exploited, this tool only verifies that the kernel (which is the core of the system) you're using has the proper protections in place. Verifying all the other software is out of the scope of this tool. As a general measure, ensure you always have the most up to date stable versions of all the software you use, especially for those who are exposed to the world, such as network daemons and browsers.
+
+This tool has been released in the hope that it'll be useful, but don't use it to jump to conclusions about your security.
diff --git a/src/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh b/src/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh
new file mode 100755
index 000000000..0f3c10575
--- /dev/null
+++ b/src/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh
@@ -0,0 +1,2855 @@
+#! /bin/sh
+# Spectre & Meltdown checker
+#
+# Check for the latest version at:
+# https://github.com/speed47/spectre-meltdown-checker
+# git clone https://github.com/speed47/spectre-meltdown-checker.git
+# or wget https://meltdown.ovh -O spectre-meltdown-checker.sh
+# or curl -L https://meltdown.ovh -o spectre-meltdown-checker.sh
+#
+# Stephane Lesimple
+#
+VERSION='0.37'
+
+trap 'exit_cleanup' EXIT
+trap '_warn "interrupted, cleaning up..."; exit_cleanup; exit 1' INT
+exit_cleanup()
+{
+ # cleanup the temp decompressed config & kernel image
+ [ -n "$dumped_config" ] && [ -f "$dumped_config" ] && rm -f "$dumped_config"
+ [ -n "$kerneltmp" ] && [ -f "$kerneltmp" ] && rm -f "$kerneltmp"
+ [ -n "$kerneltmp2" ] && [ -f "$kerneltmp2" ] && rm -f "$kerneltmp2"
+ [ "$mounted_debugfs" = 1 ] && umount /sys/kernel/debug 2>/dev/null
+ [ "$mounted_procfs" = 1 ] && umount "$procfs" 2>/dev/null
+ [ "$insmod_cpuid" = 1 ] && rmmod cpuid 2>/dev/null
+ [ "$insmod_msr" = 1 ] && rmmod msr 2>/dev/null
+ [ "$kldload_cpuctl" = 1 ] && kldunload cpuctl 2>/dev/null
+}
+
+show_usage()
+{
+ # shellcheck disable=SC2086
+ cat <] [--config ] [--map ]
+
+ Modes:
+ Two modes are available.
+
+ First mode is the "live" mode (default), it does its best to find information about the currently running kernel.
+ To run under this mode, just start the script without any option (you can also use --live explicitly)
+
+ Second mode is the "offline" mode, where you can inspect a non-running kernel.
+ You'll need to specify the location of the kernel file, config and System.map files:
+
+ --kernel kernel_file specify a (possibly compressed) Linux or BSD kernel file
+ --config kernel_config specify a kernel config file (Linux only)
+ --map kernel_map_file specify a kernel System.map file (Linux only)
+
+ Options:
+ --no-color don't use color codes
+ --verbose, -v increase verbosity level, possibly several times
+ --no-explain don't produce a human-readable explanation of actions to take to mitigate a vulnerability
+ --paranoid require IBPB to deem Variant 2 as mitigated
+
+ --no-sysfs don't use the /sys interface even if present [Linux]
+ --sysfs-only only use the /sys interface, don't run our own checks [Linux]
+ --coreos special mode for CoreOS (use an ephemeral toolbox to inspect kernel) [Linux]
+
+ --arch-prefix PREFIX specify a prefix for cross-inspecting a kernel of a different arch, for example "aarch64-linux-gnu-",
+ so that invoked tools will be prefixed with this (i.e. aarch64-linux-gnu-objdump)
+ --batch text produce machine readable output, this is the default if --batch is specified alone
+ --batch json produce JSON output formatted for Puppet, Ansible, Chef...
+ --batch nrpe produce machine readable output formatted for NRPE
+ --batch prometheus produce output for consumption by prometheus-node-exporter
+
+ --variant [1,2,3] specify which variant you'd like to check, by default all variants are checked,
+ can be specified multiple times (e.g. --variant 2 --variant 3)
+ --hw-only only check for CPU information, don't check for any variant
+ --no-hw skip CPU information and checks, if you're inspecting a kernel not to be run on this host
+
+ Return codes:
+ 0 (not vulnerable), 2 (vulnerable), 3 (unknown), 255 (error)
+
+ IMPORTANT:
+ A false sense of security is worse than no security at all.
+ Please use the --disclaimer option to understand exactly what this script does.
+
+EOF
+}
+
+show_disclaimer()
+{
+ cat </dev/null 2>&1; then
+ echo_cmd=$(which printf)
+ echo_cmd_type=printf
+elif which echo >/dev/null 2>&1; then
+ echo_cmd=$(which echo)
+else
+ # which command is broken?
+ [ -x /bin/echo ] && echo_cmd=/bin/echo
+ # for Android
+ [ -x /system/bin/echo ] && echo_cmd=/system/bin/echo
+fi
+# still empty ? fallback to builtin
+[ -z "$echo_cmd" ] && echo_cmd=echo
+__echo()
+{
+ opt="$1"
+ shift
+ _msg="$*"
+
+ if [ "$opt_no_color" = 1 ] ; then
+ # strip ANSI color codes
+ # some sed versions (i.e. toybox) can't seem to handle
+ # \033 aka \x1B correctly, so do it for them.
+ if [ "$echo_cmd_type" = printf ]; then
+ _interpret_chars=''
+ else
+ _interpret_chars='-e'
+ fi
+ _ctrlchar=$($echo_cmd $_interpret_chars "\033")
+ _msg=$($echo_cmd $_interpret_chars "$_msg" | sed -r "s/$_ctrlchar\[([0-9][0-9]?(;[0-9][0-9]?)?)?m//g")
+ fi
+ if [ "$echo_cmd_type" = printf ]; then
+ if [ "$opt" = "-n" ]; then
+ $echo_cmd "$_msg"
+ else
+ $echo_cmd "$_msg\n"
+ fi
+ else
+ # shellcheck disable=SC2086
+ $echo_cmd $opt -e "$_msg"
+ fi
+}
+
+_echo()
+{
+ if [ "$opt_verbose" -ge "$1" ]; then
+ shift
+ __echo '' "$*"
+ fi
+}
+
+_echo_nol()
+{
+ if [ "$opt_verbose" -ge "$1" ]; then
+ shift
+ __echo -n "$*"
+ fi
+}
+
+_warn()
+{
+ _echo 0 "\033[31m$*\033[0m" >&2
+}
+
+_info()
+{
+ _echo 1 "$*"
+}
+
+_info_nol()
+{
+ _echo_nol 1 "$*"
+}
+
+_verbose()
+{
+ _echo 2 "$*"
+}
+
+_verbose_nol()
+{
+ _echo_nol 2 "$*"
+}
+
+_debug()
+{
+ _echo 3 "\033[34m(debug) $*\033[0m"
+}
+
+explain()
+{
+ if [ "$opt_no_explain" != 1 ] ; then
+ _info ''
+ _info "> \033[41m\033[30mHow to fix:\033[0m $*"
+ fi
+}
+
+is_cpu_vulnerable_cached=0
+_is_cpu_vulnerable_cached()
+{
+ # shellcheck disable=SC2086
+ [ "$1" = 1 ] && return $variant1
+ # shellcheck disable=SC2086
+ [ "$1" = 2 ] && return $variant2
+ # shellcheck disable=SC2086
+ [ "$1" = 3 ] && return $variant3
+ echo "$0: error: invalid variant '$1' passed to is_cpu_vulnerable()" >&2
+ exit 255
+}
+
+is_cpu_vulnerable()
+{
+ # param: 1, 2 or 3 (variant)
+ # returns 0 if vulnerable, 1 if not vulnerable
+ # (note that in shell, a return of 0 is success)
+ # by default, everything is vulnerable, we work in a "whitelist" logic here.
+ # usage: is_cpu_vulnerable 2 && do something if vulnerable
+ if [ "$is_cpu_vulnerable_cached" = 1 ]; then
+ _is_cpu_vulnerable_cached "$1"
+ return $?
+ fi
+
+ variant1=''
+ variant2=''
+ variant3=''
+
+ if is_cpu_specex_free; then
+ variant1=immune
+ variant2=immune
+ variant3=immune
+ elif is_intel; then
+ # Intel
+ # https://github.com/crozone/SpectrePoC/issues/1 ^F E5200 => spectre 2 not vulnerable
+ # https://github.com/paboldin/meltdown-exploit/issues/19 ^F E5200 => meltdown vulnerable
+ # model name : Pentium(R) Dual-Core CPU E5200 @ 2.50GHz
+ if grep -qE '^model name.+ Pentium\(R\) Dual-Core[[:space:]]+CPU[[:space:]]+E[0-9]{4}K? ' "$procfs/cpuinfo"; then
+ variant1=vuln
+ [ -z "$variant2" ] && variant2=immune
+ variant3=vuln
+ fi
+ if [ "$capabilities_rdcl_no" = 1 ]; then
+ # capability bit for future Intel processor that will explicitly state
+ # that they're not vulnerable to Meltdown
+ # this var is set in check_cpu()
+ variant3=immune
+ _debug "is_cpu_vulnerable: RDCL_NO is set so not vuln to meltdown"
+ fi
+ elif is_amd; then
+ # AMD revised their statement about variant2 => vulnerable
+ # https://www.amd.com/en/corporate/speculative-execution
+ variant1=vuln
+ variant2=vuln
+ [ -z "$variant3" ] && variant3=immune
+ elif [ "$cpu_vendor" = ARM ]; then
+ # ARM
+ # reference: https://developer.arm.com/support/security-update
+ # some devices (phones or other) have several ARMs and as such different part numbers,
+ # an example is "bigLITTLE". we shouldn't rely on the first CPU only, so we check the whole list
+ i=0
+ for cpupart in $cpu_part_list
+ do
+ i=$(( i + 1 ))
+ # do NOT quote $cpu_arch_list below
+ # shellcheck disable=SC2086
+ cpuarch=$(echo $cpu_arch_list | awk '{ print $'$i' }')
+ _debug "checking cpu$i: <$cpupart> <$cpuarch>"
+ # some kernels report AArch64 instead of 8
+ [ "$cpuarch" = "AArch64" ] && cpuarch=8
+ if [ -n "$cpupart" ] && [ -n "$cpuarch" ]; then
+ # Cortex-R7 and Cortex-R8 are real-time and only used in medical devices or such
+ # I can't find their CPU part number, but it's probably not that useful anyway
+ # model R7 R8 A9 A15 A17 A57 A72 A73 A75
+ # part ? ? 0xc09 0xc0f 0xc0e 0xd07 0xd08 0xd09 0xd0a
+ # arch 7? 7? 7 7 7 8 8 8 8
+ #
+ # variant 1 & variant 2
+ if [ "$cpuarch" = 7 ] && echo "$cpupart" | grep -Eq '^0x(c09|c0f|c0e)$'; then
+ # armv7 vulnerable chips
+ _debug "checking cpu$i: this armv7 vulnerable to spectre 1 & 2"
+ variant1=vuln
+ variant2=vuln
+ elif [ "$cpuarch" = 8 ] && echo "$cpupart" | grep -Eq '^0x(d07|d08|d09|d0a)$'; then
+ # armv8 vulnerable chips
+ _debug "checking cpu$i: this armv8 vulnerable to spectre 1 & 2"
+ variant1=vuln
+ variant2=vuln
+ else
+ _debug "checking cpu$i: this arm non vulnerable to 1 & 2"
+ # others are not vulnerable
+ [ -z "$variant1" ] && variant1=immune
+ [ -z "$variant2" ] && variant2=immune
+ fi
+
+ # for variant3, only A75 is vulnerable
+ if [ "$cpuarch" = 8 ] && [ "$cpupart" = 0xd0a ]; then
+ _debug "checking cpu$i: arm A75 vulnerable to meltdown"
+ variant3=vuln
+ else
+ _debug "checking cpu$i: this arm non vulnerable to meltdown"
+ [ -z "$variant3" ] && variant3=immune
+ fi
+ fi
+ _debug "is_cpu_vulnerable: for cpu$i and so far, we have <$variant1> <$variant2> <$variant3>"
+ done
+ fi
+ _debug "is_cpu_vulnerable: temp results are <$variant1> <$variant2> <$variant3>"
+ # if at least one of the cpu is vulnerable, then the system is vulnerable
+ [ "$variant1" = "immune" ] && variant1=1 || variant1=0
+ [ "$variant2" = "immune" ] && variant2=1 || variant2=0
+ [ "$variant3" = "immune" ] && variant3=1 || variant3=0
+ _debug "is_cpu_vulnerable: final results are <$variant1> <$variant2> <$variant3>"
+ is_cpu_vulnerable_cached=1
+ _is_cpu_vulnerable_cached "$1"
+ return $?
+}
+
+is_cpu_specex_free()
+{
+ # return true (0) if the CPU doesn't do speculative execution, false (1) if it does.
+ # if it's not in the list we know, return false (1).
+ # source: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/kernel/cpu/common.c#n882
+ # { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CEDARVIEW, X86_FEATURE_ANY },
+ # { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CLOVERVIEW, X86_FEATURE_ANY },
+ # { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_LINCROFT, X86_FEATURE_ANY },
+ # { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PENWELL, X86_FEATURE_ANY },
+ # { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PINEVIEW, X86_FEATURE_ANY },
+ # { X86_VENDOR_CENTAUR, 5 },
+ # { X86_VENDOR_INTEL, 5 },
+ # { X86_VENDOR_NSC, 5 },
+ # { X86_VENDOR_ANY, 4 },
+ parse_cpu_details
+ if is_intel; then
+ if [ "$cpu_family" = 6 ]; then
+ if [ "$cpu_model" = "$INTEL_FAM6_ATOM_CEDARVIEW" ] || \
+ [ "$cpu_model" = "$INTEL_FAM6_ATOM_CLOVERVIEW" ] || \
+ [ "$cpu_model" = "$INTEL_FAM6_ATOM_LINCROFT" ] || \
+ [ "$cpu_model" = "$INTEL_FAM6_ATOM_PENWELL" ] || \
+ [ "$cpu_model" = "$INTEL_FAM6_ATOM_PINEVIEW" ]; then
+ return 0
+ fi
+ elif [ "$cpu_family" = 5 ]; then
+ return 0
+ fi
+ fi
+ [ "$cpu_family" = 4 ] && return 0
+ return 1
+}
+
+show_header()
+{
+ _info "Spectre and Meltdown mitigation detection tool v$VERSION"
+ _info
+}
+
+parse_opt_file()
+{
+ # parse_opt_file option_name option_value
+ option_name="$1"
+ option_value="$2"
+ if [ -z "$option_value" ]; then
+ show_header
+ show_usage
+ echo "$0: error: --$option_name expects one parameter (a file)" >&2
+ exit 1
+ elif [ ! -e "$option_value" ]; then
+ show_header
+ echo "$0: error: couldn't find file $option_value" >&2
+ exit 1
+ elif [ ! -f "$option_value" ]; then
+ show_header
+ echo "$0: error: $option_value is not a file" >&2
+ exit 1
+ elif [ ! -r "$option_value" ]; then
+ show_header
+ echo "$0: error: couldn't read $option_value (are you root?)" >&2
+ exit 1
+ fi
+ echo "$option_value"
+ exit 0
+}
+
+while [ -n "$1" ]; do
+ if [ "$1" = "--kernel" ]; then
+ opt_kernel=$(parse_opt_file kernel "$2"); ret=$?
+ [ $ret -ne 0 ] && exit 255
+ shift 2
+ opt_live=0
+ elif [ "$1" = "--config" ]; then
+ opt_config=$(parse_opt_file config "$2"); ret=$?
+ [ $ret -ne 0 ] && exit 255
+ shift 2
+ opt_live=0
+ elif [ "$1" = "--map" ]; then
+ opt_map=$(parse_opt_file map "$2"); ret=$?
+ [ $ret -ne 0 ] && exit 255
+ shift 2
+ opt_live=0
+ elif [ "$1" = "--arch-prefix" ]; then
+ opt_arch_prefix="$2"
+ shift 2
+ elif [ "$1" = "--live" ]; then
+ opt_live_explicit=1
+ shift
+ elif [ "$1" = "--no-color" ]; then
+ opt_no_color=1
+ shift
+ elif [ "$1" = "--no-sysfs" ]; then
+ opt_no_sysfs=1
+ shift
+ elif [ "$1" = "--sysfs-only" ]; then
+ opt_sysfs_only=1
+ shift
+ elif [ "$1" = "--coreos" ]; then
+ opt_coreos=1
+ shift
+ elif [ "$1" = "--coreos-within-toolbox" ]; then
+ # don't use directly: used internally by --coreos
+ opt_coreos=0
+ shift
+ elif [ "$1" = "--paranoid" ]; then
+ opt_paranoid=1
+ shift
+ elif [ "$1" = "--hw-only" ]; then
+ opt_hw_only=1
+ shift
+ elif [ "$1" = "--no-hw" ]; then
+ opt_no_hw=1
+ shift
+ elif [ "$1" = "--no-explain" ]; then
+ opt_no_explain=1
+ shift
+ elif [ "$1" = "--batch" ]; then
+ opt_batch=1
+ opt_verbose=0
+ shift
+ case "$1" in
+ text|nrpe|json|prometheus) opt_batch_format="$1"; shift;;
+ --*) ;; # allow subsequent flags
+ '') ;; # allow nothing at all
+ *)
+ echo "$0: error: unknown batch format '$1'" >&2
+ echo "$0: error: --batch expects a format from: text, nrpe, json" >&2
+ exit 255
+ ;;
+ esac
+ elif [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then
+ opt_verbose=$(( opt_verbose + 1 ))
+ shift
+ elif [ "$1" = "--variant" ]; then
+ if [ -z "$2" ]; then
+ echo "$0: error: option --variant expects a parameter (1, 2 or 3)" >&2
+ exit 255
+ fi
+ case "$2" in
+ 1) opt_variant1=1; opt_allvariants=0;;
+ 2) opt_variant2=1; opt_allvariants=0;;
+ 3) opt_variant3=1; opt_allvariants=0;;
+ *)
+ echo "$0: error: invalid parameter '$2' for --variant, expected either 1, 2 or 3" >&2;
+ exit 255
+ ;;
+ esac
+ shift 2
+ elif [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+ show_header
+ show_usage
+ exit 0
+ elif [ "$1" = "--version" ]; then
+ opt_no_color=1
+ show_header
+ exit 0
+ elif [ "$1" = "--disclaimer" ]; then
+ show_header
+ show_disclaimer
+ exit 0
+ else
+ show_header
+ show_usage
+ echo "$0: error: unknown option '$1'"
+ exit 255
+ fi
+done
+
+show_header
+
+if [ "$opt_no_sysfs" = 1 ] && [ "$opt_sysfs_only" = 1 ]; then
+ _warn "Incompatible options specified (--no-sysfs and --sysfs-only), aborting"
+ exit 255
+fi
+
+if [ "$opt_no_hw" = 1 ] && [ "$opt_hw_only" = 1 ]; then
+ _warn "Incompatible options specified (--no-hw and --hw-only), aborting"
+ exit 255
+fi
+
+# print status function
+pstatus()
+{
+ if [ "$opt_no_color" = 1 ]; then
+ _info_nol "$2"
+ else
+ case "$1" in
+ red) col="\033[41m\033[30m";;
+ green) col="\033[42m\033[30m";;
+ yellow) col="\033[43m\033[30m";;
+ blue) col="\033[44m\033[30m";;
+ *) col="";;
+ esac
+ _info_nol "$col $2 \033[0m"
+ fi
+ [ -n "$3" ] && _info_nol " ($3)"
+ _info
+ unset col
+}
+
+# Print the final status of a vulnerability (incl. batch mode)
+# Arguments are: CVE UNK/OK/VULN description
+pvulnstatus()
+{
+ pvulnstatus_last_cve="$1"
+ if [ "$opt_batch" = 1 ]; then
+ case "$1" in
+ CVE-2017-5753) aka="SPECTRE VARIANT 1";;
+ CVE-2017-5715) aka="SPECTRE VARIANT 2";;
+ CVE-2017-5754) aka="MELTDOWN";;
+ esac
+
+ case "$opt_batch_format" in
+ text) _echo 0 "$1: $2 ($3)";;
+ json)
+ case "$2" in
+ UNK) is_vuln="null";;
+ VULN) is_vuln="true";;
+ OK) is_vuln="false";;
+ esac
+ json_output="${json_output:-[}{\"NAME\":\"$aka\",\"CVE\":\"$1\",\"VULNERABLE\":$is_vuln,\"INFOS\":\"$3\"},"
+ ;;
+
+ nrpe) [ "$2" = VULN ] && nrpe_vuln="$nrpe_vuln $1";;
+ prometheus)
+ prometheus_output="${prometheus_output:+$prometheus_output\n}specex_vuln_status{name=\"$aka\",cve=\"$1\",status=\"$2\",info=\"$3\"} 1"
+ ;;
+ esac
+ fi
+
+ # always fill global_* vars because we use that do decide the program exit code
+ case "$2" in
+ UNK) global_unknown="1";;
+ VULN) global_critical="1";;
+ esac
+
+ # display info if we're not in quiet/batch mode
+ vulnstatus="$2"
+ shift 2
+ _info_nol "> \033[46m\033[30mSTATUS:\033[0m "
+ case "$vulnstatus" in
+ UNK) pstatus yellow 'UNKNOWN' "$@";;
+ VULN) pstatus red 'VULNERABLE' "$@";;
+ OK) pstatus green 'NOT VULNERABLE' "$@";;
+ esac
+}
+
+
+# The 3 below functions are taken from the extract-linux script, available here:
+# https://github.com/torvalds/linux/blob/master/scripts/extract-vmlinux
+# The functions have been modified for better integration to this script
+# The original header of the file has been retained below
+
+# ----------------------------------------------------------------------
+# extract-vmlinux - Extract uncompressed vmlinux from a kernel image
+#
+# Inspired from extract-ikconfig
+# (c) 2009,2010 Dick Streefland
+#
+# (c) 2011 Corentin Chary
+#
+# Licensed under the GNU General Public License, version 2 (GPLv2).
+# ----------------------------------------------------------------------
+
+kernel=''
+kernel_err=''
+check_kernel()
+{
+ _file="$1"
+ _desperate_mode="$2"
+ # checking the return code of readelf -h is not enough, we could get
+ # a damaged ELF file and validate it, check for stderr warnings too
+ _readelf_warnings=$("${opt_arch_prefix}readelf" -S "$_file" 2>&1 >/dev/null | tr "\n" "/"); ret=$?
+ _readelf_sections=$("${opt_arch_prefix}readelf" -S "$_file" 2>/dev/null | grep -c -e data -e text -e init)
+ _kernel_size=$(stat -c %s "$_file" 2>/dev/null || stat -f %z "$_file" 2>/dev/null || echo 10000)
+ _debug "check_kernel: ret=$? size=$_kernel_size sections=$_readelf_sections warnings=$_readelf_warnings"
+ if [ -n "$_desperate_mode" ]; then
+ if "${opt_arch_prefix}strings" "$_file" | grep -Eq '^Linux version '; then
+ _debug "check_kernel (desperate): ... matched!"
+ return 0
+ else
+ _debug "check_kernel (desperate): ... invalid"
+ fi
+ else
+ if [ $ret -eq 0 ] && [ -z "$_readelf_warnings" ] && [ "$_readelf_sections" -gt 0 ]; then
+ if [ "$_kernel_size" -ge 100000 ]; then
+ _debug "check_kernel: ... file is valid"
+ return 0
+ else
+ _debug "check_kernel: ... file seems valid but is too small, ignoring"
+ fi
+ else
+ _debug "check_kernel: ... file is invalid"
+ fi
+ fi
+ return 1
+}
+
+try_decompress()
+{
+ # The obscure use of the "tr" filter is to work around older versions of
+ # "grep" that report the byte offset of the line instead of the pattern.
+
+ # Try to find the header ($1) and decompress from here
+ _debug "try_decompress: looking for $3 magic in $6"
+ for pos in $(tr "$1\n$2" "\n$2=" < "$6" | grep -abo "^$2")
+ do
+ _debug "try_decompress: magic for $3 found at offset $pos"
+ if ! which "$3" >/dev/null 2>&1; then
+ kernel_err="missing '$3' tool, please install it, usually it's in the '$5' package"
+ return 0
+ fi
+ pos=${pos%%:*}
+ # shellcheck disable=SC2086
+ tail -c+$pos "$6" 2>/dev/null | $3 $4 > "$kerneltmp" 2>/dev/null; ret=$?
+ if [ ! -s "$kerneltmp" ]; then
+ # don't rely on $ret, sometimes it's != 0 but worked
+ # (e.g. gunzip ret=2 just means there was trailing garbage)
+ _debug "try_decompress: decompression with $3 failed (err=$ret)"
+ elif check_kernel "$kerneltmp" "$7"; then
+ kernel="$kerneltmp"
+ _debug "try_decompress: decompressed with $3 successfully!"
+ return 0
+ elif [ "$3" != "cat" ]; then
+ _debug "try_decompress: decompression with $3 worked but result is not a kernel, trying with an offset"
+ [ -z "$kerneltmp2" ] && kerneltmp2=$(mktemp /tmp/kernel-XXXXXX)
+ cat "$kerneltmp" > "$kerneltmp2"
+ try_decompress '\177ELF' xxy 'cat' '' cat "$kerneltmp2" && return 0
+ else
+ _debug "try_decompress: decompression with $3 worked but result is not a kernel"
+ fi
+ done
+ return 1
+}
+
+extract_kernel()
+{
+ [ -n "$1" ] || return 1
+ # Prepare temp files:
+ kerneltmp="$(mktemp /tmp/kernel-XXXXXX)"
+
+ # Initial attempt for uncompressed images or objects:
+ if check_kernel "$1"; then
+ cat "$1" > "$kerneltmp"
+ kernel=$kerneltmp
+ return 0
+ fi
+
+ # That didn't work, so retry after decompression.
+ for mode in '' 'desperate'; do
+ try_decompress '\037\213\010' xy gunzip '' gunzip "$1" "$mode" && return 0
+ try_decompress '\3757zXZ\000' abcde unxz '' xz-utils "$1" "$mode" && return 0
+ try_decompress 'BZh' xy bunzip2 '' bzip2 "$1" "$mode" && return 0
+ try_decompress '\135\0\0\0' xxx unlzma '' xz-utils "$1" "$mode" && return 0
+ try_decompress '\211\114\132' xy 'lzop' '-d' lzop "$1" "$mode" && return 0
+ try_decompress '\002\041\114\030' xyy 'lz4' '-d -l' liblz4-tool "$1" "$mode" && return 0
+ try_decompress '\177ELF' xxy 'cat' '' cat "$1" "$mode" && return 0
+ done
+ _verbose "Couldn't extract the kernel image, accuracy might be reduced"
+ return 1
+}
+
+# end of extract-vmlinux functions
+
+mount_debugfs()
+{
+ if [ ! -e /sys/kernel/debug/sched_features ]; then
+ # try to mount the debugfs hierarchy ourselves and remember it to umount afterwards
+ mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null && mounted_debugfs=1
+ fi
+}
+
+load_msr()
+{
+ if [ "$os" = Linux ]; then
+ modprobe msr 2>/dev/null && insmod_msr=1
+ _debug "attempted to load module msr, insmod_msr=$insmod_msr"
+ else
+ if ! kldstat -q -m cpuctl; then
+ kldload cpuctl 2>/dev/null && kldload_cpuctl=1
+ _debug "attempted to load module cpuctl, kldload_cpuctl=$kldload_cpuctl"
+ else
+ _debug "cpuctl module already loaded"
+ fi
+ fi
+}
+
+load_cpuid()
+{
+ if [ "$os" = Linux ]; then
+ modprobe cpuid 2>/dev/null && insmod_cpuid=1
+ _debug "attempted to load module cpuid, insmod_cpuid=$insmod_cpuid"
+ else
+ if ! kldstat -q -m cpuctl; then
+ kldload cpuctl 2>/dev/null && kldload_cpuctl=1
+ _debug "attempted to load module cpuctl, kldload_cpuctl=$kldload_cpuctl"
+ else
+ _debug "cpuctl module already loaded"
+ fi
+ fi
+}
+
+# shellcheck disable=SC2034
+{
+EAX=1; EBX=2; ECX=3; EDX=4;
+}
+read_cpuid()
+{
+ # leaf is the value of the eax register when calling the cpuid instruction:
+ _leaf="$1"
+ # eax=1 ebx=2 ecx=3 edx=4:
+ _register="$2"
+ # number of bits to shift the register right to:
+ _shift="$3"
+ # mask to apply as an AND operand to the shifted register value
+ _mask="$4"
+ # wanted value (optional), if present we return 0(true) if the obtained value is equal, 1 otherwise:
+ _wanted="$5"
+ # in any case, the read value is globally available in $read_cpuid_value
+
+ read_cpuid_value=''
+ if [ ! -e /dev/cpu/0/cpuid ] && [ ! -e /dev/cpuctl0 ]; then
+ # try to load the module ourselves (and remember it so we can rmmod it afterwards)
+ load_cpuid
+ fi
+
+ if [ -e /dev/cpu/0/cpuid ]; then
+ # Linux
+ # we need _leaf to be converted to decimal for dd
+ _leaf=$(( _leaf ))
+ _cpuid=$(dd if=/dev/cpu/0/cpuid bs=16 skip="$_leaf" iflag=skip_bytes count=1 2>/dev/null | od -A n -t u4)
+ elif [ -e /dev/cpuctl0 ]; then
+ # BSD
+ _cpuid=$(cpucontrol -i "$_leaf" /dev/cpuctl0 2>/dev/null | awk '{print $4,$5,$6,$7}')
+ # cpuid level 0x1: 0x000306d4 0x00100800 0x4dfaebbf 0xbfebfbff
+ else
+ return 2
+ fi
+
+ _debug "cpuid: leaf$_leaf on cpu0, eax-ebx-ecx-edx: $_cpuid"
+ [ -z "$_cpuid" ] && return 2
+ # get the value of the register we want
+ _reg=$(echo "$_cpuid" | awk '{print $'"$_register"'}')
+ # Linux returns it as decimal, BSD as hex, normalize to decimal
+ _reg=$(( _reg ))
+ # shellcheck disable=SC2046
+ _debug "cpuid: wanted register ($_register) has value $_reg aka "$(printf "%08x" "$_reg")
+ _reg_shifted=$(( _reg >> _shift ))
+ # shellcheck disable=SC2046
+ _debug "cpuid: shifted value by $_shift is $_reg_shifted aka "$(printf "%x" "$_reg_shifted")
+ read_cpuid_value=$(( _reg_shifted & _mask ))
+ # shellcheck disable=SC2046
+ _debug "cpuid: after AND $_mask, final value is $read_cpuid_value aka "$(printf "%x" "$read_cpuid_value")
+ if [ -n "$_wanted" ]; then
+ _debug "cpuid: wanted $_wanted and got $read_cpuid_value"
+ if [ "$read_cpuid_value" = "$_wanted" ]; then
+ return 0
+ else
+ return 1
+ fi
+ fi
+
+ return 0
+}
+
+dmesg_grep()
+{
+ # grep for something in dmesg, ensuring that the dmesg buffer
+ # has not been truncated
+ dmesg_grepped=''
+ if ! dmesg | grep -qE -e '(^|\] )Linux version [0-9]' -e '^FreeBSD is a registered' ; then
+ # dmesg truncated
+ return 2
+ fi
+ dmesg_grepped=$(dmesg | grep -E "$1" | head -1)
+ # not found:
+ [ -z "$dmesg_grepped" ] && return 1
+ # found, output is in $dmesg_grepped
+ return 0
+}
+
+is_coreos()
+{
+ which coreos-install >/dev/null 2>&1 && which toolbox >/dev/null 2>&1 && return 0
+ return 1
+}
+
+parse_cpu_details()
+{
+ [ "$parse_cpu_details_done" = 1 ] && return 0
+
+ if [ -e "$procfs/cpuinfo" ]; then
+ cpu_vendor=$( grep '^vendor_id' "$procfs/cpuinfo" | awk '{print $3}' | head -1)
+ cpu_friendly_name=$(grep '^model name' "$procfs/cpuinfo" | cut -d: -f2- | head -1 | sed -e 's/^ *//')
+ # special case for ARM follows
+ if grep -qi 'CPU implementer[[:space:]]*:[[:space:]]*0x41' "$procfs/cpuinfo"; then
+ cpu_vendor='ARM'
+ # some devices (phones or other) have several ARMs and as such different part numbers,
+ # an example is "bigLITTLE", so we need to store the whole list, this is needed for is_cpu_vulnerable
+ cpu_part_list=$(awk '/CPU part/ {print $4}' "$procfs/cpuinfo")
+ cpu_arch_list=$(awk '/CPU architecture/ {print $3}' "$procfs/cpuinfo")
+ # take the first one to fill the friendly name, do NOT quote the vars below
+ # shellcheck disable=SC2086
+ cpu_arch=$(echo $cpu_arch_list | awk '{ print $1 }')
+ # shellcheck disable=SC2086
+ cpu_part=$(echo $cpu_part_list | awk '{ print $1 }')
+ [ "$cpu_arch" = "AArch64" ] && cpu_arch=8
+ cpu_friendly_name="ARM"
+ [ -n "$cpu_arch" ] && cpu_friendly_name="$cpu_friendly_name v$cpu_arch"
+ [ -n "$cpu_part" ] && cpu_friendly_name="$cpu_friendly_name model $cpu_part"
+ fi
+
+ cpu_family=$( grep '^cpu family' "$procfs/cpuinfo" | awk '{print $4}' | grep -E '^[0-9]+$' | head -1)
+ cpu_model=$( grep '^model' "$procfs/cpuinfo" | awk '{print $3}' | grep -E '^[0-9]+$' | head -1)
+ cpu_stepping=$(grep '^stepping' "$procfs/cpuinfo" | awk '{print $3}' | grep -E '^[0-9]+$' | head -1)
+ cpu_ucode=$( grep '^microcode' "$procfs/cpuinfo" | awk '{print $3}' | head -1)
+ else
+ cpu_friendly_name=$(sysctl -n hw.model)
+ fi
+
+ # get raw cpuid, it's always useful (referenced in the Intel doc for firmware updates for example)
+ if read_cpuid 0x1 $EAX 0 0xFFFFFFFF; then
+ cpuid="$read_cpuid_value"
+ fi
+
+ # under BSD, linprocfs often doesn't export ucode information, so fetch it ourselves the good old way
+ if [ -z "$cpu_ucode" ] && [ "$os" != Linux ]; then
+ load_cpuid
+ if [ -e /dev/cpuctl0 ]; then
+ # init MSR with NULLs
+ cpucontrol -m 0x8b=0 /dev/cpuctl0
+ # call CPUID
+ cpucontrol -i 1 /dev/cpuctl0 >/dev/null
+ # read MSR
+ cpu_ucode=$(cpucontrol -m 0x8b /dev/cpuctl0 | awk '{print $3}')
+ # convert to decimal
+ cpu_ucode=$(( cpu_ucode ))
+ # convert back to hex
+ cpu_ucode=$(printf "0x%x" "$cpu_ucode")
+ fi
+ fi
+
+ echo "$cpu_ucode" | grep -q ^0x && cpu_ucode_decimal=$(( cpu_ucode ))
+ ucode_found="model $cpu_model stepping $cpu_stepping ucode $cpu_ucode cpuid "$(printf "0x%x" "$cpuid")
+
+ # also define those that we will need in other funcs
+ # taken from ttps://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/include/asm/intel-family.h
+ # shellcheck disable=SC2034
+ {
+ INTEL_FAM6_CORE_YONAH=$(( 0x0E ))
+
+ INTEL_FAM6_CORE2_MEROM=$(( 0x0F ))
+ INTEL_FAM6_CORE2_MEROM_L=$(( 0x16 ))
+ INTEL_FAM6_CORE2_PENRYN=$(( 0x17 ))
+ INTEL_FAM6_CORE2_DUNNINGTON=$(( 0x1D ))
+
+ INTEL_FAM6_NEHALEM=$(( 0x1E ))
+ INTEL_FAM6_NEHALEM_G=$(( 0x1F ))
+ INTEL_FAM6_NEHALEM_EP=$(( 0x1A ))
+ INTEL_FAM6_NEHALEM_EX=$(( 0x2E ))
+
+ INTEL_FAM6_WESTMERE=$(( 0x25 ))
+ INTEL_FAM6_WESTMERE_EP=$(( 0x2C ))
+ INTEL_FAM6_WESTMERE_EX=$(( 0x2F ))
+
+ INTEL_FAM6_SANDYBRIDGE=$(( 0x2A ))
+ INTEL_FAM6_SANDYBRIDGE_X=$(( 0x2D ))
+ INTEL_FAM6_IVYBRIDGE=$(( 0x3A ))
+ INTEL_FAM6_IVYBRIDGE_X=$(( 0x3E ))
+
+ INTEL_FAM6_HASWELL_CORE=$(( 0x3C ))
+ INTEL_FAM6_HASWELL_X=$(( 0x3F ))
+ INTEL_FAM6_HASWELL_ULT=$(( 0x45 ))
+ INTEL_FAM6_HASWELL_GT3E=$(( 0x46 ))
+
+ INTEL_FAM6_BROADWELL_CORE=$(( 0x3D ))
+ INTEL_FAM6_BROADWELL_GT3E=$(( 0x47 ))
+ INTEL_FAM6_BROADWELL_X=$(( 0x4F ))
+ INTEL_FAM6_BROADWELL_XEON_D=$(( 0x56 ))
+
+ INTEL_FAM6_SKYLAKE_MOBILE=$(( 0x4E ))
+ INTEL_FAM6_SKYLAKE_DESKTOP=$(( 0x5E ))
+ INTEL_FAM6_SKYLAKE_X=$(( 0x55 ))
+ INTEL_FAM6_KABYLAKE_MOBILE=$(( 0x8E ))
+ INTEL_FAM6_KABYLAKE_DESKTOP=$(( 0x9E ))
+
+ # /* "Small Core" Processors (Atom) */
+
+ INTEL_FAM6_ATOM_PINEVIEW=$(( 0x1C ))
+ INTEL_FAM6_ATOM_LINCROFT=$(( 0x26 ))
+ INTEL_FAM6_ATOM_PENWELL=$(( 0x27 ))
+ INTEL_FAM6_ATOM_CLOVERVIEW=$(( 0x35 ))
+ INTEL_FAM6_ATOM_CEDARVIEW=$(( 0x36 ))
+ INTEL_FAM6_ATOM_SILVERMONT1=$(( 0x37 ))
+ INTEL_FAM6_ATOM_SILVERMONT2=$(( 0x4D ))
+ INTEL_FAM6_ATOM_AIRMONT=$(( 0x4C ))
+ INTEL_FAM6_ATOM_MERRIFIELD=$(( 0x4A ))
+ INTEL_FAM6_ATOM_MOOREFIELD=$(( 0x5A ))
+ INTEL_FAM6_ATOM_GOLDMONT=$(( 0x5C ))
+ INTEL_FAM6_ATOM_DENVERTON=$(( 0x5F ))
+ INTEL_FAM6_ATOM_GEMINI_LAKE=$(( 0x7A ))
+
+ # /* Xeon Phi */
+
+ INTEL_FAM6_XEON_PHI_KNL=$(( 0x57 ))
+ INTEL_FAM6_XEON_PHI_KNM=$(( 0x85 ))
+ }
+ parse_cpu_details_done=1
+}
+
+is_amd()
+{
+ [ "$cpu_vendor" = AuthenticAMD ] && return 0
+ return 1
+}
+
+is_intel()
+{
+ [ "$cpu_vendor" = GenuineIntel ] && return 0
+ return 1
+}
+
+is_cpu_smt_enabled()
+{
+ # SMT / HyperThreading is enabled if siblings != cpucores
+ if [ -e "$procfs/cpuinfo" ]; then
+ _siblings=$(awk '/^siblings/ {print $3;exit}' "$procfs/cpuinfo")
+ _cpucores=$(awk '/^cpu cores/ {print $4;exit}' "$procfs/cpuinfo")
+ if [ -n "$_siblings" ] && [ -n "$_cpucores" ]; then
+ if [ "$_siblings" = "$_cpucores" ]; then
+ return 1
+ else
+ return 0
+ fi
+ fi
+ fi
+ # we can't tell
+ return 2
+}
+
+is_ucode_blacklisted()
+{
+ parse_cpu_details
+ # if it's not an Intel, don't bother: it's not blacklisted
+ is_intel || return 1
+ # it also needs to be family=6
+ [ "$cpu_family" = 6 ] || return 1
+ # now, check each known bad microcode
+ # source: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/kernel/cpu/intel.c#n105
+ # 2018-02-08 update: https://newsroom.intel.com/wp-content/uploads/sites/11/2018/02/microcode-update-guidance.pdf
+ # model,stepping,microcode
+ for tuple in \
+ $INTEL_FAM6_KABYLAKE_DESKTOP,0x0B,0x80 \
+ $INTEL_FAM6_KABYLAKE_DESKTOP,0x0A,0x80 \
+ $INTEL_FAM6_KABYLAKE_DESKTOP,0x09,0x80 \
+ $INTEL_FAM6_KABYLAKE_MOBILE,0x0A,0x80 \
+ $INTEL_FAM6_KABYLAKE_MOBILE,0x09,0x80 \
+ $INTEL_FAM6_SKYLAKE_X,0x03,0x0100013e \
+ $INTEL_FAM6_SKYLAKE_X,0x04,0x02000036 \
+ $INTEL_FAM6_SKYLAKE_X,0x04,0x0200003a \
+ $INTEL_FAM6_SKYLAKE_X,0x04,0x0200003c \
+ $INTEL_FAM6_BROADWELL_CORE,0x04,0x28 \
+ $INTEL_FAM6_BROADWELL_GT3E,0x01,0x1b \
+ $INTEL_FAM6_BROADWELL_XEON_D,0x02,0x14 \
+ $INTEL_FAM6_BROADWELL_XEON_D,0x03,0x07000011 \
+ $INTEL_FAM6_BROADWELL_X,0x01,0x0b000023 \
+ $INTEL_FAM6_BROADWELL_X,0x01,0x0b000025 \
+ $INTEL_FAM6_HASWELL_ULT,0x01,0x21 \
+ $INTEL_FAM6_HASWELL_GT3E,0x01,0x18 \
+ $INTEL_FAM6_HASWELL_CORE,0x03,0x23 \
+ $INTEL_FAM6_HASWELL_X,0x02,0x3b \
+ $INTEL_FAM6_HASWELL_X,0x04,0x10 \
+ $INTEL_FAM6_IVYBRIDGE_X,0x04,0x42a \
+ $INTEL_FAM6_SANDYBRIDGE_X,0x06,0x61b \
+ $INTEL_FAM6_SANDYBRIDGE_X,0x07,0x712
+ do
+ model=$(echo $tuple | cut -d, -f1)
+ stepping=$(( $(echo $tuple | cut -d, -f2) ))
+ ucode=$(echo $tuple | cut -d, -f3)
+ echo "$ucode" | grep -q ^0x && ucode_decimal=$(( ucode ))
+ if [ "$cpu_model" = "$model" ] && [ "$cpu_stepping" = "$stepping" ]; then
+ if [ "$cpu_ucode_decimal" = "$ucode_decimal" ] || [ "$cpu_ucode" = "$ucode" ]; then
+ _debug "is_ucode_blacklisted: we have a match! ($cpu_model/$cpu_stepping/$cpu_ucode)"
+ return 0
+ fi
+ fi
+ done
+ _debug "is_ucode_blacklisted: no ($cpu_model/$cpu_stepping/$cpu_ucode)"
+ return 1
+}
+
+is_skylake_cpu()
+{
+ # is this a skylake cpu?
+ # return 0 if yes, 1 otherwise
+ #if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
+ # boot_cpu_data.x86 == 6) {
+ # switch (boot_cpu_data.x86_model) {
+ # case INTEL_FAM6_SKYLAKE_MOBILE:
+ # case INTEL_FAM6_SKYLAKE_DESKTOP:
+ # case INTEL_FAM6_SKYLAKE_X:
+ # case INTEL_FAM6_KABYLAKE_MOBILE:
+ # case INTEL_FAM6_KABYLAKE_DESKTOP:
+ # return true;
+ parse_cpu_details
+ is_intel || return 1
+ [ "$cpu_family" = 6 ] || return 1
+ if [ "$cpu_model" = $INTEL_FAM6_SKYLAKE_MOBILE ] || \
+ [ "$cpu_model" = $INTEL_FAM6_SKYLAKE_DESKTOP ] || \
+ [ "$cpu_model" = $INTEL_FAM6_SKYLAKE_X ] || \
+ [ "$cpu_model" = $INTEL_FAM6_KABYLAKE_MOBILE ] || \
+ [ "$cpu_model" = $INTEL_FAM6_KABYLAKE_DESKTOP ]; then
+ return 0
+ fi
+ return 1
+}
+
+is_zen_cpu()
+{
+ # is this CPU from the AMD ZEN family ? (ryzen, epyc, ...)
+ parse_cpu_details
+ is_amd || return 1
+ [ "$cpu_family" = 23 ] && return 0
+ return 1
+}
+
+# ENTRYPOINT
+
+# we can't do anything useful under WSL
+if uname -a | grep -qE -- '-Microsoft #[0-9]+-Microsoft '; then
+ _warn "This script doesn't work under Windows Subsystem for Linux"
+ _warn "You should use the official Microsoft tool instead."
+ _warn "It can be found under https://aka.ms/SpeculationControlPS"
+ exit 1
+fi
+
+# check for mode selection inconsistency
+if [ "$opt_live_explicit" = 1 ]; then
+ if [ -n "$opt_kernel" ] || [ -n "$opt_config" ] || [ -n "$opt_map" ]; then
+ show_usage
+ echo "$0: error: incompatible modes specified, use either --live or --kernel/--config/--map" >&2
+ exit 255
+ fi
+fi
+if [ "$opt_hw_only" = 1 ]; then
+ if [ "$opt_allvariants" = 0 ]; then
+ show_usage
+ echo "$0: error: incompatible modes specified, --hw-only vs --variant" >&2
+ exit 255
+ else
+ opt_allvariants=0
+ opt_variant1=0
+ opt_variant2=0
+ opt_variant3=0
+ fi
+fi
+
+# coreos mode
+if [ "$opt_coreos" = 1 ]; then
+ if ! is_coreos; then
+ _warn "CoreOS mode asked, but we're not under CoreOS!"
+ exit 255
+ fi
+ _warn "CoreOS mode, starting an ephemeral toolbox to launch the script"
+ load_msr
+ load_cpuid
+ mount_debugfs
+ toolbox --ephemeral --bind-ro /dev/cpu:/dev/cpu -- sh -c "dnf install -y binutils which && /media/root$PWD/$0 $* --coreos-within-toolbox"
+ exitcode=$?
+ exit $exitcode
+else
+ if is_coreos; then
+ _warn "You seem to be running CoreOS, you might want to use the --coreos option for better results"
+ _warn
+ fi
+fi
+
+# if we're under a BSD, try to mount linprocfs for "$procfs/cpuinfo"
+procfs=/proc
+if echo "$os" | grep -q BSD; then
+ _debug "We're under BSD, check if we have procfs"
+ procfs=$(mount | awk '/^linprocfs/ { print $3; exit; }')
+ if [ -z "$procfs" ]; then
+ _debug "we don't, try to mount it"
+ procfs=/proc
+ [ -d /compat/linux/proc ] && procfs=/compat/linux/proc
+ test -d $procfs || mkdir $procfs
+ if mount -t linprocfs linprocfs $procfs 2>/dev/null; then
+ mounted_procfs=1
+ _debug "procfs just mounted at $procfs"
+ else
+ procfs=''
+ fi
+ else
+ _debug "We do: $procfs"
+ fi
+fi
+
+parse_cpu_details
+if [ "$opt_live" = 1 ]; then
+ # root check (only for live mode, for offline mode, we already checked if we could read the files)
+ if [ "$(id -u)" -ne 0 ]; then
+ _warn "Note that you should launch this script with root privileges to get accurate information."
+ _warn "We'll proceed but you might see permission denied errors."
+ _warn "To run it as root, you can try the following command: sudo $0"
+ _warn
+ fi
+ _info "Checking for vulnerabilities on current system"
+ _info "Kernel is \033[35m$(uname -s) $(uname -r) $(uname -v) $(uname -m)\033[0m"
+ _info "CPU is \033[35m$cpu_friendly_name\033[0m"
+
+ # try to find the image of the current running kernel
+ # first, look for the BOOT_IMAGE hint in the kernel cmdline
+ if [ -r /proc/cmdline ] && grep -q 'BOOT_IMAGE=' /proc/cmdline; then
+ opt_kernel=$(grep -Eo 'BOOT_IMAGE=[^ ]+' /proc/cmdline | cut -d= -f2)
+ _debug "found opt_kernel=$opt_kernel in /proc/cmdline"
+ # if we have a dedicated /boot partition, our bootloader might have just called it /
+ # so try to prepend /boot and see if we find anything
+ [ -e "/boot/$opt_kernel" ] && opt_kernel="/boot/$opt_kernel"
+ # special case for CoreOS if we're inside the toolbox
+ [ -e "/media/root/boot/$opt_kernel" ] && opt_kernel="/media/root/boot/$opt_kernel"
+ _debug "opt_kernel is now $opt_kernel"
+ # else, the full path is already there (most probably /boot/something)
+ fi
+ # if we didn't find a kernel, default to guessing
+ if [ ! -e "$opt_kernel" ]; then
+ # Fedora:
+ [ -e "/lib/modules/$(uname -r)/vmlinuz" ] && opt_kernel="/lib/modules/$(uname -r)/vmlinuz"
+ # Slackare:
+ [ -e "/boot/vmlinuz" ] && opt_kernel="/boot/vmlinuz"
+ # Arch:
+ [ -e "/boot/vmlinuz-linux" ] && opt_kernel="/boot/vmlinuz-linux"
+ # Linux-Libre:
+ [ -e "/boot/vmlinuz-linux-libre" ] && opt_kernel="/boot/vmlinuz-linux-libre"
+ # pine64
+ [ -e "/boot/pine64/Image" ] && opt_kernel="/boot/pine64/Image"
+ # generic:
+ [ -e "/boot/vmlinuz-$(uname -r)" ] && opt_kernel="/boot/vmlinuz-$(uname -r)"
+ [ -e "/boot/kernel-$( uname -r)" ] && opt_kernel="/boot/kernel-$( uname -r)"
+ [ -e "/boot/bzImage-$(uname -r)" ] && opt_kernel="/boot/bzImage-$(uname -r)"
+ # Gentoo:
+ [ -e "/boot/kernel-genkernel-$(uname -m)-$(uname -r)" ] && opt_kernel="/boot/kernel-genkernel-$(uname -m)-$(uname -r)"
+ # NixOS:
+ [ -e "/run/booted-system/kernel" ] && opt_kernel="/run/booted-system/kernel"
+ # systemd kernel-install:
+ [ -e "/etc/machine-id" ] && [ -e "/boot/$(cat /etc/machine-id)/$(uname -r)/linux" ] && opt_kernel="/boot/$(cat /etc/machine-id)/$(uname -r)/linux"
+ fi
+
+ # system.map
+ if [ -e /proc/kallsyms ] ; then
+ opt_map=/proc/kallsyms
+ elif [ -e "/lib/modules/$(uname -r)/System.map" ] ; then
+ opt_map="/lib/modules/$(uname -r)/System.map"
+ elif [ -e "/boot/System.map-$(uname -r)" ] ; then
+ opt_map="/boot/System.map-$(uname -r)"
+ fi
+
+ # config
+ if [ -e /proc/config.gz ] ; then
+ dumped_config="$(mktemp /tmp/config-XXXXXX)"
+ gunzip -c /proc/config.gz > "$dumped_config"
+ # dumped_config will be deleted at the end of the script
+ opt_config="$dumped_config"
+ elif [ -e "/lib/modules/$(uname -r)/config" ]; then
+ opt_config="/lib/modules/$(uname -r)/config"
+ elif [ -e "/boot/config-$(uname -r)" ]; then
+ opt_config="/boot/config-$(uname -r)"
+ fi
+else
+ _info "Checking for vulnerabilities against specified kernel"
+ _info "CPU is \033[35m$cpu_friendly_name\033[0m"
+fi
+
+if [ -n "$opt_kernel" ]; then
+ _verbose "Will use kernel image \033[35m$opt_kernel\033[0m"
+else
+ _verbose "Will use no kernel image (accuracy might be reduced)"
+ bad_accuracy=1
+fi
+
+if [ "$os" = Linux ]; then
+ if [ -n "$opt_config" ] && ! grep -q '^CONFIG_' "$opt_config"; then
+ # given file is invalid!
+ _warn "The kernel config file seems invalid, was expecting a plain-text file, ignoring it!"
+ opt_config=''
+ fi
+
+ if [ -n "$dumped_config" ] && [ -n "$opt_config" ]; then
+ _verbose "Will use kconfig \033[35m/proc/config.gz (decompressed)\033[0m"
+ elif [ -n "$opt_config" ]; then
+ _verbose "Will use kconfig \033[35m$opt_config\033[0m"
+ else
+ _verbose "Will use no kconfig (accuracy might be reduced)"
+ bad_accuracy=1
+ fi
+
+ if [ -n "$opt_map" ]; then
+ _verbose "Will use System.map file \033[35m$opt_map\033[0m"
+ else
+ _verbose "Will use no System.map file (accuracy might be reduced)"
+ bad_accuracy=1
+ fi
+
+ if [ "$bad_accuracy" = 1 ]; then
+ _info "We're missing some kernel info (see -v), accuracy might be reduced"
+ fi
+fi
+
+if [ -e "$opt_kernel" ]; then
+ if ! which "${opt_arch_prefix}readelf" >/dev/null 2>&1; then
+ _debug "readelf not found"
+ kernel_err="missing '${opt_arch_prefix}readelf' tool, please install it, usually it's in the 'binutils' package"
+ elif [ "$opt_sysfs_only" = 1 ]; then
+ kernel_err='kernel image decompression skipped'
+ else
+ extract_kernel "$opt_kernel"
+ fi
+else
+ _debug "no opt_kernel defined"
+ kernel_err="couldn't find your kernel image in /boot, if you used netboot, this is normal"
+fi
+if [ -z "$kernel" ] || [ ! -r "$kernel" ]; then
+ [ -z "$kernel_err" ] && kernel_err="couldn't extract your kernel from $opt_kernel"
+else
+ # vanilla kernels have with ^Linux version
+ # also try harder with some kernels (such as Red Hat) that don't have ^Linux version before their version string
+ # and check for FreeBSD
+ kernel_version=$("${opt_arch_prefix}strings" "$kernel" 2>/dev/null | grep -E \
+ -e '^Linux version ' \
+ -e '^[[:alnum:]][^[:space:]]+ \([^[:space:]]+\) #[0-9]+ .+ (19|20)[0-9][0-9]$' \
+ -e '^FreeBSD [0-9]' | head -1)
+ if [ -z "$kernel_version" ]; then
+ # try even harder with some kernels (such as ARM) that split the release (uname -r) and version (uname -v) in 2 adjacent strings
+ kernel_version=$("${opt_arch_prefix}strings" "$kernel" 2>/dev/null | grep -E -B1 '^#[0-9]+ .+ (19|20)[0-9][0-9]$' | tr "\n" " ")
+ fi
+ if [ -n "$kernel_version" ]; then
+ # in live mode, check if the img we found is the correct one
+ if [ "$opt_live" = 1 ]; then
+ _verbose "Kernel image is \033[35m$kernel_version"
+ if ! echo "$kernel_version" | grep -qF "$(uname -r)"; then
+ _warn "Possible disrepancy between your running kernel '$(uname -r)' and the image '$kernel_version' we found ($opt_kernel), results might be incorrect"
+ fi
+ else
+ _info "Kernel image is \033[35m$kernel_version"
+ fi
+ else
+ _verbose "Kernel image version is unknown"
+ fi
+fi
+
+_info
+
+# end of header stuff
+
+# now we define some util functions and the check_*() funcs, as
+# the user can choose to execute only some of those
+
+sys_interface_check()
+{
+ [ "$opt_live" = 1 ] && [ "$opt_no_sysfs" = 0 ] && [ -r "$1" ] || return 1
+ _info_nol "* Mitigated according to the /sys interface: "
+ msg=$(cat "$1")
+ if grep -qi '^not affected' "$1"; then
+ # Not affected
+ status=OK
+ pstatus green YES "$msg"
+ elif grep -qi '^mitigation' "$1"; then
+ # Mitigation: PTI
+ status=OK
+ pstatus green YES "$msg"
+ elif grep -qi '^vulnerable' "$1"; then
+ # Vulnerable
+ status=VULN
+ pstatus yellow NO "$msg"
+ else
+ status=UNK
+ pstatus yellow UNKNOWN "$msg"
+ fi
+ _debug "sys_interface_check: $1=$msg"
+ return 0
+}
+
+number_of_cpus()
+{
+ if echo "$os" | grep -q BSD; then
+ n=$(sysctl -n hw.ncpu 2>/dev/null || echo 1)
+ elif [ -e "$procfs/cpuinfo" ]; then
+ n=$(grep -c ^processor "$procfs/cpuinfo" 2>/dev/null || echo 1)
+ else
+ # if we don't know, default to 1 CPU
+ n=1
+ fi
+ return "$n"
+}
+
+# $1 - msr number
+# $2 - cpu index
+write_msr()
+{
+ if [ "$os" != Linux ]; then
+ cpucontrol -m "$1=0" "/dev/cpuctl$2" >/dev/null 2>&1; ret=$?
+ else
+ # convert to decimal
+ _msrindex=$(( $1 ))
+ if [ ! -w /dev/cpu/"$2"/msr ]; then
+ ret=200 # permission error
+ else
+ dd if=/dev/zero of=/dev/cpu/"$2"/msr bs=8 count=1 seek="$_msrindex" oflag=seek_bytes 2>/dev/null; ret=$?
+ fi
+ fi
+ _debug "write_msr: for cpu $2 on msr $1 ($_msrindex), ret=$ret"
+ return $ret
+}
+
+read_msr()
+{
+ # _msr must be in hex, in the form 0x1234:
+ _msr="$1"
+ # cpu index, starting from 0:
+ _cpu="$2"
+ read_msr_value=''
+ if [ "$os" != Linux ]; then
+ _msr=$(cpucontrol -m "$_msr" "/dev/cpuctl$_cpu" 2>/dev/null); ret=$?
+ [ $ret -ne 0 ] && return 1
+ # MSR 0x10: 0x000003e1 0xb106dded
+ _msr_h=$(echo "$_msr" | awk '{print $3}');
+ _msr_h="$(( _msr_h >> 24 & 0xFF )) $(( _msr_h >> 16 & 0xFF )) $(( _msr_h >> 8 & 0xFF )) $(( _msr_h & 0xFF ))"
+ _msr_l=$(echo "$_msr" | awk '{print $4}');
+ _msr_l="$(( _msr_l >> 24 & 0xFF )) $(( _msr_l >> 16 & 0xFF )) $(( _msr_l >> 8 & 0xFF )) $(( _msr_l & 0xFF ))"
+ read_msr_value="$_msr_h $_msr_l"
+ else
+ # convert to decimal
+ _msr=$(( _msr ))
+ if [ ! -r /dev/cpu/"$_cpu"/msr ]; then
+ return 200 # permission error
+ fi
+ read_msr_value=$(dd if=/dev/cpu/"$_cpu"/msr bs=8 count=1 skip="$_msr" iflag=skip_bytes 2>/dev/null | od -t u1 -A n)
+ if [ -z "$read_msr_value" ]; then
+ # MSR doesn't exist, don't check for $? because some versions of dd still return 0!
+ return 1
+ fi
+ fi
+ _debug "read_msr: MSR=$1 value is $read_msr_value"
+ return 0
+}
+
+
+check_cpu()
+{
+ _info "\033[1;34mHardware check\033[0m"
+
+ if ! uname -m | grep -qwE 'x86_64|i[3-6]86|amd64'; then
+ return
+ fi
+
+ _info "* Hardware support (CPU microcode) for mitigation techniques"
+ _info " * Indirect Branch Restricted Speculation (IBRS)"
+ _info_nol " * SPEC_CTRL MSR is available: "
+ number_of_cpus
+ ncpus=$?
+ idx_max_cpu=$((ncpus-1))
+ if [ ! -e /dev/cpu/0/msr ] && [ ! -e /dev/cpuctl0 ]; then
+ # try to load the module ourselves (and remember it so we can rmmod it afterwards)
+ load_msr
+ fi
+ if [ ! -e /dev/cpu/0/msr ] && [ ! -e /dev/cpuctl0 ]; then
+ spec_ctrl_msr=-1
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ else
+ # the new MSR 'SPEC_CTRL' is at offset 0x48
+ # here we use dd, it's the same as using 'rdmsr 0x48' but without needing the rdmsr tool
+ # if we get a read error, the MSR is not there. bs has to be 8 for msr
+ # skip=9 because 8*9=72=0x48
+ val=0
+ cpu_mismatch=0
+ for i in $(seq 0 "$idx_max_cpu")
+ do
+ read_msr 0x48 "$i"; ret=$?
+ if [ "$i" -eq 0 ]; then
+ val=$ret
+ else
+ if [ "$ret" -eq $val ]; then
+ continue
+ else
+ cpu_mismatch=1
+ fi
+ fi
+ done
+ if [ $val -eq 0 ]; then
+ if [ $cpu_mismatch -eq 0 ]; then
+ spec_ctrl_msr=1
+ pstatus green YES
+ else
+ spec_ctrl_msr=1
+ pstatus green YES "But not in all CPUs"
+ fi
+ elif [ $val -eq 200 ]; then
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ spec_ctrl_msr=-1
+ else
+ spec_ctrl_msr=0
+ pstatus yellow NO
+ fi
+ fi
+
+ _info_nol " * CPU indicates IBRS capability: "
+ # from kernel src: { X86_FEATURE_SPEC_CTRL, CPUID_EDX,26, 0x00000007, 0 },
+ # amd: https://developer.amd.com/wp-content/resources/Architecture_Guidelines_Update_Indirect_Branch_Control.pdf
+ # amd: 8000_0008 EBX[14]=1
+ if is_intel; then
+ read_cpuid 0x7 $EDX 26 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES "SPEC_CTRL feature bit"
+ cpuid_spec_ctrl=1
+ cpuid_ibrs='SPEC_CTRL'
+ fi
+ elif is_amd; then
+ read_cpuid 0x80000008 $EBX 14 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES "IBRS_SUPPORT feature bit"
+ cpuid_ibrs='IBRS_SUPPORT'
+ fi
+ else
+ ret=-1
+ pstatus yellow UNKNOWN "unknown CPU"
+ fi
+ if [ $ret -eq 1 ]; then
+ pstatus yellow NO
+ elif [ $ret -eq 2 ]; then
+ pstatus yellow UNKNOWN "is cpuid kernel module available?"
+ cpuid_spec_ctrl=-1
+ fi
+
+ if is_amd; then
+ _info_nol " * CPU indicates preferring IBRS always-on: "
+ # amd
+ read_cpuid 0x80000008 $EBX 16 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+
+ _info_nol " * CPU indicates preferring IBRS over retpoline: "
+ # amd
+ read_cpuid 0x80000008 $EBX 18 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+ # IBPB
+ _info " * Indirect Branch Prediction Barrier (IBPB)"
+ _info_nol " * PRED_CMD MSR is available: "
+ if [ ! -e /dev/cpu/0/msr ] && [ ! -e /dev/cpuctl0 ]; then
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ else
+ # the new MSR 'PRED_CTRL' is at offset 0x49, write-only
+ # here we use dd, it's the same as using 'wrmsr 0x49 0' but without needing the wrmsr tool
+ # if we get a write error, the MSR is not there
+ val=0
+ cpu_mismatch=0
+ for i in $(seq 0 "$idx_max_cpu")
+ do
+ write_msr 0x49 "$i"; ret=$?
+ if [ "$i" -eq 0 ]; then
+ val=$ret
+ else
+ if [ "$ret" -eq $val ]; then
+ continue
+ else
+ cpu_mismatch=1
+ fi
+ fi
+ done
+
+ if [ $val -eq 0 ]; then
+ if [ $cpu_mismatch -eq 0 ]; then
+ pstatus green YES
+ else
+ pstatus green YES "But not in all CPUs"
+ fi
+ elif [ $val -eq 200 ]; then
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+ _info_nol " * CPU indicates IBPB capability: "
+ # CPUID EAX=0x80000008, ECX=0x00 return EBX[12] indicates support for just IBPB.
+ if [ "$cpuid_spec_ctrl" = 1 ]; then
+ # spec_ctrl implies ibpb
+ cpuid_ibpb='SPEC_CTRL'
+ pstatus green YES "SPEC_CTRL feature bit"
+ elif is_intel; then
+ if [ "$cpuid_spec_ctrl" = -1 ]; then
+ pstatus yellow UNKNOWN "is cpuid kernel module available?"
+ else
+ pstatus yellow NO
+ fi
+ elif is_amd; then
+ read_cpuid 0x80000008 $EBX 12 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ cpuid_ibpb='IBPB_SUPPORT'
+ pstatus green YES "IBPB_SUPPORT feature bit"
+ elif [ $ret -eq 1 ]; then
+ pstatus yellow NO
+ else
+ pstatus yellow UNKNOWN "is cpuid kernel module available?"
+ fi
+ fi
+
+ # STIBP
+ _info " * Single Thread Indirect Branch Predictors (STIBP)"
+ _info_nol " * SPEC_CTRL MSR is available: "
+ if [ "$spec_ctrl_msr" = 1 ]; then
+ pstatus green YES
+ elif [ "$spec_ctrl_msr" = 0 ]; then
+ pstatus yellow NO
+ else
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ fi
+
+ _info_nol " * CPU indicates STIBP capability: "
+ # intel: A processor supports STIBP if it enumerates CPUID (EAX=7H,ECX=0):EDX[27] as 1
+ # amd: 8000_0008 EBX[15]=1
+ if is_intel; then
+ read_cpuid 0x7 $EDX 27 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES "Intel STIBP feature bit"
+ #cpuid_stibp='Intel STIBP'
+ fi
+ elif is_amd; then
+ read_cpuid 0x80000008 $EBX 15 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES "AMD STIBP feature bit"
+ #cpuid_stibp='AMD STIBP'
+ fi
+ else
+ ret=-1
+ pstatus yellow UNKNOWN "unknown CPU"
+ fi
+ if [ $ret -eq 1 ]; then
+ pstatus yellow NO
+ elif [ $ret -eq 2 ]; then
+ pstatus yellow UNKNOWN "is cpuid kernel module available?"
+ fi
+
+
+ if is_amd; then
+ _info_nol " * CPU indicates preferring STIBP always-on: "
+ read_cpuid 0x80000008 $EBX 17 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+ if is_intel; then
+ _info " * Enhanced IBRS (IBRS_ALL)"
+ _info_nol " * CPU indicates ARCH_CAPABILITIES MSR availability: "
+ cpuid_arch_capabilities=-1
+ # A processor supports the ARCH_CAPABILITIES MSR if it enumerates CPUID (EAX=7H,ECX=0):EDX[29] as 1
+ read_cpuid 0x7 $EDX 29 1 1; ret=$?
+ if [ $ret -eq 0 ]; then
+ pstatus green YES
+ cpuid_arch_capabilities=1
+ elif [ $ret -eq 2 ]; then
+ pstatus yellow UNKNOWN "is cpuid kernel module available?"
+ else
+ pstatus yellow NO
+ cpuid_arch_capabilities=0
+ fi
+
+ _info_nol " * ARCH_CAPABILITIES MSR advertises IBRS_ALL capability: "
+ capabilities_rdcl_no=-1
+ capabilities_ibrs_all=-1
+ if [ "$cpuid_arch_capabilities" = -1 ]; then
+ pstatus yellow UNKNOWN
+ elif [ "$cpuid_arch_capabilities" != 1 ]; then
+ capabilities_rdcl_no=0
+ capabilities_ibrs_all=0
+ pstatus yellow NO
+ elif [ ! -e /dev/cpu/0/msr ] && [ ! -e /dev/cpuctl0 ]; then
+ spec_ctrl_msr=-1
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ else
+ # the new MSR 'ARCH_CAPABILITIES' is at offset 0x10a
+ # here we use dd, it's the same as using 'rdmsr 0x10a' but without needing the rdmsr tool
+ # if we get a read error, the MSR is not there. bs has to be 8 for msr
+ val=0
+ val_cap_msr=0
+ cpu_mismatch=0
+ for i in $(seq 0 "$idx_max_cpu")
+ do
+ read_msr 0x10a "$i"; ret=$?
+ capabilities=$(echo "$read_msr_value" | awk '{print $8}')
+ if [ "$i" -eq 0 ]; then
+ val=$ret
+ val_cap_msr=$capabilities
+ else
+ if [ "$ret" -eq "$val" ] && [ "$capabilities" -eq "$val_cap_msr" ]; then
+ continue
+ else
+ cpu_mismatch=1
+ fi
+ fi
+ done
+ capabilities=$val_cap_msr
+ capabilities_rdcl_no=0
+ capabilities_ibrs_all=0
+ if [ $val -eq 0 ]; then
+ _debug "capabilities MSR lower byte is $capabilities (decimal)"
+ [ $(( capabilities & 1 )) -eq 1 ] && capabilities_rdcl_no=1
+ [ $(( capabilities & 2 )) -eq 2 ] && capabilities_ibrs_all=1
+ _debug "capabilities says rdcl_no=$capabilities_rdcl_no ibrs_all=$capabilities_ibrs_all"
+ if [ "$capabilities_ibrs_all" = 1 ]; then
+ if [ $cpu_mismatch -eq 0 ]; then
+ pstatus green YES
+ else:
+ pstatus green YES "But not in all CPUs"
+ fi
+ else
+ pstatus yellow NO
+ fi
+ elif [ $val -eq 200 ]; then
+ pstatus yellow UNKNOWN "is msr kernel module available?"
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+ _info_nol " * CPU explicitly indicates not being vulnerable to Meltdown (RDCL_NO): "
+ if [ "$capabilities_rdcl_no" = -1 ]; then
+ pstatus yellow UNKNOWN
+ elif [ "$capabilities_rdcl_no" = 1 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+ _info_nol " * CPU microcode is known to cause stability problems: "
+ if is_ucode_blacklisted; then
+ pstatus red YES "$ucode_found"
+ _warn
+ _warn "The microcode your CPU is running on is known to cause instability problems,"
+ _warn "such as intempestive reboots or random crashes."
+ _warn "You are advised to either revert to a previous microcode version (that might not have"
+ _warn "the mitigations for Spectre), or upgrade to a newer one if available."
+ _warn
+ else
+ pstatus blue NO "$ucode_found"
+ fi
+}
+
+check_cpu_vulnerabilities()
+{
+ _info "* CPU vulnerability to the three speculative execution attack variants"
+ for v in 1 2 3; do
+ _info_nol " * Vulnerable to Variant $v: "
+ if is_cpu_vulnerable $v; then
+ pstatus yellow YES
+ else
+ pstatus green NO
+ fi
+ done
+}
+
+check_redhat_canonical_spectre()
+{
+ # if we were already called, don't do it again
+ [ -n "$redhat_canonical_spectre" ] && return
+
+ if ! which "${opt_arch_prefix}strings" >/dev/null 2>&1; then
+ redhat_canonical_spectre=-1
+ elif [ -n "$kernel_err" ]; then
+ redhat_canonical_spectre=-2
+ else
+ # Red Hat / Ubuntu specific variant1 patch is difficult to detect,
+ # let's use the two same tricks than the official Red Hat detection script uses:
+ if "${opt_arch_prefix}strings" "$kernel" | grep -qw noibrs && "${opt_arch_prefix}strings" "$kernel" | grep -qw noibpb; then
+ # 1) detect their specific variant2 patch. If it's present, it means
+ # that the variant1 patch is also present (both were merged at the same time)
+ _debug "found redhat/canonical version of the variant2 patch (implies variant1)"
+ redhat_canonical_spectre=1
+ elif "${opt_arch_prefix}strings" "$kernel" | grep -q 'x86/pti:'; then
+ # 2) detect their specific variant3 patch. If it's present, but the variant2
+ # is not, it means that only variant1 is present in addition to variant3
+ _debug "found redhat/canonical version of the variant3 patch (implies variant1 but not variant2)"
+ redhat_canonical_spectre=2
+ else
+ redhat_canonical_spectre=0
+ fi
+ fi
+}
+
+
+###################
+# SPECTRE VARIANT 1
+check_variant1()
+{
+ _info "\033[1;34mCVE-2017-5753 [bounds check bypass] aka 'Spectre Variant 1'\033[0m"
+ if [ "$os" = Linux ]; then
+ check_variant1_linux
+ elif echo "$os" | grep -q BSD; then
+ check_variant1_bsd
+ else
+ _warn "Unsupported OS ($os)"
+ fi
+}
+
+check_variant1_linux()
+{
+ status=UNK
+ sys_interface_available=0
+ msg=''
+ if sys_interface_check "/sys/devices/system/cpu/vulnerabilities/spectre_v1"; then
+ # this kernel has the /sys interface, trust it over everything
+ # v0.33+: don't. some kernels have backported the array_index_mask_nospec() workaround without
+ # modifying the vulnerabilities/spectre_v1 file. that's bad. we can't trust it when it says Vulnerable :(
+ # see "silent backport" detection at the bottom of this func
+ sys_interface_available=1
+ fi
+ if [ "$opt_sysfs_only" != 1 ]; then
+ # no /sys interface (or offline mode), fallback to our own ways
+ _info_nol "* Kernel has array_index_mask_nospec (x86): "
+ # vanilla: look for the Linus' mask aka array_index_mask_nospec()
+ # that is inlined at least in raw_copy_from_user (__get_user_X symbols)
+ #mov PER_CPU_VAR(current_task), %_ASM_DX
+ #cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+ #jae bad_get_user
+ # /* array_index_mask_nospec() are the 2 opcodes that follow */
+ #+sbb %_ASM_DX, %_ASM_DX
+ #+and %_ASM_DX, %_ASM_AX
+ #ASM_STAC
+ # x86 64bits: jae(0x0f 0x83 0x?? 0x?? 0x?? 0x??) sbb(0x48 0x19 0xd2) and(0x48 0x21 0xd0)
+ # x86 32bits: cmp(0x3b 0x82 0x?? 0x?? 0x00 0x00) jae(0x73 0x??) sbb(0x19 0xd2) and(0x21 0xd0)
+ if [ -n "$kernel_err" ]; then
+ pstatus yellow UNKNOWN "couldn't check ($kernel_err)"
+ elif ! which perl >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing 'perl' binary, please install it"
+ else
+ perl -ne '/\x0f\x83....\x48\x19\xd2\x48\x21\xd0/ and $found++; END { exit($found) }' "$kernel"; ret=$?
+ if [ $ret -gt 0 ]; then
+ pstatus green YES "$ret occurrence(s) found of 64 bits array_index_mask_nospec()"
+ v1_mask_nospec="64 bits array_index_mask_nospec"
+ else
+ perl -ne '/\x3b\x82..\x00\x00\x73.\x19\xd2\x21\xd0/ and $found++; END { exit($found) }' "$kernel"; ret=$?
+ if [ $ret -gt 0 ]; then
+ pstatus green YES "$ret occurrence(s) found of 32 bits array_index_mask_nospec()"
+ v1_mask_nospec="32 bits array_index_mask_nospec"
+ else
+ pstatus yellow NO
+ fi
+ fi
+ fi
+
+ _info_nol "* Kernel has the Red Hat/Ubuntu patch: "
+ check_redhat_canonical_spectre
+ if [ "$redhat_canonical_spectre" = -1 ]; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}strings' tool, please install it, usually it's in the binutils package"
+ elif [ "$redhat_canonical_spectre" = -2 ]; then
+ pstatus yellow UNKNOWN "couldn't check ($kernel_err)"
+ elif [ "$redhat_canonical_spectre" = 1 ]; then
+ pstatus green YES
+ elif [ "$redhat_canonical_spectre" = 2 ]; then
+ pstatus green YES "but without IBRS"
+ else
+ pstatus yellow NO
+ fi
+
+ _info_nol "* Kernel has mask_nospec64 (arm): "
+ #.macro mask_nospec64, idx, limit, tmp
+ #sub \tmp, \idx, \limit
+ #bic \tmp, \tmp, \idx
+ #and \idx, \idx, \tmp, asr #63
+ #csdb
+ #.endm
+ #$ aarch64-linux-gnu-objdump -d vmlinux | grep -w bic -A1 -B1 | grep -w sub -A2 | grep -w and -B2
+ #ffffff8008082e44: cb190353 sub x19, x26, x25
+ #ffffff8008082e48: 8a3a0273 bic x19, x19, x26
+ #ffffff8008082e4c: 8a93ff5a and x26, x26, x19, asr #63
+ #ffffff8008082e50: d503229f hint #0x14
+ # /!\ can also just be "csdb" instead of "hint #0x14" for native objdump
+ #
+ # if we have v1_mask_nospec or redhat_canonical_spectre>0, don't bother disassembling the kernel, the answer is no.
+ if [ -n "$v1_mask_nospec" ] || [ "$redhat_canonical_spectre" -gt 0 ]; then
+ pstatus yellow NO
+ elif [ -n "$kernel_err" ]; then
+ pstatus yellow UNKNOWN "couldn't check ($kernel_err)"
+ elif ! which perl >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing 'perl' binary, please install it"
+ elif ! which "${opt_arch_prefix}objdump" >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}objdump' tool, please install it, usually it's in the binutils package"
+ else
+ "${opt_arch_prefix}objdump" -d "$kernel" | perl -ne 'push @r, $_; /\s(hint|csdb)\s/ && $r[0]=~/\ssub\s+(x\d+)/ && $r[1]=~/\sbic\s+$1,\s+$1,/ && $r[2]=~/\sand\s/ && exit(9); shift @r if @r>3'; ret=$?
+ if [ "$ret" -eq 9 ]; then
+ pstatus green YES "mask_nospec64 macro is present and used"
+ v1_mask_nospec="arm mask_nospec64"
+ else
+ pstatus yellow NO
+ fi
+ fi
+
+
+ if [ "$opt_verbose" -ge 2 ] || ( [ -z "$v1_mask_nospec" ] && [ "$redhat_canonical_spectre" != 1 ] && [ "$redhat_canonical_spectre" != 2 ] ); then
+ # this is a slow heuristic and we don't need it if we already know the kernel is patched
+ # but still show it in verbose mode
+ _info_nol "* Checking count of LFENCE instructions following a jump in kernel... "
+ if [ -n "$kernel_err" ]; then
+ pstatus yellow UNKNOWN "couldn't check ($kernel_err)"
+ else
+ if ! which "${opt_arch_prefix}objdump" >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}objdump' tool, please install it, usually it's in the binutils package"
+ else
+ # here we disassemble the kernel and count the number of occurrences of the LFENCE opcode
+ # in non-patched kernels, this has been empirically determined as being around 40-50
+ # in patched kernels, this is more around 70-80, sometimes way higher (100+)
+ # v0.13: 68 found in a 3.10.23-xxxx-std-ipv6-64 (with lots of modules compiled-in directly), which doesn't have the LFENCE patches,
+ # so let's push the threshold to 70.
+ # v0.33+: now only count lfence opcodes after a jump, way less error-prone
+ # non patched kernel have between 0 and 20 matches, patched ones have at least 40-45
+ nb_lfence=$("${opt_arch_prefix}objdump" -d "$kernel" 2>/dev/null | grep -w -B1 lfence | grep -Ewc 'jmp|jne|je')
+ if [ "$nb_lfence" -lt 30 ]; then
+ pstatus yellow NO "only $nb_lfence jump-then-lfence instructions found, should be >= 30 (heuristic)"
+ else
+ v1_lfence=1
+ pstatus green YES "$nb_lfence jump-then-lfence instructions found, which is >= 30 (heuristic)"
+ fi
+ fi
+ fi
+ fi
+
+ else
+ # we have no sysfs but were asked to use it only!
+ msg="/sys vulnerability interface use forced, but it's not available!"
+ status=UNK
+ fi
+
+ # report status
+ cve='CVE-2017-5753'
+
+ if ! is_cpu_vulnerable 1; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ elif [ -z "$msg" ]; then
+ # if msg is empty, sysfs check didn't fill it, rely on our own test
+ if [ -n "$v1_mask_nospec" ]; then
+ pvulnstatus $cve OK "Kernel source has been patched to mitigate the vulnerability ($v1_mask_nospec)"
+ elif [ "$redhat_canonical_spectre" = 1 ] || [ "$redhat_canonical_spectre" = 2 ]; then
+ pvulnstatus $cve OK "Kernel source has been patched to mitigate the vulnerability (Red Hat/Ubuntu patch)"
+ elif [ "$v1_lfence" = 1 ]; then
+ pvulnstatus $cve OK "Kernel source has PROBABLY been patched to mitigate the vulnerability (jump-then-lfence instructions heuristic)"
+ elif [ "$kernel_err" ]; then
+ pvulnstatus $cve UNK "Couldn't find kernel image or tools missing to execute the checks"
+ explain "Re-run this script with root privileges, after installing the missing tools indicated above"
+ else
+ pvulnstatus $cve VULN "Kernel source needs to be patched to mitigate the vulnerability"
+ explain "Your kernel is too old to have the mitigation for Variant 1, you should upgrade to a newer kernel. If you're using a Linux distro and didn't compile the kernel yourself, you should upgrade your distro to get a newer kernel."
+ fi
+ else
+ if [ "$msg" = "Vulnerable" ] && [ -n "$v1_mask_nospec" ]; then
+ pvulnstatus $cve OK "Kernel source has been patched to mitigate the vulnerability (silent backport of array_index_mask_nospec)"
+ else
+ if [ "$msg" = "Vulnerable" ]; then
+ msg="Kernel source needs to be patched to mitigate the vulnerability"
+ _explain="Your kernel is too old to have the mitigation for Variant 1, you should upgrade to a newer kernel. If you're using a Linux distro and didn't compile the kernel yourself, you should upgrade your distro to get a newer kernel."
+ fi
+ pvulnstatus $cve "$status" "$msg"
+ [ -n "$_explain" ] && explain "$_explain"
+ unset _explain
+ fi
+ fi
+}
+
+check_variant1_bsd()
+{
+ cve='CVE-2017-5753'
+ if ! is_cpu_vulnerable 1; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ else
+ pvulnstatus $cve VULN "no mitigation for BSD yet"
+ fi
+}
+
+
+###################
+# SPECTRE VARIANT 2
+check_variant2()
+{
+ _info "\033[1;34mCVE-2017-5715 [branch target injection] aka 'Spectre Variant 2'\033[0m"
+ if [ "$os" = Linux ]; then
+ check_variant2_linux
+ elif echo "$os" | grep -q BSD; then
+ check_variant2_bsd
+ else
+ _warn "Unsupported OS ($os)"
+ fi
+}
+
+check_variant2_linux()
+{
+ status=UNK
+ sys_interface_available=0
+ msg=''
+ if sys_interface_check "/sys/devices/system/cpu/vulnerabilities/spectre_v2"; then
+ # this kernel has the /sys interface, trust it over everything
+ sys_interface_available=1
+ fi
+ if [ "$opt_sysfs_only" != 1 ]; then
+ _info "* Mitigation 1"
+
+ ibrs_can_tell=0
+ ibrs_supported=''
+ ibrs_enabled=''
+ ibpb_can_tell=0
+ ibpb_supported=''
+ ibpb_enabled=''
+
+ if [ "$opt_live" = 1 ]; then
+ # in live mode, we can check for the ibrs_enabled file in debugfs
+ # all versions of the patches have it (NOT the case of IBPB or KPTI)
+ ibrs_can_tell=1
+ mount_debugfs
+ for dir in \
+ /sys/kernel/debug \
+ /sys/kernel/debug/x86 \
+ /proc/sys/kernel; do
+ if [ -e "$dir/ibrs_enabled" ]; then
+ # if the file is there, we have IBRS compiled-in
+ # /sys/kernel/debug/ibrs_enabled: vanilla
+ # /sys/kernel/debug/x86/ibrs_enabled: Red Hat (see https://access.redhat.com/articles/3311301)
+ # /proc/sys/kernel/ibrs_enabled: OpenSUSE tumbleweed
+ specex_knob_dir=$dir
+ ibrs_supported="$dir/ibrs_enabled exists"
+ ibrs_enabled=$(cat "$dir/ibrs_enabled" 2>/dev/null)
+ _debug "ibrs: found $dir/ibrs_enabled=$ibrs_enabled"
+ # if ibrs_enabled is there, ibpb_enabled will be in the same dir
+ if [ -e "$dir/ibpb_enabled" ]; then
+ # if the file is there, we have IBPB compiled-in (see note above for IBRS)
+ ibpb_supported="$dir/ibpb_enabled exists"
+ ibpb_enabled=$(cat "$dir/ibpb_enabled" 2>/dev/null)
+ _debug "ibpb: found $dir/ibpb_enabled=$ibpb_enabled"
+ else
+ _debug "ibpb: $dir/ibpb_enabled file doesn't exist"
+ fi
+ break
+ else
+ _debug "ibrs: $dir/ibrs_enabled file doesn't exist"
+ fi
+ done
+ # on some newer kernels, the spec_ctrl_ibrs flag in "$procfs/cpuinfo"
+ # is set when ibrs has been administratively enabled (usually from cmdline)
+ # which in that case means ibrs is supported *and* enabled for kernel & user
+ # as per the ibrs patch series v3
+ if [ -z "$ibrs_supported" ]; then
+ if grep ^flags "$procfs/cpuinfo" | grep -qw spec_ctrl_ibrs; then
+ _debug "ibrs: found spec_ctrl_ibrs flag in $procfs/cpuinfo"
+ ibrs_supported="spec_ctrl_ibrs flag in $procfs/cpuinfo"
+ # enabled=2 -> kernel & user
+ ibrs_enabled=2
+ # XXX and what about ibpb ?
+ fi
+ fi
+ if [ -e "/sys/devices/system/cpu/vulnerabilities/spectre_v2" ]; then
+ # when IBPB is enabled on 4.15+, we can see it in sysfs
+ if grep -q ', IBPB' "/sys/devices/system/cpu/vulnerabilities/spectre_v2"; then
+ _debug "ibpb: found enabled in sysfs"
+ [ -z "$ibpb_supported" ] && ibpb_supported='IBPB found enabled in sysfs'
+ [ -z "$ibpb_enabled" ] && ibpb_enabled=1
+ fi
+ # when IBRS_FW is enabled on 4.15+, we can see it in sysfs
+ if grep -q ', IBRS_FW' "/sys/devices/system/cpu/vulnerabilities/spectre_v2"; then
+ _debug "ibrs: found IBRS_FW in sysfs"
+ [ -z "$ibrs_supported" ] && ibrs_supported='found IBRS_FW in sysfs'
+ ibrs_fw_enabled=1
+ fi
+ # when IBRS is enabled on 4.15+, we can see it in sysfs
+ if grep -q 'Indirect Branch Restricted Speculation' "/sys/devices/system/cpu/vulnerabilities/spectre_v2"; then
+ _debug "ibrs: found IBRS in sysfs"
+ [ -z "$ibrs_supported" ] && ibrs_supported='found IBRS in sysfs'
+ [ -z "$ibrs_enabled" ] && ibrs_enabled=3
+ fi
+ fi
+ # in live mode, if ibrs or ibpb is supported and we didn't find these are enabled, then they are not
+ [ -n "$ibrs_supported" ] && [ -z "$ibrs_enabled" ] && ibrs_enabled=0
+ [ -n "$ibpb_supported" ] && [ -z "$ibpb_enabled" ] && ibpb_enabled=0
+ fi
+ if [ -z "$ibrs_supported" ]; then
+ check_redhat_canonical_spectre
+ if [ "$redhat_canonical_spectre" = 1 ]; then
+ ibrs_supported="Red Hat/Ubuntu variant"
+ ibpb_supported="Red Hat/Ubuntu variant"
+ fi
+ fi
+ if [ -z "$ibrs_supported" ] && [ -n "$kernel" ]; then
+ if ! which "${opt_arch_prefix}strings" >/dev/null 2>&1; then
+ :
+ else
+ ibrs_can_tell=1
+ ibrs_supported=$("${opt_arch_prefix}strings" "$kernel" | grep -Fw -e ', IBRS_FW' | head -1)
+ if [ -n "$ibrs_supported" ]; then
+ _debug "ibrs: found ibrs evidence in kernel image ($ibrs_supported)"
+ ibrs_supported="found '$ibrs_supported' in kernel image"
+ fi
+ fi
+ fi
+ if [ -z "$ibrs_supported" ] && [ -n "$opt_map" ]; then
+ ibrs_can_tell=1
+ if grep -q spec_ctrl "$opt_map"; then
+ ibrs_supported="found spec_ctrl in symbols file"
+ _debug "ibrs: found '*spec_ctrl*' symbol in $opt_map"
+ fi
+ fi
+ # recent (4.15) vanilla kernels have IBPB but not IBRS, and without the debugfs tunables of Red Hat
+ # we can detect it directly in the image
+ if [ -z "$ibpb_supported" ] && [ -n "$kernel" ]; then
+ if ! which "${opt_arch_prefix}strings" >/dev/null 2>&1; then
+ :
+ else
+ ibpb_can_tell=1
+ ibpb_supported=$("${opt_arch_prefix}strings" "$kernel" | grep -Fw -e 'ibpb' -e ', IBPB' | head -1)
+ if [ -n "$ibpb_supported" ]; then
+ _debug "ibpb: found ibpb evidence in kernel image ($ibpb_supported)"
+ ibpb_supported="found '$ibpb_supported' in kernel image"
+ fi
+ fi
+ fi
+
+ _info_nol " * Kernel is compiled with IBRS support: "
+ if [ -z "$ibrs_supported" ]; then
+ if [ "$ibrs_can_tell" = 1 ]; then
+ pstatus yellow NO
+ else
+ # if we're in offline mode without System.map, we can't really know
+ pstatus yellow UNKNOWN "in offline mode, we need the kernel image and System.map to be able to tell"
+ fi
+ else
+ if [ "$opt_verbose" -ge 2 ]; then
+ pstatus green YES "$ibrs_supported"
+ else
+ pstatus green YES
+ fi
+ fi
+
+ _info_nol " * IBRS enabled and active: "
+ if [ "$opt_live" = 1 ]; then
+ if [ "$ibpb_enabled" = 2 ]; then
+ # if ibpb=2, ibrs is forcefully=0
+ pstatus blue NO "IBPB used instead of IBRS in all kernel entrypoints"
+ else
+ # 0 means disabled
+ # 1 is enabled only for kernel space
+ # 2 is enabled for kernel and user space
+ # 3 is enabled
+ case "$ibrs_enabled" in
+ 0)
+ if [ "$ibrs_fw_enabled" = 1 ]; then
+ pstatus blue YES "for firmware code only"
+ else
+ pstatus yellow NO
+ fi
+ ;;
+ 1) if [ "$ibrs_fw_enabled" = 1 ]; then pstatus green YES "for kernel space and firmware code"; else pstatus green YES "for kernel space"; fi;;
+ 2) if [ "$ibrs_fw_enabled" = 1 ]; then pstatus green YES "for kernel, user space, and firmware code" ; else pstatus green YES "for both kernel and user space"; fi;;
+ 3) if [ "$ibrs_fw_enabled" = 1 ]; then pstatus green YES "for kernel and firmware code"; else pstatus green YES; fi;;
+ *) pstatus yellow UNKNOWN;;
+ esac
+ fi
+ else
+ pstatus blue N/A "not testable in offline mode"
+ fi
+
+ _info_nol " * Kernel is compiled with IBPB support: "
+ if [ -z "$ibpb_supported" ]; then
+ if [ "$ibpb_can_tell" = 1 ]; then
+ pstatus yellow NO
+ else
+ # if we're in offline mode without System.map, we can't really know
+ pstatus yellow UNKNOWN "in offline mode, we need the kernel image to be able to tell"
+ fi
+ else
+ if [ "$opt_verbose" -ge 2 ]; then
+ pstatus green YES "$ibpb_supported"
+ else
+ pstatus green YES
+ fi
+ fi
+
+ _info_nol " * IBPB enabled and active: "
+ if [ "$opt_live" = 1 ]; then
+ case "$ibpb_enabled" in
+ "")
+ if [ "$ibrs_supported" = 1 ]; then
+ pstatus yellow UNKNOWN
+ else
+ pstatus yellow NO
+ fi
+ ;;
+ 0)
+ pstatus yellow NO
+ ;;
+ 1) pstatus green YES;;
+ 2) pstatus green YES "IBPB used instead of IBRS in all kernel entrypoints";;
+ *) pstatus yellow UNKNOWN;;
+ esac
+ else
+ pstatus blue N/A "not testable in offline mode"
+ fi
+
+ _info "* Mitigation 2"
+ _info_nol " * Kernel has branch predictor hardening (arm): "
+ if [ -r "$opt_config" ]; then
+ bp_harden_can_tell=1
+ bp_harden=$(grep -w 'CONFIG_HARDEN_BRANCH_PREDICTOR=y' "$opt_config")
+ if [ -n "$bp_harden" ]; then
+ pstatus green YES
+ _debug "bp_harden: found '$bp_harden' in $opt_config"
+ fi
+ fi
+ if [ -z "$bp_harden" ] && [ -n "$opt_map" ]; then
+ bp_harden_can_tell=1
+ bp_harden=$(grep -w bp_hardening_data "$opt_map")
+ if [ -n "$bp_harden" ]; then
+ pstatus green YES
+ _debug "bp_harden: found '$bp_harden' in $opt_map"
+ fi
+ fi
+ if [ -z "$bp_harden" ]; then
+ if [ "$bp_harden_can_tell" = 1 ]; then
+ pstatus yellow NO
+ else
+ pstatus yellow UNKNOWN
+ fi
+ fi
+
+ _info_nol " * Kernel compiled with retpoline option: "
+ # We check the RETPOLINE kernel options
+ if [ -r "$opt_config" ]; then
+ if grep -q '^CONFIG_RETPOLINE=y' "$opt_config"; then
+ pstatus green YES
+ retpoline=1
+ # shellcheck disable=SC2046
+ _debug 'retpoline: found '$(grep '^CONFIG_RETPOLINE' "$opt_config")" in $opt_config"
+ else
+ pstatus yellow NO
+ fi
+ else
+ pstatus yellow UNKNOWN "couldn't read your kernel configuration"
+ fi
+
+ if [ "$retpoline" = 1 ]; then
+ # Now check if the compiler used to compile the kernel knows how to insert retpolines in generated asm
+ # For gcc, this is -mindirect-branch=thunk-extern (detected by the kernel makefiles)
+ # See gcc commit https://github.com/hjl-tools/gcc/commit/23b517d4a67c02d3ef80b6109218f2aadad7bd79
+ # In latest retpoline LKML patches, the noretpoline_setup symbol exists only if CONFIG_RETPOLINE is set
+ # *AND* if the compiler is retpoline-compliant, so look for that symbol
+ #
+ # if there is "retpoline" in the file and NOT "minimal", then it's full retpoline
+ # (works for vanilla and Red Hat variants)
+ if [ "$opt_live" = 1 ] && [ -e "/sys/devices/system/cpu/vulnerabilities/spectre_v2" ]; then
+ if grep -qwi retpoline /sys/devices/system/cpu/vulnerabilities/spectre_v2; then
+ if grep -qwi minimal /sys/devices/system/cpu/vulnerabilities/spectre_v2; then
+ retpoline_compiler=0
+ retpoline_compiler_reason="kernel reports minimal retpoline compilation"
+ else
+ retpoline_compiler=1
+ retpoline_compiler_reason="kernel reports full retpoline compilation"
+ fi
+ fi
+ elif [ -n "$opt_map" ]; then
+ # look for the symbol
+ if grep -qw noretpoline_setup "$opt_map"; then
+ retpoline_compiler=1
+ retpoline_compiler_reason="noretpoline_setup symbol found in System.map"
+ fi
+ elif [ -n "$kernel" ]; then
+ # look for the symbol
+ if which "${opt_arch_prefix}nm" >/dev/null 2>&1; then
+ # the proper way: use nm and look for the symbol
+ if "${opt_arch_prefix}nm" "$kernel" 2>/dev/null | grep -qw 'noretpoline_setup'; then
+ retpoline_compiler=1
+ retpoline_compiler_reason="noretpoline_setup found in kernel symbols"
+ fi
+ elif grep -q noretpoline_setup "$kernel"; then
+ # if we don't have nm, nevermind, the symbol name is long enough to not have
+ # any false positive using good old grep directly on the binary
+ retpoline_compiler=1
+ retpoline_compiler_reason="noretpoline_setup found in kernel"
+ fi
+ fi
+ if [ -n "$retpoline_compiler" ]; then
+ _info_nol " * Kernel compiled with a retpoline-aware compiler: "
+ if [ "$retpoline_compiler" = 1 ]; then
+ if [ -n "$retpoline_compiler_reason" ]; then
+ pstatus green YES "$retpoline_compiler_reason"
+ else
+ pstatus green YES
+ fi
+ else
+ if [ -n "$retpoline_compiler_reason" ]; then
+ pstatus red NO "$retpoline_compiler_reason"
+ else
+ pstatus red NO
+ fi
+ fi
+ fi
+ fi
+
+ # only Red Hat has a tunable to disable it on runtime
+ if [ "$opt_live" = 1 ]; then
+ if [ -e "$specex_knob_dir/retp_enabled" ]; then
+ retp_enabled=$(cat "$specex_knob_dir/retp_enabled" 2>/dev/null)
+ _debug "retpoline: found $specex_knob_dir/retp_enabled=$retp_enabled"
+ _info_nol " * Retpoline is enabled: "
+ if [ "$retp_enabled" = 1 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+ fi
+ fi
+
+ # only for information, in verbose mode
+ if [ "$opt_verbose" -ge 2 ]; then
+ _info_nol " * Local gcc is retpoline-aware: "
+ if which gcc >/dev/null 2>&1; then
+ if [ -n "$(gcc -mindirect-branch=thunk-extern --version 2>&1 >/dev/null)" ]; then
+ pstatus blue NO
+ else
+ pstatus green YES
+ fi
+ else
+ pstatus blue NO "gcc is not installed"
+ fi
+ fi
+
+ if is_skylake_cpu || [ "$opt_verbose" -ge 2 ]; then
+ _info_nol " * Kernel supports RSB filling: "
+ if ! which "${opt_arch_prefix}strings" >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}strings' tool, please install it, usually it's in the binutils package"
+ elif [ -z "$kernel" ]; then
+ pstatus yellow UNKNOWN "kernel image missing"
+ else
+ rsb_filling=$("${opt_arch_prefix}strings" "$kernel" | grep -w 'Filling RSB on context switch')
+ if [ -n "$rsb_filling" ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+ fi
+ fi
+
+ elif [ "$sys_interface_available" = 0 ]; then
+ # we have no sysfs but were asked to use it only!
+ msg="/sys vulnerability interface use forced, but it's not available!"
+ status=UNK
+ fi
+
+ cve='CVE-2017-5715'
+ if ! is_cpu_vulnerable 2; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ else
+ if [ "$retpoline" = 1 ] && [ "$retpoline_compiler" = 1 ] && [ "$retp_enabled" != 0 ] && [ -n "$ibpb_enabled" ] && [ "$ibpb_enabled" -ge 1 ] && ( ! is_skylake_cpu || [ -n "$rsb_filling" ] ); then
+ pvulnstatus $cve OK "Full retpoline + IBPB are mitigating the vulnerability"
+ elif [ "$retpoline" = 1 ] && [ "$retpoline_compiler" = 1 ] && [ "$retp_enabled" != 0 ] && [ "$opt_paranoid" = 0 ] && ( ! is_skylake_cpu || [ -n "$rsb_filling" ] ); then
+ pvulnstatus $cve OK "Full retpoline is mitigating the vulnerability"
+ if [ -n "$cpuid_ibpb" ]; then
+ _warn "You should enable IBPB to complete retpoline as a Variant 2 mitigation"
+ else
+ _warn "IBPB is considered as a good addition to retpoline for Variant 2 mitigation, but your CPU microcode doesn't support it"
+ fi
+ elif [ -n "$ibrs_enabled" ] && [ -n "$ibpb_enabled" ] && [ "$ibrs_enabled" -ge 1 ] && [ "$ibpb_enabled" -ge 1 ]; then
+ pvulnstatus $cve OK "IBRS + IBPB are mitigating the vulnerability"
+ elif [ "$ibpb_enabled" = 2 ] && ! is_cpu_smt_enabled; then
+ pvulnstatus $cve OK "Full IBPB is mitigating the vulnerability"
+ elif [ -n "$bp_harden" ]; then
+ pvulnstatus $cve OK "Branch predictor hardening mitigates the vulnerability"
+ elif [ -z "$bp_harden" ] && [ "$cpu_vendor" = ARM ]; then
+ pvulnstatus $cve VULN "Branch predictor hardening is needed to mitigate the vulnerability"
+ explain "Your kernel has not been compiled with the CONFIG_UNMAP_KERNEL_AT_EL0 option, recompile it with this option enabled."
+ elif [ "$opt_live" != 1 ]; then
+ if [ "$retpoline" = 1 ] && [ -n "$ibpb_supported" ]; then
+ pvulnstatus $cve OK "offline mode: kernel supports retpoline + IBPB to mitigate the vulnerability"
+ elif [ -n "$ibrs_supported" ] && [ -n "$ibpb_supported" ]; then
+ pvulnstatus $cve OK "offline mode: kernel supports IBRS + IBPB to mitigate the vulnerability"
+ elif [ "$ibrs_can_tell" != 1 ]; then
+ pvulnstatus $cve UNK "offline mode: not enough information"
+ explain "Re-run this script with root privileges, and give it the kernel image (--kernel), the kernel configuration (--config) and the System.map file (--map) corresponding to the kernel you would like to inspect."
+ fi
+ fi
+
+ # if we arrive here and didn't already call pvulnstatus, then it's VULN, let's explain why
+ if [ "$pvulnstatus_last_cve" != "$cve" ]; then
+ # explain what's needed for this CPU
+ if is_skylake_cpu; then
+ pvulnstatus $cve VULN "IBRS+IBPB or retpoline+IBPB+RBS filling, is needed to mitigate the vulnerability"
+ explain "To mitigate this vulnerability, you need either IBRS + IBPB, both requiring hardware support from your CPU microcode in addition to kernel support, or a kernel compiled with retpoline and IBPB, with retpoline requiring a retpoline-aware compiler (re-run this script with -v to know if your version of gcc is retpoline-aware) and IBPB requiring hardware support from your CPU microcode. You also need a recent-enough kernel that supports RSB filling if you plan to use retpoline. For Skylake+ CPUs, the IBRS + IBPB approach is generally preferred as it guarantees complete protection, and the performance impact is not as high as with older CPUs in comparison with retpoline. More information about how to enable the missing bits for those two possible mitigations on your system follow. You only need to take one of the two approaches."
+ elif is_zen_cpu; then
+ pvulnstatus $cve VULN "retpoline+IBPB is needed to mitigate the vulnerability"
+ explain "To mitigate this vulnerability, You need a kernel compiled with retpoline + IBPB support, with retpoline requiring a retpoline-aware compiler (re-run this script with -v to know if your version of gcc is retpoline-aware) and IBPB requiring hardware support from your CPU microcode."
+ elif is_intel || is_amd; then
+ pvulnstatus $cve VULN "IBRS+IBPB or retpoline+IBPB is needed to mitigate the vulnerability"
+ explain "To mitigate this vulnerability, you need either IBRS + IBPB, both requiring hardware support from your CPU microcode in addition to kernel support, or a kernel compiled with retpoline and IBPB, with retpoline requiring a retpoline-aware compiler (re-run this script with -v to know if your version of gcc is retpoline-aware) and IBPB requiring hardware support from your CPU microcode. The retpoline + IBPB approach is generally preferred as the performance impact is lower. More information about how to enable the missing bits for those two possible mitigations on your system follow. You only need to take one of the two approaches."
+ else
+ # in that case, we might want to trust sysfs if it's there
+ if [ -n "$msg" ]; then
+ [ "$msg" = Vulnerable ] && msg="no known mitigation exists for your CPU vendor ($cpu_vendor)"
+ pvulnstatus $cve $status "$msg"
+ else
+ pvulnstatus $cve VULN "no known mitigation exists for your CPU vendor ($cpu_vendor)"
+ fi
+ fi
+ fi
+
+ # if we are in live mode, we can check for a lot more stuff and explain further
+ if [ "$opt_live" = 1 ] && [ "$vulnstatus" != "OK" ]; then
+ _explain_hypervisor="An updated CPU microcode will have IBRS/IBPB capabilities indicated in the Hardware Check section above. If you're running under an hypervisor (KVM, Xen, VirtualBox, VMware, ...), the hypervisor needs to be up to date to be able to export the new host CPU flags to the guest. You can run this script on the host to check if the host CPU is IBRS/IBPB. If it is, and it doesn't show up in the guest, upgrade the hypervisor."
+ # IBPB (amd & intel)
+ if ( [ -z "$ibpb_enabled" ] || [ "$ibpb_enabled" = 0 ] ) && ( is_intel || is_amd ); then
+ if [ -z "$cpuid_ibpb" ]; then
+ explain "The microcode of your CPU needs to be upgraded to be able to use IBPB. This is usually done at boot time by your kernel (the upgrade is not persistent across reboots which is why it's done at each boot). If you're using a distro, make sure you are up to date, as microcode updates are usually shipped alongside with the distro kernel. Availability of a microcode update for you CPU model depends on your CPU vendor. You can usually find out online if a microcode update is available for your CPU by searching for your CPUID (indicated in the Hardware Check section). $_explain_hypervisor"
+ fi
+ if [ -z "$ibpb_supported" ]; then
+ explain "Your kernel doesn't have IBPB support, so you need to either upgrade your kernel (if you're using a distro) or recompiling a more recent kernel."
+ fi
+ if [ -n "$cpuid_ibpb" ] && [ -n "$ibpb_supported" ]; then
+ if [ -e "$specex_knob_dir/ibpb_enabled" ]; then
+ # newer (April 2018) Red Hat kernels have ibpb_enabled as ro, and automatically enables it with retpoline
+ if [ ! -w "$specex_knob_dir/ibpb_enabled" ] && [ -e "$specex_knob_dir/retp_enabled" ]; then
+ explain "Both your CPU and your kernel have IBPB support, but it is currently disabled. You kernel should enable IBPB automatically if you enable retpoline. You may enable it with \`echo 1 > $specex_knob_dir/retp_enabled\`."
+ else
+ explain "Both your CPU and your kernel have IBPB support, but it is currently disabled. You may enable it with \`echo 1 > $specex_knob_dir/ibpb_enabled\`."
+ fi
+ else
+ explain "Both your CPU and your kernel have IBPB support, but it is currently disabled. You may enable it. Check in your distro's documentation on how to do this."
+ fi
+ fi
+ elif [ "$ibpb_enabled" = 2 ] && is_cpu_smt_enabled; then
+ explain "You have ibpb_enabled set to 2, but it only offers sufficient protection when simultaneous multi-threading (aka SMT or HyperThreading) is disabled. You should reboot your system with the kernel parameter \`nosmt\`."
+ fi
+ # /IBPB
+
+ # IBRS (amd & intel)
+ if ( [ -z "$ibrs_enabled" ] || [ "$ibrs_enabled" = 0 ] ) && ( is_intel || is_amd ); then
+ if [ -z "$cpuid_ibrs" ]; then
+ explain "The microcode of your CPU needs to be upgraded to be able to use IBRS. This is usually done at boot time by your kernel (the upgrade is not persistent across reboots which is why it's done at each boot). If you're using a distro, make sure you are up to date, as microcode updates are usually shipped alongside with the distro kernel. Availability of a microcode update for you CPU model depends on your CPU vendor. You can usually find out online if a microcode update is available for your CPU by searching for your CPUID (indicated in the Hardware Check section). $_explain_hypervisor"
+ fi
+ if [ -z "$ibrs_supported" ]; then
+ explain "Your kernel doesn't have IBRS support, so you need to either upgrade your kernel (if you're using a distro) or recompiling a more recent kernel."
+ fi
+ if [ -n "$cpuid_ibrs" ] && [ -n "$ibrs_supported" ]; then
+ if [ -e "$specex_knob_dir/ibrs_enabled" ]; then
+ explain "Both your CPU and your kernel have IBRS support, but it is currently disabled. You may enable it with \`echo 1 > $specex_knob_dir/ibrs_enabled\`."
+ else
+ explain "Both your CPU and your kernel have IBRS support, but it is currently disabled. You may enable it. Check in your distro's documentation on how to do this."
+ fi
+ fi
+ fi
+ # /IBRS
+ unset _explain_hypervisor
+
+ # RETPOLINE (amd & intel)
+ if is_amd || is_intel; then
+ if [ "$retpoline" = 0 ]; then
+ explain "Your kernel is not compiled with retpoline support, so you need to either upgrade your kernel (if you're using a distro) or recompile your kernel with the CONFIG_RETPOLINE option enabled. You also need to compile your kernel with a retpoline-aware compiler (re-run this script with -v to know if your version of gcc is retpoline-aware)."
+ elif [ "$retpoline" = 1 ] && [ "$retpoline_compiler" = 0 ]; then
+ explain "Your kernel is compiled with retpoline, but without a retpoline-aware compiler (re-run this script with -v to know if your version of gcc is retpoline-aware)."
+ elif [ "$retpoline" = 1 ] && [ "$retpoline_compiler" = 1 ] && [ "$retp_enabled" = 0 ]; then
+ explain "Your kernel has retpoline support and has been compiled with a retpoline-aware compiler, but retpoline is disabled. You should enable it with \`echo 1 > $specex_knob_dir/retp_enabled\`."
+ fi
+ fi
+ # /RETPOLINE
+ fi
+ fi
+ # sysfs msgs:
+ #1 "Vulnerable"
+ #2 "Vulnerable: Minimal generic ASM retpoline"
+ #2 "Vulnerable: Minimal AMD ASM retpoline"
+ # "Mitigation: Full generic retpoline"
+ # "Mitigation: Full AMD retpoline"
+ # $MITIGATION + ", IBPB"
+ # $MITIGATION + ", IBRS_FW"
+ #5 $MITIGATION + " - vulnerable module loaded"
+ # Red Hat only:
+ #2 "Vulnerable: Minimal ASM retpoline",
+ #3 "Vulnerable: Retpoline without IBPB",
+ #4 "Vulnerable: Retpoline on Skylake+",
+ #5 "Vulnerable: Retpoline with unsafe module(s)",
+ # "Mitigation: Full retpoline",
+ # "Mitigation: Full retpoline and IBRS (user space)",
+ # "Mitigation: IBRS (kernel)",
+ # "Mitigation: IBRS (kernel and user space)",
+ # "Mitigation: IBP disabled",
+}
+
+check_variant2_bsd()
+{
+ _info "* Mitigation 1"
+ _info_nol " * Kernel supports IBRS: "
+ ibrs_disabled=$(sysctl -n hw.ibrs_disable 2>/dev/null)
+ if [ -z "$ibrs_disabled" ]; then
+ pstatus yellow NO
+ else
+ pstatus green YES
+ fi
+
+ _info_nol " * IBRS enabled and active: "
+ ibrs_active=$(sysctl -n hw.ibrs_active 2>/dev/null)
+ if [ "$ibrs_active" = 1 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+
+ _info "* Mitigation 2"
+ _info_nol " * Kernel compiled with RETPOLINE: "
+ if [ -n "$kernel_err" ]; then
+ pstatus yellow UNKNOWN "couldn't check ($kernel_err)"
+ else
+ if ! which "${opt_arch_prefix}readelf" >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}readelf' tool, please install it, usually it's in the binutils package"
+ else
+ nb_thunks=$("${opt_arch_prefix}readelf" -s "$kernel" | grep -c -e __llvm_retpoline_ -e __llvm_external_retpoline_ -e __x86_indirect_thunk_)
+ if [ "$nb_thunks" -gt 0 ]; then
+ retpoline=1
+ pstatus green YES "found $nb_thunks thunk(s)"
+ else
+ pstatus yellow NO
+ fi
+ fi
+ fi
+
+ cve='CVE-2017-5715'
+ if ! is_cpu_vulnerable 2; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ elif [ "$retpoline" = 1 ]; then
+ pvulnstatus $cve OK "Retpoline mitigates the vulnerability"
+ elif [ "$ibrs_active" = 1 ]; then
+ pvulnstatus $cve OK "IBRS mitigates the vulnerability"
+ elif [ "$ibrs_disabled" = 0 ]; then
+ pvulnstatus $cve VULN "IBRS is supported by your kernel but your CPU microcode lacks support"
+ explain "The microcode of your CPU needs to be upgraded to be able to use IBRS. Availability of a microcode update for you CPU model depends on your CPU vendor. You can usually find out online if a microcode update is available for your CPU by searching for your CPUID (indicated in the Hardware Check section). To do a microcode update, you can search the ports for the \`cpupdate\` tool. Microcode updates done this way are not reboot-proof, so be sure to do it every time the system boots up."
+ elif [ "$ibrs_disabled" = 1 ]; then
+ pvulnstatus $cve VULN "IBRS is supported but administratively disabled on your system"
+ explain "To enable IBRS, use \`sysctl hw.ibrs_disable=0\`"
+ else
+ pvulnstatus $cve VULN "IBRS is needed to mitigate the vulnerability but your kernel is missing support"
+ explain "You need to either upgrade your kernel or recompile yourself a more recent version having IBRS support"
+ fi
+}
+
+########################
+# MELTDOWN aka VARIANT 3
+
+# no security impact but give a hint to the user in verbose mode
+# about PCID/INVPCID cpuid features that must be present to avoid
+# too big a performance impact with PTI
+# refs:
+# https://marc.info/?t=151532047900001&r=1&w=2
+# https://groups.google.com/forum/m/#!topic/mechanical-sympathy/L9mHTbeQLNU
+pti_performance_check()
+{
+ _info_nol " * Reduced performance impact of PTI: "
+ if [ -e "$procfs/cpuinfo" ] && grep ^flags "$procfs/cpuinfo" | grep -qw pcid; then
+ cpu_pcid=1
+ else
+ read_cpuid 0x1 $ECX 17 1 1; ret=$?
+ [ $ret -eq 0 ] && cpu_pcid=1
+ fi
+
+ if [ -e "$procfs/cpuinfo" ] && grep ^flags "$procfs/cpuinfo" | grep -qw invpcid; then
+ cpu_invpcid=1
+ else
+ read_cpuid 0x7 $EBX 10 1 1; ret=$?
+ [ $ret -eq 0 ] && cpu_invpcid=1
+ fi
+
+ if [ "$cpu_invpcid" = 1 ]; then
+ pstatus green YES 'CPU supports INVPCID, performance impact of PTI will be greatly reduced'
+ elif [ "$cpu_pcid" = 1 ]; then
+ pstatus green YES 'CPU supports PCID, performance impact of PTI will be reduced'
+ else
+ pstatus blue NO 'PCID/INVPCID not supported, performance impact of PTI will be significant'
+ fi
+}
+
+check_variant3()
+{
+ _info "\033[1;34mCVE-2017-5754 [rogue data cache load] aka 'Meltdown' aka 'Variant 3'\033[0m"
+ if [ "$os" = Linux ]; then
+ check_variant3_linux
+ elif echo "$os" | grep -q BSD; then
+ check_variant3_bsd
+ else
+ _warn "Unsupported OS ($os)"
+ fi
+}
+
+check_variant3_linux()
+{
+ status=UNK
+ sys_interface_available=0
+ msg=''
+ if sys_interface_check "/sys/devices/system/cpu/vulnerabilities/meltdown"; then
+ # this kernel has the /sys interface, trust it over everything
+ sys_interface_available=1
+ fi
+ if [ "$opt_sysfs_only" != 1 ]; then
+ _info_nol "* Kernel supports Page Table Isolation (PTI): "
+ kpti_support=''
+ kpti_can_tell=0
+ if [ -n "$opt_config" ]; then
+ kpti_can_tell=1
+ kpti_support=$(grep -w -e CONFIG_PAGE_TABLE_ISOLATION=y -e CONFIG_KAISER=y -e CONFIG_UNMAP_KERNEL_AT_EL0=y "$opt_config")
+ if [ -n "$kpti_support" ]; then
+ _debug "kpti_support: found option '$kpti_support' in $opt_config"
+ fi
+ fi
+ if [ -z "$kpti_support" ] && [ -n "$opt_map" ]; then
+ # it's not an elif: some backports don't have the PTI config but still include the patch
+ # so we try to find an exported symbol that is part of the PTI patch in System.map
+ # parse_kpti: arm
+ kpti_can_tell=1
+ kpti_support=$(grep -w -e kpti_force_enabled -e parse_kpti "$opt_map")
+ if [ -n "$kpti_support" ]; then
+ _debug "kpti_support: found '$kpti_support' in $opt_map"
+ fi
+ fi
+ if [ -z "$kpti_support" ] && [ -n "$kernel" ]; then
+ # same as above but in case we don't have System.map and only kernel, look for the
+ # nopti option that is part of the patch (kernel command line option)
+ # 'kpti=': arm
+ kpti_can_tell=1
+ if ! which "${opt_arch_prefix}strings" >/dev/null 2>&1; then
+ pstatus yellow UNKNOWN "missing '${opt_arch_prefix}strings' tool, please install it, usually it's in the binutils package"
+ else
+ kpti_support=$("${opt_arch_prefix}strings" "$kernel" | grep -w -e nopti -e kpti=)
+ if [ -n "$kpti_support" ]; then
+ _debug "kpti_support: found '$kpti_support' in $kernel"
+ fi
+ fi
+ fi
+
+ if [ -n "$kpti_support" ]; then
+ if [ "$opt_verbose" -ge 2 ]; then
+ pstatus green YES "found '$kpti_support'"
+ else
+ pstatus green YES
+ fi
+ elif [ "$kpti_can_tell" = 1 ]; then
+ pstatus yellow NO
+ else
+ pstatus yellow UNKNOWN "couldn't read your kernel configuration nor System.map file"
+ fi
+
+ mount_debugfs
+ _info_nol " * PTI enabled and active: "
+ if [ "$opt_live" = 1 ]; then
+ dmesg_grep="Kernel/User page tables isolation: enabled"
+ dmesg_grep="$dmesg_grep|Kernel page table isolation enabled"
+ dmesg_grep="$dmesg_grep|x86/pti: Unmapping kernel while in userspace"
+ if grep ^flags "$procfs/cpuinfo" | grep -qw pti; then
+ # vanilla PTI patch sets the 'pti' flag in cpuinfo
+ _debug "kpti_enabled: found 'pti' flag in $procfs/cpuinfo"
+ kpti_enabled=1
+ elif grep ^flags "$procfs/cpuinfo" | grep -qw kaiser; then
+ # kernel line 4.9 sets the 'kaiser' flag in cpuinfo
+ _debug "kpti_enabled: found 'kaiser' flag in $procfs/cpuinfo"
+ kpti_enabled=1
+ elif [ -e /sys/kernel/debug/x86/pti_enabled ]; then
+ # Red Hat Backport creates a dedicated file, see https://access.redhat.com/articles/3311301
+ kpti_enabled=$(cat /sys/kernel/debug/x86/pti_enabled 2>/dev/null)
+ _debug "kpti_enabled: file /sys/kernel/debug/x86/pti_enabled exists and says: $kpti_enabled"
+ fi
+ if [ -z "$kpti_enabled" ]; then
+ dmesg_grep "$dmesg_grep"; ret=$?
+ if [ $ret -eq 0 ]; then
+ _debug "kpti_enabled: found hint in dmesg: $dmesg_grepped"
+ kpti_enabled=1
+ elif [ $ret -eq 2 ]; then
+ _debug "kpti_enabled: dmesg truncated"
+ kpti_enabled=-1
+ fi
+ fi
+ if [ -z "$kpti_enabled" ]; then
+ _debug "kpti_enabled: couldn't find any hint that PTI is enabled"
+ kpti_enabled=0
+ fi
+ if [ "$kpti_enabled" = 1 ]; then
+ pstatus green YES
+ elif [ "$kpti_enabled" = -1 ]; then
+ pstatus yellow UNKNOWN "dmesg truncated, please reboot and relaunch this script"
+ else
+ pstatus yellow NO
+ fi
+ else
+ pstatus blue N/A "not testable in offline mode"
+ fi
+
+ pti_performance_check
+
+ elif [ "$sys_interface_available" = 0 ]; then
+ # we have no sysfs but were asked to use it only!
+ msg="/sys vulnerability interface use forced, but it's not available!"
+ status=UNK
+ fi
+
+
+ # Test if the current host is a Xen PV Dom0 / DomU
+ if [ -d "/proc/xen" ]; then
+ # XXX do we have a better way that relying on dmesg?
+ dmesg_grep 'Booting paravirtualized kernel on Xen$'; ret=$?
+ if [ $ret -eq 2 ]; then
+ _warn "dmesg truncated, Xen detection will be unreliable. Please reboot and relaunch this script"
+ elif [ $ret -eq 0 ]; then
+ if [ -e /proc/xen/capabilities ] && grep -q "control_d" /proc/xen/capabilities; then
+ xen_pv_domo=1
+ else
+ xen_pv_domu=1
+ fi
+ # PVHVM guests also print 'Booting paravirtualized kernel', so we need this check.
+ dmesg_grep 'Xen HVM callback vector for event delivery is enabled$'; ret=$?
+ if [ $ret -eq 0 ]; then
+ xen_pv_domu=0
+ fi
+ fi
+ fi
+
+ if [ "$opt_live" = 1 ]; then
+ # checking whether we're running under Xen PV 64 bits. If yes, we are affected by variant3
+ # (unless we are a Dom0)
+ _info_nol "* Running as a Xen PV DomU: "
+ if [ "$xen_pv_domu" = 1 ]; then
+ pstatus yellow YES
+ else
+ pstatus blue NO
+ fi
+ fi
+
+ cve='CVE-2017-5754'
+ if ! is_cpu_vulnerable 3; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ elif [ -z "$msg" ]; then
+ # if msg is empty, sysfs check didn't fill it, rely on our own test
+ if [ "$opt_live" = 1 ]; then
+ if [ "$kpti_enabled" = 1 ]; then
+ pvulnstatus $cve OK "PTI mitigates the vulnerability"
+ elif [ "$xen_pv_domo" = 1 ]; then
+ pvulnstatus $cve OK "Xen Dom0s are safe and do not require PTI"
+ elif [ "$xen_pv_domu" = 1 ]; then
+ pvulnstatus $cve VULN "Xen PV DomUs are vulnerable and need to be run in HVM, PVHVM, PVH mode, or the Xen hypervisor must have the Xen's own PTI patch"
+ explain "Go to https://blog.xenproject.org/2018/01/22/xen-project-spectre-meltdown-faq-jan-22-update/ for more information"
+ elif [ "$kpti_enabled" = -1 ]; then
+ pvulnstatus $cve UNK "couldn't find any clue of PTI activation due to a truncated dmesg, please reboot and relaunch this script"
+ else
+ pvulnstatus $cve VULN "PTI is needed to mitigate the vulnerability"
+ if [ -n "$kpti_support" ]; then
+ if [ -e "/sys/kernel/debug/x86/pti_enabled" ]; then
+ explain "Your kernel supports PTI but it's disabled, you can enable it with \`echo 1 > /sys/kernel/debug/x86/pti_enabled\`"
+ elif grep -q -w nopti -w pti=off /proc/cmdline; then
+ explain "Your kernel supports PTI but it has been disabled on command-line, remove the nopti or pti=off option from your bootloader configuration"
+ else
+ explain "Your kernel supports PTI but it has been disabled, check \`dmesg\` right after boot to find clues why the system disabled it"
+ fi
+ else
+ explain "If you're using a distro kernel, upgrade your distro to get the latest kernel available. Otherwise, recompile the kernel with the CONFIG_PAGE_TABLE_ISOLATION option (named CONFIG_KAISER for some kernels), or the CONFIG_UNMAP_KERNEL_AT_EL0 option (for ARM64)"
+ fi
+ fi
+ else
+ if [ -n "$kpti_support" ]; then
+ pvulnstatus $cve OK "offline mode: PTI will mitigate the vulnerability if enabled at runtime"
+ elif [ "$kpti_can_tell" = 1 ]; then
+ pvulnstatus $cve VULN "PTI is needed to mitigate the vulnerability"
+ explain "If you're using a distro kernel, upgrade your distro to get the latest kernel available. Otherwise, recompile the kernel with the CONFIG_PAGE_TABLE_ISOLATION option (named CONFIG_KAISER for some kernels), or the CONFIG_UNMAP_KERNEL_AT_EL0 option (for ARM64)"
+ else
+ pvulnstatus $cve UNK "offline mode: not enough information"
+ explain "Re-run this script with root privileges, and give it the kernel image (--kernel), the kernel configuration (--config) and the System.map file (--map) corresponding to the kernel you would like to inspect."
+ fi
+ fi
+ else
+ if [ "$xen_pv_domo" = 1 ]; then
+ msg="Xen Dom0s are safe and do not require PTI"
+ status="OK"
+ elif [ "$xen_pv_domu" = 1 ]; then
+ msg="Xen PV DomUs are vulnerable and need to be run in HVM, PVHVM, PVH mode, or the Xen hypervisor must have the Xen's own PTI patch"
+ status="VULN"
+ _explain="Go to https://blog.xenproject.org/2018/01/22/xen-project-spectre-meltdown-faq-jan-22-update/ for more information"
+ elif [ "$msg" = "Vulnerable" ]; then
+ msg="PTI is needed to mitigate the vulnerability"
+ _explain="If you're using a distro kernel, upgrade your distro to get the latest kernel available. Otherwise, recompile the kernel with the CONFIG_PAGE_TABLE_ISOLATION option (named CONFIG_KAISER for some kernels), or the CONFIG_UNMAP_KERNEL_AT_EL0 option (for ARM64)"
+ fi
+ pvulnstatus $cve "$status" "$msg"
+ [ -z "$_explain" ] && [ "$msg" = "Vulnerable" ] && _explain="If you're using a distro kernel, upgrade your distro to get the latest kernel available. Otherwise, recompile the kernel with the CONFIG_PAGE_TABLE_ISOLATION option (named CONFIG_KAISER for some kernels), or the CONFIG_UNMAP_KERNEL_AT_EL0 option (for ARM64)"
+ [ -n "$_explain" ] && explain "$_explain"
+ unset _explain
+ fi
+
+ # Warn the user about XSA-254 recommended mitigations
+ if [ "$xen_pv_domo" = 1 ]; then
+ _warn
+ _warn "This host is a Xen Dom0. Please make sure that you are running your DomUs"
+ _warn "in HVM, PVHVM or PVH mode to prevent any guest-to-host / host-to-guest attacks."
+ _warn
+ _warn "See https://blog.xenproject.org/2018/01/22/xen-project-spectre-meltdown-faq-jan-22-update/ and XSA-254 for details."
+ fi
+}
+
+check_variant3_bsd()
+{
+ _info_nol "* Kernel supports Page Table Isolation (PTI): "
+ kpti_enabled=$(sysctl -n vm.pmap.pti 2>/dev/null)
+ if [ -z "$kpti_enabled" ]; then
+ pstatus yellow NO
+ else
+ pstatus green YES
+ fi
+
+ _info_nol " * PTI enabled and active: "
+ if [ "$kpti_enabled" = 1 ]; then
+ pstatus green YES
+ else
+ pstatus yellow NO
+ fi
+
+ pti_performance_check
+
+ cve='CVE-2017-5754'
+ if ! is_cpu_vulnerable 3; then
+ # override status & msg in case CPU is not vulnerable after all
+ pvulnstatus $cve OK "your CPU vendor reported your CPU model as not vulnerable"
+ elif [ "$kpti_enabled" = 1 ]; then
+ pvulnstatus $cve OK "PTI mitigates the vulnerability"
+ elif [ -n "$kpti_enabled" ]; then
+ pvulnstatus $cve VULN "PTI is supported but disabled on your system"
+ else
+ pvulnstatus $cve VULN "PTI is needed to mitigate the vulnerability"
+ fi
+}
+
+if [ "$opt_no_hw" = 0 ] && [ -z "$opt_arch_prefix" ]; then
+ check_cpu
+ check_cpu_vulnerabilities
+ _info
+fi
+
+# now run the checks the user asked for
+if [ "$opt_variant1" = 1 ] || [ "$opt_allvariants" = 1 ]; then
+ check_variant1
+ _info
+fi
+if [ "$opt_variant2" = 1 ] || [ "$opt_allvariants" = 1 ]; then
+ check_variant2
+ _info
+fi
+if [ "$opt_variant3" = 1 ] || [ "$opt_allvariants" = 1 ]; then
+ check_variant3
+ _info
+fi
+
+_vars=$(set | grep -Ev '^[A-Z_[:space:]]' | sort | tr "\n" '|')
+_debug "variables at end of script: $_vars"
+
+_info "A false sense of security is worse than no security at all, see --disclaimer"
+
+if [ "$opt_batch" = 1 ] && [ "$opt_batch_format" = "nrpe" ]; then
+ if [ ! -z "$nrpe_vuln" ]; then
+ echo "Vulnerable:$nrpe_vuln"
+ else
+ echo "OK"
+ fi
+fi
+
+if [ "$opt_batch" = 1 ] && [ "$opt_batch_format" = "json" ]; then
+ _echo 0 "${json_output%?}]"
+fi
+
+if [ "$opt_batch" = 1 ] && [ "$opt_batch_format" = "prometheus" ]; then
+ echo "# TYPE specex_vuln_status untyped"
+ echo "# HELP specex_vuln_status Exposure of system to speculative execution vulnerabilities"
+ echo "$prometheus_output"
+fi
+
+# exit with the proper exit code
+[ "$global_critical" = 1 ] && exit 2 # critical
+[ "$global_unknown" = 1 ] && exit 3 # unknown
+exit 0 # ok
diff --git a/tests/_test_m18nkeys.py b/tests/_test_m18nkeys.py
new file mode 100644
index 000000000..ee8df0dc6
--- /dev/null
+++ b/tests/_test_m18nkeys.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+import re
+import glob
+import json
+import yaml
+
+###############################################################################
+# Find used keys in python code #
+###############################################################################
+
+# This regex matches « foo » in patterns like « m18n.n( "foo" »
+p = re.compile(r'm18n\.n\(\s*[\"\']([a-zA-Z1-9_]+)[\"\']')
+
+python_files = glob.glob("/vagrant/yunohost/src/yunohost/*.py")
+python_files.extend(glob.glob("/vagrant/yunohost/src/yunohost/utils/*.py"))
+python_files.append("/vagrant/yunohost/bin/yunohost")
+
+python_keys = set()
+for python_file in python_files:
+ with open(python_file) as f:
+ keys_in_file = p.findall(f.read())
+ for key in keys_in_file:
+ python_keys.add(key)
+
+###############################################################################
+# Find keys used in actionmap #
+###############################################################################
+
+actionmap_keys = set()
+actionmap = yaml.load(open("../data/actionsmap/yunohost.yml"))
+for _, category in actionmap.items():
+ if "actions" not in category.keys():
+ continue
+ for _, action in category["actions"].items():
+ if "arguments" not in action.keys():
+ continue
+ for _, argument in action["arguments"].items():
+ if "extra" not in argument.keys():
+ continue
+ if "password" in argument["extra"]:
+ actionmap_keys.add(argument["extra"]["password"])
+ if "ask" in argument["extra"]:
+ actionmap_keys.add(argument["extra"]["ask"])
+ if "pattern" in argument["extra"]:
+ actionmap_keys.add(argument["extra"]["pattern"][1])
+ if "help" in argument["extra"]:
+ print argument["extra"]["help"]
+
+# These keys are used but difficult to parse
+actionmap_keys.add("admin_password")
+
+###############################################################################
+# Load en locale json keys #
+###############################################################################
+
+en_locale_file = "/vagrant/yunohost/locales/en.json"
+with open(en_locale_file) as f:
+ en_locale_json = json.loads(f.read())
+
+en_locale_keys = set(en_locale_json.keys())
+
+###############################################################################
+# Compare keys used and keys defined #
+###############################################################################
+
+used_keys = python_keys.union(actionmap_keys)
+
+keys_used_but_not_defined = used_keys.difference(en_locale_keys)
+keys_defined_but_not_used = en_locale_keys.difference(used_keys)
+
+if len(keys_used_but_not_defined) != 0:
+ print "> Error ! Those keys are used in some files but not defined :"
+ for key in sorted(keys_used_but_not_defined):
+ print " - %s" % key
+
+if len(keys_defined_but_not_used) != 0:
+ print "> Warning ! Those keys are defined but seems unused :"
+ for key in sorted(keys_defined_but_not_used):
+ print " - %s" % key
+
+
diff --git a/tests/test_actionmap.py b/tests/test_actionmap.py
new file mode 100644
index 000000000..08b868839
--- /dev/null
+++ b/tests/test_actionmap.py
@@ -0,0 +1,4 @@
+import yaml
+
+def test_yaml_syntax():
+ yaml.load(open("data/actionsmap/yunohost.yml"))
+