Merge branch 'unstable' into testing

This commit is contained in:
Laurent Peuch 2017-01-09 18:35:54 +01:00
commit 388f2fa8be
22 changed files with 359 additions and 185 deletions

5
.travis.yml Normal file
View file

@ -0,0 +1,5 @@
language: python
install: "pip install pytest pyyaml"
python:
- "2.7"
script: "py.test tests"

88
CONTRIBUTORS.md Normal file
View file

@ -0,0 +1,88 @@
YunoHost core contributors
==========================
YunoHost is built and maintained by the YunoHost project community.
Everyone is encouraged to submit issues and changes, and to contribute in other ways -- see https://yunohost.org/contribute to find out how.
--
Initial YunoHost core was built by Kload & beudbeud, for YunoHost v2.
Most of code was written by Kload and jerome, with help of numerous contributors.
Translation is made by a bunch of lovely people all over the world.
We would like to thank anyone who ever helped the YunoHost project <3
YunoHost core Contributors
--------------------------
- Jérôme Lebleu
- Kload
- Laurent 'Bram' Peuch
- Julien 'ju' Malik
- opi
- Aleks
- Adrien 'beudbeud' Beudin
- M5oul
- Valentin 'zamentur' / 'ljf' Grimaud
- Jocelyn Delalande
- infertux
- Taziden
- ZeHiro
- Josue-T
- nahoj
- a1ex
- JimboJoe
- vetetix
- jellium
- Sebastien 'sebian' Badia
- lmangani
- Julien Vaubourg
YunoHost core Translators
-------------------------
If you want to help translation, please visit https://translate.yunohost.org/projects/yunohost/yunohost/
### Dutch
- DUBWiSE
- marut
### English
- Bugsbane
### French
- aoz roon
- Genma
- Jean-Baptiste Holcroft
- Jérôme Lebleu
### German
- david.bartke
- Felix Bartels
- Philip Gatzka
### Hindi
- Anmol
### Italian
- Thomas Bille
### Portuguese
- Deleted User
### Spanish
- Juanu

View file

@ -9,7 +9,9 @@ do_pre_regen() {
install -D -m 644 rmilter.conf \
"${pending_dir}/etc/rmilter.conf"
install -D -m 644 rmilter.socket \
# Remove old socket file (we stopped using it, since rspamd 1.3.1)
# Regen-conf system need an empty file to delete it
install -D -m 644 /dev/null \
"${pending_dir}/etc/systemd/system/rmilter.socket"
}
@ -37,17 +39,19 @@ do_post_regen() {
sudo chown _rmilter /etc/dkim/*.mail.key
sudo chmod 400 /etc/dkim/*.mail.key
# fix rmilter socket permission (postfix is chrooted in /var/spool/postfix )
sudo mkdir -p /var/spool/postfix/run/rmilter
sudo chown -R postfix:_rmilter /var/spool/postfix/run/rmilter
sudo chmod g+w /var/spool/postfix/run/rmilter
[ -z "$regen_conf_files" ] && exit 0
# reload systemd daemon
[[ "$regen_conf_files" =~ rmilter\.socket ]] && {
sudo systemctl -q daemon-reload
}
# ensure that the socket is listening and stop the service - it will be
# started again by the socket as needed
sudo systemctl -q start rmilter.socket
sudo systemctl -q stop rmilter.service 2>&1 || true
# Restart rmilter due to the rspamd update
# https://rspamd.com/announce/2016/08/01/rspamd-1.3.1.html
sudo systemctl -q restart rmilter.service
}
FORCE=${2:-0}

View file

@ -25,10 +25,9 @@ do_post_regen() {
sudo systemctl restart dovecot
}
# ensure that the socket is listening and stop the service - it will be
# started again by the socket as needed
sudo systemctl -q start rspamd.socket
sudo systemctl -q stop rspamd.service 2>&1 || true
# Restart rspamd due to the upgrade
# https://rspamd.com/announce/2016/08/01/rspamd-1.3.1.html
sudo systemctl -q restart rspamd.service
}
FORCE=${2:-0}

View file

@ -14,7 +14,7 @@
# (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
# Values: TEXT
#
failregex = access.lua:[1-9]+: authenticate\(\): Connection failed for: .*, client: <HOST>
failregex = helpers.lua:[1-9]+: authenticate\(\): Connection failed for: .*, client: <HOST>
^<HOST> -.*\"POST /yunohost/api/login HTTP/1.1\" 401 22
# Option: ignoreregex

View file

@ -141,7 +141,7 @@ smtp_reply_filter = pcre:/etc/postfix/smtp_reply_filter
# Rmilter
milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
milter_protocol = 6
smtpd_milters = inet:localhost:11000
smtpd_milters = unix:/run/rmilter/rmilter.sock
# Skip email without checking if milter has died
milter_default_action = accept

View file

@ -5,8 +5,7 @@
# pidfile - path to pid file
pidfile = /run/rmilter/rmilter.pid;
# rmilter is socket-activated under systemd
bind_socket = fd:3;
bind_socket = unix:/var/spool/postfix/run/rmilter/rmilter.sock;
# DKIM signing
dkim {

View file

@ -1,5 +0,0 @@
.include /lib/systemd/system/rmilter.socket
[Socket]
ListenStream=
ListenStream=127.0.0.1:11000

View file

@ -253,6 +253,7 @@
"certmanager_domain_http_not_working": "It seems that the domain {domain:s} cannot be accessed through HTTP. Please check your DNS and nginx configuration is okay",
"certmanager_error_no_A_record": "No DNS 'A' record found for {domain:s}. You need to make your domain name point to your machine to be able to install a Let's Encrypt certificate! (If you know what you are doing, use --no-checks to disable those checks.)",
"certmanager_domain_dns_ip_differs_from_public_ip": "The DNS 'A' record for domain {domain:s} is different from this server IP. If you recently modified your A record, please wait for it to propagate (some DNS propagation checkers are available online). (If you know what you are doing, use --no-checks to disable those checks.)",
"certmanager_domain_not_resolved_locally": "The domain {domain:s} cannot be resolved from inside your Yunohost server. This might happen if you recently modified your DNS record. If so, please wait a few hours for it to propagate. If the issue persists, consider adding {domain:s} to /etc/hosts. (If you know what you are doing, use --no-checks to disable those checks.)",
"certmanager_cannot_read_cert": "Something wrong happened when trying to open current certificate for domain {domain:s} (file: {file:s}), reason: {reason:s}",
"certmanager_cert_install_success_selfsigned": "Successfully installed a self-signed certificate for domain {domain:s}!",
"certmanager_cert_install_success": "Successfully installed Let's Encrypt certificate for domain {domain:s}!",
@ -264,5 +265,6 @@
"certmanager_conflicting_nginx_file": "Unable to prepare domain for ACME challenge: the nginx configuration file {filepath:s} is conflicting and should be removed first",
"domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first",
"certmanager_self_ca_conf_file_not_found": "Configuration file not found for self-signing authority (file: {file:s})",
"certmanager_acme_not_configured_for_domain": "Certificate for domain {domain:s} does not appear to be correctly installed. Please run cert-install for this domain first.",
"certmanager_unable_to_parse_self_CA_name": "Unable to parse name of self-signing authority (file: {file:s})"
}

View file

@ -99,7 +99,7 @@ def app_fetchlist(url=None, name=None):
m18n.n('custom_appslist_name_required'))
list_file = '%s/%s.json' % (repo_path, name)
if os.system('wget "%s" -O "%s.tmp"' % (url, list_file)) != 0:
if os.system('wget --timeout=30 "%s" -O "%s.tmp"' % (url, list_file)) != 0:
os.remove('%s.tmp' % list_file)
raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error'))

View file

@ -313,8 +313,6 @@ def backup_create(name=None, description=None, output_directory=None,
link = "%s/%s.tar.gz" % (archives_path, name)
os.symlink(archive_file, link)
# Clean temporary directory
if tmp_dir != output_directory:
_clean_tmp_dir()

View file

@ -31,6 +31,7 @@ import grp
import smtplib
import requests
import subprocess
import socket
import dns.resolver
import glob
@ -323,7 +324,15 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
continue
# Does it expire soon?
if force or status["validity"] <= VALIDITY_LIMIT:
if status["validity"] > VALIDITY_LIMIT and not force:
continue
# Check ACME challenge configured for given domain
if not _check_acme_challenge_configuration(domain):
logger.warning(m18n.n(
'certmanager_acme_not_configured_for_domain', domain=domain))
continue
domain_list.append(domain)
if len(domain_list) == 0:
@ -341,7 +350,7 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
status = _get_status(domain)
# Does it expire soon?
if not force or status["validity"] <= VALIDITY_LIMIT:
if status["validity"] > VALIDITY_LIMIT and not force:
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_attempt_to_renew_valid_cert', domain=domain))
@ -350,6 +359,11 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_attempt_to_renew_nonLE_cert', domain=domain))
# Check ACME challenge configured for given domain
if not _check_acme_challenge_configuration(domain):
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_acme_not_configured_for_domain', domain=domain))
if staging:
logger.warning(
"Please note that you used the --staging option, and that no new certificate will actually be enabled !")
@ -362,6 +376,7 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
try:
if not no_checks:
_check_domain_is_ready_for_ACME(domain)
_fetch_and_enable_new_certificate(domain, staging)
logger.success(
@ -487,6 +502,17 @@ location '/.well-known/acme-challenge'
app_ssowatconf(auth)
def _check_acme_challenge_configuration(domain):
# Check nginx conf file exists
nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain
nginx_conf_file = "%s/000-acmechallenge.conf" % nginx_conf_folder
if not os.path.exists(nginx_conf_file):
return False
else:
return True
def _fetch_and_enable_new_certificate(domain, staging=False):
# Make sure tmp folder exists
logger.debug("Making sure tmp folders exists...")
@ -562,7 +588,9 @@ def _fetch_and_enable_new_certificate(domain, staging=False):
_set_permissions(new_cert_folder, "root", "root", 0655)
# Move the private key
shutil.move(domain_key_file, os.path.join(new_cert_folder, "key.pem"))
domain_key_file_finaldest = os.path.join(new_cert_folder, "key.pem")
shutil.move(domain_key_file, domain_key_file_finaldest)
_set_permissions(domain_key_file_finaldest, "root", "metronome", 0640)
# Write the cert
domain_cert_file = os.path.join(new_cert_folder, "crt.pem")
@ -785,6 +813,13 @@ def _check_domain_is_ready_for_ACME(domain):
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_domain_http_not_working', domain=domain))
# Check if domain is resolved locally (Might happen despite the previous
# checks because of dns propagation ?... Acme-tiny won't work in that case,
# because it explicitly requests() the domain.)
if not _domain_is_resolved_locally(public_ip, domain):
raise MoulinetteError(errno.EINVAL, m18n.n(
'certmanager_domain_not_resolved_locally', domain=domain))
def _dns_ip_match_public_ip(public_ip, domain):
try:
@ -803,12 +838,24 @@ def _dns_ip_match_public_ip(public_ip, domain):
def _domain_is_accessible_through_HTTP(ip, domain):
try:
requests.head("http://" + ip, headers={"Host": domain})
except Exception:
except Exception as e:
logger.debug("Couldn't reach domain '%s' by requesting this ip '%s' because: %s" % (domain, ip, e))
return False
return True
def _domain_is_resolved_locally(public_ip, domain):
try:
ip = socket.gethostbyname(domain)
except socket.error as e:
logger.debug("Couldn't get domain '%s' ip because: %s" % (domain, e))
return False
logger.debug("Domain '%s' ip is %s, except it to be 127.0.0.1 or %s" % (domain, public_ip))
return ip in ["127.0.0.1", public_ip]
def _name_self_CA():
ca_conf = os.path.join(SSL_DIR, "openssl.ca.cnf")

View file

@ -113,7 +113,6 @@ def domain_add(auth, domain, dyndns=False):
raise MoulinetteError(errno.EINVAL,
m18n.n('domain_dyndns_root_unknown'))
try:
yunohost.certificate._certificate_install_selfsigned([domain], False)
@ -122,7 +121,6 @@ def domain_add(auth, domain, dyndns=False):
except MoulinetteError:
raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))
attr_dict['virtualdomain'] = domain
if not auth.add('virtualdomain=%s,ou=domains' % domain, attr_dict):
@ -133,11 +131,14 @@ def domain_add(auth, domain, dyndns=False):
service_regen_conf(names=[
'nginx', 'metronome', 'dnsmasq', 'rmilter'])
os.system('yunohost app ssowatconf > /dev/null 2>&1')
except IOError: pass
except IOError:
pass
except:
# Force domain removal silently
try: domain_remove(auth, domain, True)
except: pass
try:
domain_remove(auth, domain, True)
except:
pass
raise
hook_callback('post_domain_add', args=[domain])
@ -296,6 +297,7 @@ def _get_maindomain():
maindomain = f.readline().rstrip()
return maindomain
def _set_maindomain(domain):
with open('/etc/yunohost/current_host', 'w') as f:
f.write(domain)

View file

@ -94,7 +94,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
logger.info(m18n.n('dyndns_key_generating'))
os.system('cd /etc/yunohost/dyndns && ' \
os.system('cd /etc/yunohost/dyndns && '
'dnssec-keygen -a hmac-md5 -b 128 -r /dev/urandom -n USER %s' % domain)
os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private')
@ -108,8 +108,10 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None
except requests.ConnectionError:
raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection'))
if r.status_code != 201:
try: error = json.loads(r.text)['error']
except: error = "Server error"
try:
error = json.loads(r.text)['error']
except:
error = "Server error"
raise MoulinetteError(errno.EPERM,
m18n.n('dyndns_registration_failed', error=error))

View file

@ -224,7 +224,7 @@ def firewall_reload(skip_upnp=False):
# Iterate over ports and add rule
for protocol in ['TCP', 'UDP']:
for port in firewall['ipv4'][protocol]:
rules.append("iptables -w -A INPUT -p %s --dport %s -j ACCEPT" \
rules.append("iptables -w -A INPUT -p %s --dport %s -j ACCEPT"
% (protocol, process.quote(str(port))))
rules += [
"iptables -w -A INPUT -i lo -j ACCEPT",
@ -253,7 +253,7 @@ def firewall_reload(skip_upnp=False):
# Iterate over ports and add rule
for protocol in ['TCP', 'UDP']:
for port in firewall['ipv6'][protocol]:
rules.append("ip6tables -w -A INPUT -p %s --dport %s -j ACCEPT" \
rules.append("ip6tables -w -A INPUT -p %s --dport %s -j ACCEPT"
% (protocol, process.quote(str(port))))
rules += [
"ip6tables -w -A INPUT -i lo -j ACCEPT",
@ -308,7 +308,8 @@ def firewall_upnp(action='status', no_refresh=False):
try:
# Remove old cron job
os.remove('/etc/cron.d/yunohost-firewall')
except: pass
except:
pass
action = 'status'
no_refresh = False
@ -330,7 +331,8 @@ def firewall_upnp(action='status', no_refresh=False):
try:
# Remove cron job
os.remove(upnp_cron_job)
except: pass
except:
pass
enabled = False
if action == 'status':
no_refresh = True
@ -364,7 +366,8 @@ def firewall_upnp(action='status', no_refresh=False):
if upnpc.getspecificportmapping(port, protocol):
try:
upnpc.deleteportmapping(port, protocol)
except: pass
except:
pass
if not enabled:
continue
try:
@ -444,12 +447,14 @@ def _get_ssh_port(default=22):
pass
return default
def _update_firewall_file(rules):
"""Make a backup and write new rules to firewall file"""
os.system("cp {0} {0}.old".format(firewall_file))
with open(firewall_file, 'w') as f:
yaml.safe_dump(rules, f, default_flow_style=False)
def _on_rule_command_error(returncode, cmd, output):
"""Callback for rules commands error"""
# Log error and continue commands execution

View file

@ -87,7 +87,7 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False):
# Retrieve monitoring for unit(s)
for u in units:
if u == 'io':
## Define setter
# Define setter
if len(units) > 1:
def _set(dn, dvalue):
try:
@ -111,7 +111,7 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False):
for dname in devices_names:
_set(dname, 'not-available')
elif u == 'filesystem':
## Define setter
# Define setter
if len(units) > 1:
def _set(dn, dvalue):
try:

View file

@ -315,6 +315,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Execute hooks for pre-regen
pre_args = ['pre', ] + common_args
def _pre_call(name, priority, path, args):
# create the pending conf directory for the service
service_pending_path = os.path.join(pending_conf_dir, name)
@ -336,7 +337,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Iterate over services and process pending conf
for service, conf_files in _get_pending_conf(names).items():
logger.info(m18n.n(
'service_regenconf_pending_applying' if not dry_run else \
'service_regenconf_pending_applying' if not dry_run else
'service_regenconf_dry_pending_applying',
service=service))
@ -444,7 +445,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
continue
elif not failed_regen:
logger.success(m18n.n(
'service_conf_updated' if not dry_run else \
'service_conf_updated' if not dry_run else
'service_conf_would_be_updated',
service=service))
if succeed_regen and not dry_run:
@ -462,6 +463,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
# Execute hooks for post-regen
post_args = ['post', ] + common_args
def _pre_call(name, priority, path, args):
# append coma-separated applied changes for the service
if name in result and result[name]['applied']:
@ -556,7 +558,8 @@ def _tail(file, n, offset=None):
return lines[-to_read:offset and -offset or None]
avg_line_length *= 1.3
except IOError: return []
except IOError:
return []
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):

View file

@ -178,7 +178,8 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
Keyword argument:
domain -- YunoHost main domain
ignore_dyndns -- Do not subscribe domain to a DynDNS service
ignore_dyndns -- Do not subscribe domain to a DynDNS service (only
needed for nohost.me, noho.st domains)
password -- YunoHost admin password
"""
@ -203,6 +204,10 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
else:
raise MoulinetteError(errno.EEXIST,
m18n.n('dyndns_unavailable'))
else:
dyndns = False
else:
dyndns = False
logger.info(m18n.n('yunohost_installing'))
@ -296,7 +301,6 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
os.system('service yunohost-firewall start')
service_regen_conf(force=True)
logger.success(m18n.n('yunohost_configured'))
@ -416,7 +420,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
apt.progress.base.InstallProgress())
except Exception as e:
failure = True
logging.warning('unable to upgrade packages: %s' % str(e))
logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
@ -428,7 +432,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
app_upgrade(auth)
except Exception as e:
failure = True
logging.warning('unable to upgrade apps: %s' % str(e))
logger.warning('unable to upgrade apps: %s' % str(e))
logger.error(m18n.n('app_upgrade_failed'))
if not failure:

View file

@ -192,7 +192,6 @@ def user_create(auth, username, firstname, lastname, mail, password,
raise MoulinetteError(errno.EPERM,
m18n.n('ssowat_persistent_conf_write_error', error=e.strerror))
if auth.add(rdn, attr_dict):
# Invalidate passwd to take user creation into account
subprocess.call(['nscd', '-i', 'passwd'])
@ -238,8 +237,10 @@ def user_delete(auth, username, purge=False):
# Update SFTP user group
memberlist = auth.search(filter='cn=sftpusers', attrs=['memberUid'])[0]['memberUid']
try: memberlist.remove(username)
except: pass
try:
memberlist.remove(username)
except:
pass
if auth.update('cn=sftpusers,ou=groups', {'memberUid': memberlist}):
if purge:
subprocess.call(['rm', '-rf', '/home/{0}'.format(username)])
@ -445,6 +446,7 @@ def user_info(auth, username):
else:
raise MoulinetteError(167, m18n.n('user_info_failed'))
def _convertSize(num, suffix=''):
for unit in ['K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1024.0:

View file

@ -424,6 +424,7 @@ def get_installed_version(*pkgnames, **kwargs):
return versions[pkgnames[0]]
return versions
def meets_version_specifier(pkgname, specifier):
"""Check if a package installed version meets specifier"""
spec = SpecifierSet(specifier)

View file

@ -1,5 +1,17 @@
#!/usr/bin/env python
import argparse, subprocess, json, os, sys, base64, binascii, time, hashlib, re, copy, textwrap, logging
import argparse
import subprocess
import json
import os
import sys
import base64
import binascii
import time
import hashlib
import re
import copy
import textwrap
import logging
try:
from urllib.request import urlopen # Python 3
except ImportError:
@ -12,6 +24,7 @@ LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(logging.StreamHandler())
LOGGER.setLevel(logging.INFO)
def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA):
# helper function base64 encode for jose spec
def _b64(b):
@ -165,6 +178,7 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA):
return """-----BEGIN CERTIFICATE-----\n{0}\n-----END CERTIFICATE-----\n""".format(
"\n".join(textwrap.wrap(base64.b64encode(result).decode('utf8'), 64)))
def main(argv):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,

4
tests/test_actionmap.py Normal file
View file

@ -0,0 +1,4 @@
import yaml
def test_yaml_syntax():
yaml.load(open("data/actionsmap/yunohost.yml"))