From 332efb6f21446f6ae02ce751e66b8dfbea057c39 Mon Sep 17 00:00:00 2001 From: Alexis Gavoty Date: Tue, 16 Sep 2014 09:13:34 +0200 Subject: [PATCH 01/58] [enh] Assign yunohost.local servername in nginx conf at main domain changing --- tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools.py b/tools.py index f057c371e..19b652d13 100644 --- a/tools.py +++ b/tools.py @@ -174,6 +174,8 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): os.system('rm /etc/ssl/certs/yunohost_crt.pem') command_list = [ + 'rm -f /etc/nginx/conf.d/%s.d/yunohost_local.conf' % old_domain, + 'cp /usr/share/yunohost/yunohost-config/nginx/yunohost_local.conf /etc/nginx/conf.d/%s.d/' % new_domain, 'ln -s /etc/yunohost/certs/%s/key.pem /etc/ssl/private/yunohost_key.pem' % new_domain, 'ln -s /etc/yunohost/certs/%s/crt.pem /etc/ssl/certs/yunohost_crt.pem' % new_domain, 'echo %s > /etc/yunohost/current_host' % new_domain, From 4ac554fbb9b59f877423f318acec0f425a5ae738 Mon Sep 17 00:00:00 2001 From: Alexis Gavoty Date: Tue, 16 Sep 2014 13:54:06 +0200 Subject: [PATCH 02/58] [enh] Enable yunohost-firewall at startup --- tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools.py b/tools.py index 19b652d13..aa175abb0 100644 --- a/tools.py +++ b/tools.py @@ -325,6 +325,9 @@ def tools_postinstall(domain, password, dyndns=False): except MoulinetteError: firewall_upnp(action=['disable']) + # Enable iptables at boot time + os.system('update-rc.d yunohost-firewall defaults') + os.system('touch /etc/yunohost/installed') msignals.display(m18n.n('yunohost_configured'), 'success') From a65f90db958db1d4925ec5199c85bffd990035e0 Mon Sep 17 00:00:00 2001 From: Alexis Gavoty Date: Tue, 16 Sep 2014 14:00:11 +0200 Subject: [PATCH 03/58] [fix] Do not reload iptables every 50 minutes --- firewall.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/firewall.py b/firewall.py index c7b0aae15..9abea8ec2 100644 --- a/firewall.py +++ b/firewall.py @@ -235,8 +235,7 @@ def firewall_upnp(action=None): with open('/etc/cron.d/yunohost-firewall', 'w+') as f: f.write('PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - \n*/50 * * * * root yunohost firewall upnp reload >>/dev/null \ - \n*/50 * * * * root iptables -L | grep ^fail2ban-dovecot > /dev/null 2>&1; if [ $? != 0 ]; then yunohost firewall reload; fi >>/dev/null') + \n*/50 * * * * root yunohost firewall upnp reload >>/dev/null') msignals.display(m18n.n('upnp_enabled'), 'success') From c018bb07105ba109e2608112ec13120f278fa80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Tue, 7 Oct 2014 21:39:01 +0200 Subject: [PATCH 04/58] [fix] Change /home/yunohost.app permissions at postinstall (fix #40) --- tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools.py b/tools.py index aa175abb0..5667c4d09 100644 --- a/tools.py +++ b/tools.py @@ -257,6 +257,9 @@ def tools_postinstall(domain, password, dyndns=False): try: os.listdir(folder) except OSError: os.makedirs(folder) + # Change folders permissions + os.system('chmod 755 /home/yunohost.app') + # Set hostname to avoid amavis bug if os.system('hostname -d') != 0: os.system('hostname yunohost.yunohost.org') From ce5be021d3ad9b0dfa07500dcbecfbdcceb3e96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Mon, 13 Oct 2014 17:10:23 +0200 Subject: [PATCH 05/58] [fix] Fix app upgrade issue and handle errors in tools_upgrade --- app.py | 2 +- locales/en.json | 1 + locales/fr.json | 1 + tools.py | 17 +++++++++++++---- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 090d7d05b..62e3e8d5b 100644 --- a/app.py +++ b/app.py @@ -261,7 +261,7 @@ def app_map(app=None, raw=False, user=None): return result -def app_upgrade(auth, app, url=None, file=None): +def app_upgrade(auth, app=[], url=None, file=None): """ Upgrade app diff --git a/locales/en.json b/locales/en.json index 56da1da88..65cf911fb 100644 --- a/locales/en.json +++ b/locales/en.json @@ -20,6 +20,7 @@ "custom_app_url_required" : "You must provide an URL to upgrade your custom app {:s}", "app_recent_version_required" : "{:s} requires a more recent version of the moulinette", "app_upgraded" : "{:s} successfully upgraded", + "app_upgrade_failed" : "Unable to upgrade all apps", "app_id_invalid" : "Invalid app id", "app_already_installed" : "{:s} is already installed", "app_removed" : "{:s} successfully removed", diff --git a/locales/fr.json b/locales/fr.json index 874485d3f..375ece04d 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -20,6 +20,7 @@ "custom_app_url_required" : "Vous devez spécifier une URL pour mettre à jour votre application locale {:s}", "app_recent_version_required" : "{:s} nécessite une version plus récente de la moulinette", "app_upgraded" : "{:s} mis à jour avec succès", + "app_upgrade_failed" : "Impossible de mettre à jour toutes les applications", "app_id_invalid" : "Id d'application incorrect", "app_already_installed" : "{:s} est déjà installé", "app_removed" : "{:s} supprimé avec succès", diff --git a/tools.py b/tools.py index 5667c4d09..b58cb5149 100644 --- a/tools.py +++ b/tools.py @@ -31,6 +31,7 @@ import getpass import requests import json import errno +import logging import apt import apt.progress @@ -330,7 +331,7 @@ def tools_postinstall(domain, password, dyndns=False): # Enable iptables at boot time os.system('update-rc.d yunohost-firewall defaults') - + os.system('touch /etc/yunohost/installed') msignals.display(m18n.n('yunohost_configured'), 'success') @@ -416,6 +417,9 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False): """ from yunohost.app import app_upgrade + failure = False + + # Retrieve interface is_api = True if msettings.get('interface') == 'api' else False if not ignore_packages: @@ -449,6 +453,7 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False): cache.commit(apt.progress.text.AcquireProgress(), apt.progress.base.InstallProgress()) except Exception as e: + failure = True logging.warning('unable to upgrade packages: %s' % str(e)) msignals.display(m18n.n('packages_upgrade_failed'), 'error') else: @@ -459,11 +464,15 @@ def tools_upgrade(auth, ignore_apps=False, ignore_packages=False): if not ignore_apps: try: app_upgrade(auth) - except: pass + except Exception as e: + failure = True + logging.warning('unable to upgrade apps: %s' % str(e)) + msignals.display(m18n.n('app_upgrade_failed'), 'error') - msignals.display(m18n.n('system_upgraded'), 'success') + if not failure: + msignals.display(m18n.n('system_upgraded'), 'success') # Return API logs if it is an API call - if msettings.get('interface') == 'api': + if is_api: from yunohost.service import service_log return { "log": service_log('yunohost-api', number="100").values()[0] } From e32ea4180aef6c89972fa95715d5a8a724f9b22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Mon, 20 Oct 2014 12:21:26 +0200 Subject: [PATCH 06/58] [fix] Raise if the requested app to install is unknown --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 62e3e8d5b..0f1e6bc78 100644 --- a/app.py +++ b/app.py @@ -392,8 +392,10 @@ def app_install(auth, app, label=None, args=None): if app in app_list(raw=True) or ('@' in app) or ('http://' in app) or ('https://' in app): manifest = _fetch_app_from_git(app) - else: + elif os.path.exists(app): manifest = _extract_app_from_file(app) + else: + raise MoulinetteError(errno.EINVAL, m18n.n('app_unknown')) # Check ID if 'id' not in manifest or '__' in manifest['id']: From b0735af6cec3c8562c839f2060a61ebb30bb43da Mon Sep 17 00:00:00 2001 From: opi Date: Thu, 23 Oct 2014 12:07:49 +0200 Subject: [PATCH 07/58] [fix] Do not load conf.json before overriding content. --- app.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 0f1e6bc78..f57e1fdfb 100644 --- a/app.py +++ b/app.py @@ -924,10 +924,7 @@ def app_ssowatconf(auth): for domain in domains: skipped_urls.extend(['/yunohost/admin', '/yunohost/api']) - with open('/etc/ssowat/conf.json') as f: - conf_dict = json.load(f) - - conf_dict.update({ + conf_dict = { 'portal_domain': main_domain, 'portal_path': '/yunohost/sso/', 'additional_headers': { @@ -945,7 +942,7 @@ def app_ssowatconf(auth): 'protected_regex': protected_regex, 'redirected_regex': redirected_regex, 'users': users, - }) + } with open('/etc/ssowat/conf.json', 'w+') as f: json.dump(conf_dict, f, sort_keys=True, indent=4) From 0608461482d81eb851fa0c4c5ec30ed16f7cbf91 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 17:47:49 +0200 Subject: [PATCH 08/58] [enh] Execute backup hook, and avoid dyndns domain subscription when specifically asked at postinstall --- actionsmap/yunohost.yml | 12 ++++-------- backup.py | 34 +++++++++++++++++----------------- locales/en.json | 2 ++ locales/fr.json | 2 ++ tools.py | 7 ++----- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index a3012d582..21bc23094 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -559,12 +559,8 @@ backup: ### backup_init() init: - action_help: Init Tahoe-LAFS configuration - # api: POST /backup/init - arguments: - --helper: - help: Init as a helper node rather than a "helped" one - action: store_true + action_help: Create an encrypted backup tarball + api: POST /backup ############################# @@ -1025,8 +1021,8 @@ tools: pattern: - '^.{3,}$' - pattern_password - --dyndns: - help: Subscribe domain to a DynDNS service + --ignore-dyndns: + help: Do not subscribe domain to a DynDNS service action: store_true ### tools_update() diff --git a/backup.py b/backup.py index 12b629d4e..e51e22885 100644 --- a/backup.py +++ b/backup.py @@ -26,27 +26,27 @@ import os import sys import json -import yaml -import glob +import time from moulinette.core import MoulinetteError -def backup_init(helper=False): +def backup(): """ - Init Tahoe-LAFS configuration - - Keyword argument: - helper -- Init as a helper node rather than a "helped" one + Create an encrypted backup tarball """ - tahoe_cfg_dir = '/usr/share/yunohost/yunohost-config/backup' - if helper: - configure_cmd = '/configure_tahoe.sh helper' - else: - configure_cmd = '/configure_tahoe.sh' + from yunohost.hook import hook_callback - os.system('tahoe create-client /home/yunohost.backup/tahoe') - os.system('/bin/bash %s%s' % (tahoe_cfg_dir, configure_cmd)) - os.system('cp %s/tahoe.cfg /home/yunohost.backup/tahoe/' % tahoe_cfg_dir) - #os.system('update-rc.d tahoe-lafs defaults') - #os.system('service tahoe-lafs restart') + backup_dirname = int(time.time()) + backup_dir = "/home/yunohost.backup/tmp/%s" % backup_dirname + + # Create directory + try: os.listdir(backup_dir) + except OSError: os.makedirs(backup_dir) + + # Run hook + hook_callback('backup', [backup_dir]) + + #TODO: Compress & encrypt + + msignals.display(m18n.n('backup_completed'), 'success') diff --git a/locales/en.json b/locales/en.json index 65cf911fb..f615178fb 100644 --- a/locales/en.json +++ b/locales/en.json @@ -128,6 +128,8 @@ "packages_upgrade_failed" : "Unable to upgrade all packages", "system_upgraded" : "System successfully upgraded", + "backup_complete" : "Backup complete", + "field_invalid" : "Invalid field '{:s}'", "mail_domain_unknown" : "Unknown mail address domain '{:s}'", "mail_alias_remove_failed" : "Unable to remove mail alias '{:s}'", diff --git a/locales/fr.json b/locales/fr.json index 375ece04d..60880cb57 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -128,6 +128,8 @@ "packages_upgrade_failed" : "Impossible de mettre à jour tous les paquets", "system_upgraded" : "Système mis à jour avec succès", + "backup_complete" : "Sauvegarde terminée", + "field_invalid" : "Champ incorrect : {:s}", "mail_domain_unknown" : "Domaine '{:s}' de l'adresse mail inconnu", "mail_alias_remove_failed" : "Impossible de supprimer l'adresse mail supplémentaire '{:s}'", diff --git a/tools.py b/tools.py index b58cb5149..82d7325c8 100644 --- a/tools.py +++ b/tools.py @@ -207,7 +207,7 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): msignals.display(m18n.n('maindomain_changed'), 'success') -def tools_postinstall(domain, password, dyndns=False): +def tools_postinstall(domain, password, ignore_dyndns=False): """ YunoHost post-install @@ -230,7 +230,7 @@ def tools_postinstall(domain, password, dyndns=False): else: raise MoulinetteError(errno.EPERM, m18n.n('yunohost_already_installed')) - if len(domain.split('.')) >= 3: + if len(domain.split('.')) >= 3 and not ignore_dyndns: try: r = requests.get('http://dyndns.yunohost.org/domains') except ConnectionError: @@ -310,9 +310,6 @@ def tools_postinstall(domain, password, dyndns=False): # Initialize YunoHost LDAP base tools_ldapinit(auth) - # Initialize backup system - backup_init() - # New domain config tools_maindomain(auth, old_domain='yunohost.org', new_domain=domain, dyndns=dyndns) From 796e53bf1ffdbe8e3a6327995b7346a391117833 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 18:43:31 +0200 Subject: [PATCH 09/58] [enh] Add system backup and restore hooks --- backup.py | 2 ++ hooks/backup/11-system_mysql | 7 ++++++ hooks/backup/14-system_ssowat | 6 +++++ hooks/backup/17-system_home | 6 +++++ hooks/backup/20-system_yunohost | 6 +++++ hooks/backup/23-system_mail | 6 +++++ hooks/backup/26-system_xmpp | 7 ++++++ hooks/backup/29-system_nginx | 6 +++++ hooks/backup/32-system_cron | 6 +++++ hooks/backup/5-system_ldap | 9 ++++++++ hooks/backup/8-system_ssh | 6 +++++ hooks/restore/11-system_mysql | 7 ++++++ hooks/restore/14-system_ssowat | 5 +++++ hooks/restore/17-system_home | 5 +++++ hooks/restore/20-system_yunohost | 11 +++++++++ hooks/restore/23-system_mail | 9 ++++++++ hooks/restore/26-system_xmpp | 9 ++++++++ hooks/restore/29-system_nginx | 8 +++++++ hooks/restore/32-system_cron | 8 +++++++ hooks/restore/5-system_ldap | 38 ++++++++++++++++++++++++++++++++ hooks/restore/8-system_ssh | 6 +++++ 21 files changed, 173 insertions(+) create mode 100644 hooks/backup/11-system_mysql create mode 100644 hooks/backup/14-system_ssowat create mode 100644 hooks/backup/17-system_home create mode 100644 hooks/backup/20-system_yunohost create mode 100644 hooks/backup/23-system_mail create mode 100644 hooks/backup/26-system_xmpp create mode 100644 hooks/backup/29-system_nginx create mode 100644 hooks/backup/32-system_cron create mode 100644 hooks/backup/5-system_ldap create mode 100644 hooks/backup/8-system_ssh create mode 100644 hooks/restore/11-system_mysql create mode 100644 hooks/restore/14-system_ssowat create mode 100644 hooks/restore/17-system_home create mode 100644 hooks/restore/20-system_yunohost create mode 100644 hooks/restore/23-system_mail create mode 100644 hooks/restore/26-system_xmpp create mode 100644 hooks/restore/29-system_nginx create mode 100644 hooks/restore/32-system_cron create mode 100644 hooks/restore/5-system_ldap create mode 100644 hooks/restore/8-system_ssh diff --git a/backup.py b/backup.py index e51e22885..93bd0da37 100644 --- a/backup.py +++ b/backup.py @@ -43,6 +43,8 @@ def backup(): # Create directory try: os.listdir(backup_dir) except OSError: os.makedirs(backup_dir) + os.system('chmod 755 /home/yunohost.backup /home/yunohost.backup/tmp') + os.system('chown -hR admin: %s' % backup_dir) # Run hook hook_callback('backup', [backup_dir]) diff --git a/hooks/backup/11-system_mysql b/hooks/backup/11-system_mysql new file mode 100644 index 000000000..e5ff07c52 --- /dev/null +++ b/hooks/backup/11-system_mysql @@ -0,0 +1,7 @@ +#!/bin/bash + +backup_dir="$1/mysql" +mkdir -p $backup_dir + +mysqlpwd=$(sudo cat /etc/yunohost/mysql) +sudo mysqldump -uroot -p"$mysqlpwd" mysql > $backup_dir/mysql.sql diff --git a/hooks/backup/14-system_ssowat b/hooks/backup/14-system_ssowat new file mode 100644 index 000000000..f642ea243 --- /dev/null +++ b/hooks/backup/14-system_ssowat @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/ssowat" +mkdir -p $backup_dir + +sudo cp -a /etc/ssowat/* $backup_dir/ diff --git a/hooks/backup/17-system_home b/hooks/backup/17-system_home new file mode 100644 index 000000000..13414a755 --- /dev/null +++ b/hooks/backup/17-system_home @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/home" +mkdir -p $backup_dir + +sudo rsync -a --exclude='/yunohost*' /home/ $backup_dir/ diff --git a/hooks/backup/20-system_yunohost b/hooks/backup/20-system_yunohost new file mode 100644 index 000000000..305c82ed7 --- /dev/null +++ b/hooks/backup/20-system_yunohost @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/yunohost" +mkdir -p $backup_dir + +sudo cp -a /etc/yunohost/* $backup_dir/ diff --git a/hooks/backup/23-system_mail b/hooks/backup/23-system_mail new file mode 100644 index 000000000..ad1cf5ae5 --- /dev/null +++ b/hooks/backup/23-system_mail @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/mail" +mkdir -p $backup_dir + +sudo cp -a /var/mail/* $backup_dir/ diff --git a/hooks/backup/26-system_xmpp b/hooks/backup/26-system_xmpp new file mode 100644 index 000000000..bfa138196 --- /dev/null +++ b/hooks/backup/26-system_xmpp @@ -0,0 +1,7 @@ +#!/bin/bash + +backup_dir="$1/xmpp" +mkdir -p $backup_dir/{etc,var} + +sudo cp -a /etc/metronome/* $backup_dir/etc/ +sudo cp -a /var/lib/metronome/* $backup_dir/var/ diff --git a/hooks/backup/29-system_nginx b/hooks/backup/29-system_nginx new file mode 100644 index 000000000..9d1bc5b0b --- /dev/null +++ b/hooks/backup/29-system_nginx @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/nginx" +mkdir -p $backup_dir + +sudo cp -a /etc/nginx/conf.d/* $backup_dir/ diff --git a/hooks/backup/32-system_cron b/hooks/backup/32-system_cron new file mode 100644 index 000000000..b529b0216 --- /dev/null +++ b/hooks/backup/32-system_cron @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/cron" +mkdir -p $backup_dir + +sudo cp -a /etc/cron.d/yunohost* $backup_dir/ diff --git a/hooks/backup/5-system_ldap b/hooks/backup/5-system_ldap new file mode 100644 index 000000000..88d824b60 --- /dev/null +++ b/hooks/backup/5-system_ldap @@ -0,0 +1,9 @@ +#!/bin/bash + +backup_dir="$1/ldap" +mkdir -p $backup_dir + +sudo cp -a /etc/ldap/slapd.conf $backup_dir/ +slapcat -l $backup_dir/slapcat.ldif.raw +egrep -v "^entryCSN:" < $backup_dir/slapcat.ldif.raw > $backup_dir/slapcat.ldif +rm -f $backup_dir/slapcat.ldif.raw diff --git a/hooks/backup/8-system_ssh b/hooks/backup/8-system_ssh new file mode 100644 index 000000000..bd5db66ac --- /dev/null +++ b/hooks/backup/8-system_ssh @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/ssh" +mkdir -p $backup_dir + +sudo cp -a /etc/ssh/* $backup_dir/ diff --git a/hooks/restore/11-system_mysql b/hooks/restore/11-system_mysql new file mode 100644 index 000000000..28db51483 --- /dev/null +++ b/hooks/restore/11-system_mysql @@ -0,0 +1,7 @@ +#!/bin/bash + +backup_dir="$1/mysql" + +mysqlpwd=$(sudo cat /etc/yunohost/mysql) +sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql +sudo mysqladmin flush-privileges -p"$mysqlpwd" diff --git a/hooks/restore/14-system_ssowat b/hooks/restore/14-system_ssowat new file mode 100644 index 000000000..4c00a5ab1 --- /dev/null +++ b/hooks/restore/14-system_ssowat @@ -0,0 +1,5 @@ +#!/bin/bash + +backup_dir="$1/ssowat" + +sudo cp -a $backup_dir/* /etc/ssowat/ diff --git a/hooks/restore/17-system_home b/hooks/restore/17-system_home new file mode 100644 index 000000000..d097dad08 --- /dev/null +++ b/hooks/restore/17-system_home @@ -0,0 +1,5 @@ +#!/bin/bash + +backup_dir="$1/home" + +sudo cp -a $backup_dir/* /home/ diff --git a/hooks/restore/20-system_yunohost b/hooks/restore/20-system_yunohost new file mode 100644 index 000000000..bb23a2ddd --- /dev/null +++ b/hooks/restore/20-system_yunohost @@ -0,0 +1,11 @@ +#!/bin/bash + +backup_dir="$1/yunohost" + +sudo cp -a $backup_dir/* /etc/yunohost/ +sudo yunohost app ssowatconf +sudo yunohost firewall reload + +# Reload interface name +sudo rm /etc/yunohost/interface +sudo apt-get install --reinstall -y yunohost-config-others diff --git a/hooks/restore/23-system_mail b/hooks/restore/23-system_mail new file mode 100644 index 000000000..28eafc332 --- /dev/null +++ b/hooks/restore/23-system_mail @@ -0,0 +1,9 @@ +#!/bin/bash + +backup_dir="$1/mail" + +sudo cp -a $backup_dir/* /var/mail/ + +# Restart services to use migrated certs +sudo service postfix restart +sudo service dovecot restart diff --git a/hooks/restore/26-system_xmpp b/hooks/restore/26-system_xmpp new file mode 100644 index 000000000..c9677f944 --- /dev/null +++ b/hooks/restore/26-system_xmpp @@ -0,0 +1,9 @@ +#!/bin/bash + +backup_dir="$1/xmpp" + +sudo cp -a $backup_dir/etc/* /etc/metronome/ +sudo cp -a $backup_dir/var/* /var/lib/metronome/ + +# Restart to apply new conf and certs +sudo service metronome restart diff --git a/hooks/restore/29-system_nginx b/hooks/restore/29-system_nginx new file mode 100644 index 000000000..ca72c76c1 --- /dev/null +++ b/hooks/restore/29-system_nginx @@ -0,0 +1,8 @@ +#!/bin/bash + +backup_dir="$1/nginx" + +sudo cp -a $backup_dir/* /etc/nginx/conf.d/ + +# Restart to use new conf and certs +sudo service nginx restart diff --git a/hooks/restore/32-system_cron b/hooks/restore/32-system_cron new file mode 100644 index 000000000..c237157c7 --- /dev/null +++ b/hooks/restore/32-system_cron @@ -0,0 +1,8 @@ +#!/bin/bash + +backup_dir="$1/cron" + +sudo cp -a $backup_dir/* /etc/cron.d/ + +# Restart just in case +sudo service cron restart diff --git a/hooks/restore/5-system_ldap b/hooks/restore/5-system_ldap new file mode 100644 index 000000000..58ca04ea9 --- /dev/null +++ b/hooks/restore/5-system_ldap @@ -0,0 +1,38 @@ +#!/bin/bash + +backup_dir="$1/ldap" + +if [ -z "$2" ]; then + + # We need to execute this script as root, since the ldap + # service will be shut down during the operation (and sudo + # won't be available) + sudo bash $(pwd)/$0 $1 sudoed + +else + service slapd stop + + # Backup old configuration + mv /var/lib/ldap /var/lib/ldap.old + + # Recreate new DB folder + mkdir /var/lib/ldap + chown openldap: /var/lib/ldap + chmod go-rwx /var/lib/ldap + + # Restore LDAP configuration (just to be sure) + cp -a $backup_dir/slapd.conf /etc/ldap/slapd.conf + + # Regenerate the configuration + rm -rf /etc/ldap/slapd.d/* + slaptest -u -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d + cp -rfp /var/lib/ldap.old/DB_CONFIG /var/lib/ldap + + # Import the database + slapadd -l $backup_dir/slapcat.ldif + + # Change permissions and restart slapd + chown openldap: /var/lib/ldap/* + service slapd start + rm -rf /var/lib/ldap.old +fi diff --git a/hooks/restore/8-system_ssh b/hooks/restore/8-system_ssh new file mode 100644 index 000000000..8f3d3b671 --- /dev/null +++ b/hooks/restore/8-system_ssh @@ -0,0 +1,6 @@ +#!/bin/bash + +backup_dir="$1/ssh" + +sudo cp -a $backup_dir/* /etc/ssh/ +sudo service ssh restart From 7d99cd8f4dec142314c3dca9cc5789124365010b Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 18:47:13 +0200 Subject: [PATCH 10/58] [fix] Rename hooks to sort them properly --- hooks/backup/{5-system_ldap => 05-system_ldap} | 0 hooks/backup/{8-system_ssh => 08-system_ssh} | 0 hooks/restore/{5-system_ldap => 05-system_ldap} | 0 hooks/restore/{8-system_ssh => 08-system_ssh} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename hooks/backup/{5-system_ldap => 05-system_ldap} (100%) rename hooks/backup/{8-system_ssh => 08-system_ssh} (100%) rename hooks/restore/{5-system_ldap => 05-system_ldap} (100%) rename hooks/restore/{8-system_ssh => 08-system_ssh} (100%) diff --git a/hooks/backup/5-system_ldap b/hooks/backup/05-system_ldap similarity index 100% rename from hooks/backup/5-system_ldap rename to hooks/backup/05-system_ldap diff --git a/hooks/backup/8-system_ssh b/hooks/backup/08-system_ssh similarity index 100% rename from hooks/backup/8-system_ssh rename to hooks/backup/08-system_ssh diff --git a/hooks/restore/5-system_ldap b/hooks/restore/05-system_ldap similarity index 100% rename from hooks/restore/5-system_ldap rename to hooks/restore/05-system_ldap diff --git a/hooks/restore/8-system_ssh b/hooks/restore/08-system_ssh similarity index 100% rename from hooks/restore/8-system_ssh rename to hooks/restore/08-system_ssh From 94b45bbecb03878bcae326ad64fb46426c725626 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 18:49:19 +0200 Subject: [PATCH 11/58] [fix] Take care of hook priority --- hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook.py b/hook.py index b06964975..5144ea0f3 100644 --- a/hook.py +++ b/hook.py @@ -94,7 +94,7 @@ def hook_callback(action, args=None): elif not isinstance(args, list): args = [args] - for hook in os.listdir(hook_folder + action): + for hook in sorted(os.listdir(hook_folder + action)): try: hook_exec(file=hook_folder + action +'/'+ hook, args=args) except: pass From 1dae7bc83cbfa4bae24e7981f714478082c1f5d6 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 19:36:42 +0200 Subject: [PATCH 12/58] [fix] typo --- actionsmap/yunohost.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 21bc23094..fed4abd8f 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -557,8 +557,8 @@ backup: category_help: Manage backups actions: - ### backup_init() - init: + ### backup_backup() + backup: action_help: Create an encrypted backup tarball api: POST /backup From e42291a9e78ecc487ca3887c20baaeb42d5f09bb Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 19:44:17 +0200 Subject: [PATCH 13/58] [fix] typo --- backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.py b/backup.py index 93bd0da37..d823db7e3 100644 --- a/backup.py +++ b/backup.py @@ -30,7 +30,7 @@ import time from moulinette.core import MoulinetteError -def backup(): +def backup_backup(): """ Create an encrypted backup tarball From 7862b2a64f5712008677fdbc30a93db183851bb6 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 20:05:04 +0200 Subject: [PATCH 14/58] [fix] Typo to avoid listing files --- hooks/backup/05-system_ldap | 2 +- hooks/backup/08-system_ssh | 2 +- hooks/backup/14-system_ssowat | 2 +- hooks/backup/20-system_yunohost | 2 +- hooks/backup/23-system_mail | 3 +-- hooks/backup/26-system_xmpp | 4 ++-- hooks/backup/29-system_nginx | 2 +- hooks/restore/08-system_ssh | 2 +- hooks/restore/14-system_ssowat | 2 +- hooks/restore/17-system_home | 2 +- hooks/restore/20-system_yunohost | 2 +- hooks/restore/23-system_mail | 2 +- hooks/restore/26-system_xmpp | 4 ++-- hooks/restore/29-system_nginx | 2 +- hooks/restore/32-system_cron | 2 +- 15 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/backup/05-system_ldap b/hooks/backup/05-system_ldap index 88d824b60..65c52fc79 100644 --- a/hooks/backup/05-system_ldap +++ b/hooks/backup/05-system_ldap @@ -4,6 +4,6 @@ backup_dir="$1/ldap" mkdir -p $backup_dir sudo cp -a /etc/ldap/slapd.conf $backup_dir/ -slapcat -l $backup_dir/slapcat.ldif.raw +sudo slapcat -l $backup_dir/slapcat.ldif.raw egrep -v "^entryCSN:" < $backup_dir/slapcat.ldif.raw > $backup_dir/slapcat.ldif rm -f $backup_dir/slapcat.ldif.raw diff --git a/hooks/backup/08-system_ssh b/hooks/backup/08-system_ssh index bd5db66ac..617eee0f2 100644 --- a/hooks/backup/08-system_ssh +++ b/hooks/backup/08-system_ssh @@ -3,4 +3,4 @@ backup_dir="$1/ssh" mkdir -p $backup_dir -sudo cp -a /etc/ssh/* $backup_dir/ +sudo cp -a /etc/ssh/. $backup_dir diff --git a/hooks/backup/14-system_ssowat b/hooks/backup/14-system_ssowat index f642ea243..f4ec8c428 100644 --- a/hooks/backup/14-system_ssowat +++ b/hooks/backup/14-system_ssowat @@ -3,4 +3,4 @@ backup_dir="$1/ssowat" mkdir -p $backup_dir -sudo cp -a /etc/ssowat/* $backup_dir/ +sudo cp -a /etc/ssowat/. $backup_dir diff --git a/hooks/backup/20-system_yunohost b/hooks/backup/20-system_yunohost index 305c82ed7..22d556c61 100644 --- a/hooks/backup/20-system_yunohost +++ b/hooks/backup/20-system_yunohost @@ -3,4 +3,4 @@ backup_dir="$1/yunohost" mkdir -p $backup_dir -sudo cp -a /etc/yunohost/* $backup_dir/ +sudo cp -a /etc/yunohost/. $backup_dir diff --git a/hooks/backup/23-system_mail b/hooks/backup/23-system_mail index ad1cf5ae5..2145c0bc4 100644 --- a/hooks/backup/23-system_mail +++ b/hooks/backup/23-system_mail @@ -1,6 +1,5 @@ #!/bin/bash backup_dir="$1/mail" -mkdir -p $backup_dir -sudo cp -a /var/mail/* $backup_dir/ +sudo cp -a /var/mail/. $backup_dir diff --git a/hooks/backup/26-system_xmpp b/hooks/backup/26-system_xmpp index bfa138196..836c73078 100644 --- a/hooks/backup/26-system_xmpp +++ b/hooks/backup/26-system_xmpp @@ -3,5 +3,5 @@ backup_dir="$1/xmpp" mkdir -p $backup_dir/{etc,var} -sudo cp -a /etc/metronome/* $backup_dir/etc/ -sudo cp -a /var/lib/metronome/* $backup_dir/var/ +sudo cp -a /etc/metronome/. $backup_dir/etc +sudo cp -a /var/lib/metronome/. $backup_dir/var diff --git a/hooks/backup/29-system_nginx b/hooks/backup/29-system_nginx index 9d1bc5b0b..6bbcae2c1 100644 --- a/hooks/backup/29-system_nginx +++ b/hooks/backup/29-system_nginx @@ -3,4 +3,4 @@ backup_dir="$1/nginx" mkdir -p $backup_dir -sudo cp -a /etc/nginx/conf.d/* $backup_dir/ +sudo cp -a /etc/nginx/conf.d/. $backup_dir diff --git a/hooks/restore/08-system_ssh b/hooks/restore/08-system_ssh index 8f3d3b671..09aaf9c9a 100644 --- a/hooks/restore/08-system_ssh +++ b/hooks/restore/08-system_ssh @@ -2,5 +2,5 @@ backup_dir="$1/ssh" -sudo cp -a $backup_dir/* /etc/ssh/ +sudo cp -a $backup_dir/. /etc/ssh sudo service ssh restart diff --git a/hooks/restore/14-system_ssowat b/hooks/restore/14-system_ssowat index 4c00a5ab1..df92fe10e 100644 --- a/hooks/restore/14-system_ssowat +++ b/hooks/restore/14-system_ssowat @@ -2,4 +2,4 @@ backup_dir="$1/ssowat" -sudo cp -a $backup_dir/* /etc/ssowat/ +sudo cp -a $backup_dir/. /etc/ssowat diff --git a/hooks/restore/17-system_home b/hooks/restore/17-system_home index d097dad08..a90794ad6 100644 --- a/hooks/restore/17-system_home +++ b/hooks/restore/17-system_home @@ -2,4 +2,4 @@ backup_dir="$1/home" -sudo cp -a $backup_dir/* /home/ +sudo cp -a $backup_dir/. /home diff --git a/hooks/restore/20-system_yunohost b/hooks/restore/20-system_yunohost index bb23a2ddd..1bd332763 100644 --- a/hooks/restore/20-system_yunohost +++ b/hooks/restore/20-system_yunohost @@ -2,7 +2,7 @@ backup_dir="$1/yunohost" -sudo cp -a $backup_dir/* /etc/yunohost/ +sudo cp -a $backup_dir/. /etc/yunohost sudo yunohost app ssowatconf sudo yunohost firewall reload diff --git a/hooks/restore/23-system_mail b/hooks/restore/23-system_mail index 28eafc332..39f6f933f 100644 --- a/hooks/restore/23-system_mail +++ b/hooks/restore/23-system_mail @@ -2,7 +2,7 @@ backup_dir="$1/mail" -sudo cp -a $backup_dir/* /var/mail/ +sudo cp -a $backup_dir/. /var/mail # Restart services to use migrated certs sudo service postfix restart diff --git a/hooks/restore/26-system_xmpp b/hooks/restore/26-system_xmpp index c9677f944..c1b4e360e 100644 --- a/hooks/restore/26-system_xmpp +++ b/hooks/restore/26-system_xmpp @@ -2,8 +2,8 @@ backup_dir="$1/xmpp" -sudo cp -a $backup_dir/etc/* /etc/metronome/ -sudo cp -a $backup_dir/var/* /var/lib/metronome/ +sudo cp -a $backup_dir/etc/. /etc/metronome +sudo cp -a $backup_dir/var/. /var/lib/metronome # Restart to apply new conf and certs sudo service metronome restart diff --git a/hooks/restore/29-system_nginx b/hooks/restore/29-system_nginx index ca72c76c1..62810985b 100644 --- a/hooks/restore/29-system_nginx +++ b/hooks/restore/29-system_nginx @@ -2,7 +2,7 @@ backup_dir="$1/nginx" -sudo cp -a $backup_dir/* /etc/nginx/conf.d/ +sudo cp -a $backup_dir/. /etc/nginx/conf.d # Restart to use new conf and certs sudo service nginx restart diff --git a/hooks/restore/32-system_cron b/hooks/restore/32-system_cron index c237157c7..b1a53c3d0 100644 --- a/hooks/restore/32-system_cron +++ b/hooks/restore/32-system_cron @@ -2,7 +2,7 @@ backup_dir="$1/cron" -sudo cp -a $backup_dir/* /etc/cron.d/ +sudo cp -a $backup_dir/. /etc/cron.d # Restart just in case sudo service cron restart From 8be3473af1d6b11001cff0af50900b566a9d95ac Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 20:08:43 +0200 Subject: [PATCH 15/58] [fix] Typo --- backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.py b/backup.py index d823db7e3..52f09ac53 100644 --- a/backup.py +++ b/backup.py @@ -51,4 +51,4 @@ def backup_backup(): #TODO: Compress & encrypt - msignals.display(m18n.n('backup_completed'), 'success') + msignals.display(m18n.n('backup_complete'), 'success') From 9036c293d2e1b8dc7aa523ddf946ce651069912e Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 20:18:31 +0200 Subject: [PATCH 16/58] [fix] Import error --- tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools.py b/tools.py index 82d7325c8..31bf27b28 100644 --- a/tools.py +++ b/tools.py @@ -124,7 +124,6 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): '/etc/metronome/metronome.cfg.lua', '/etc/dovecot/dovecot.conf', '/usr/share/yunohost/yunohost-config/others/startup', - '/home/yunohost.backup/tahoe/tahoe.cfg', '/etc/amavis/conf.d/05-node_id', '/etc/amavis/conf.d/50-user' ] @@ -219,7 +218,6 @@ def tools_postinstall(domain, password, ignore_dyndns=False): """ from moulinette.core import init_authenticator - from yunohost.backup import backup_init from yunohost.app import app_ssowatconf from yunohost.firewall import firewall_upnp, firewall_reload From 31ac4d72d43498f41d94b5f0fc5a883b93e3b48c Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 20:20:31 +0200 Subject: [PATCH 17/58] [fix] typo --- tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools.py b/tools.py index 31bf27b28..2b13c7141 100644 --- a/tools.py +++ b/tools.py @@ -212,7 +212,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False): Keyword argument: domain -- YunoHost main domain - dyndns -- Subscribe domain to a DynDNS service + ignore_dyndns -- Do not subscribe domain to a DynDNS service password -- YunoHost admin password """ @@ -221,6 +221,8 @@ def tools_postinstall(domain, password, ignore_dyndns=False): from yunohost.app import app_ssowatconf from yunohost.firewall import firewall_upnp, firewall_reload + dyndns = not ignore_dyndns + try: with open('/etc/yunohost/installed') as f: pass except IOError: From a4b5ec6f2d3e013b21ad92b9a8f02da90eabffe9 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 20:39:31 +0200 Subject: [PATCH 18/58] [fix] Typo --- dyndns.py | 2 +- tools.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dyndns.py b/dyndns.py index bd2bcd486..5d5257815 100644 --- a/dyndns.py +++ b/dyndns.py @@ -52,7 +52,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None try: if requests.get('http://%s/test/%s' % (subscribe_host, domain)).status_code != 200: raise MoulinetteError(errno.EEXIST, m18n.n('dyndns_unavailable')) - except ConnectionError: + except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if key is None: diff --git a/tools.py b/tools.py index 2b13c7141..ed92bd9de 100644 --- a/tools.py +++ b/tools.py @@ -191,8 +191,7 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): raise MoulinetteError(errno.EPERM, m18n.n('maindomain_change_failed')) - if dyndns: dyndns_subscribe(domain=new_domain) - elif len(new_domain.split('.')) >= 3: + if dyndns and len(new_domain.split('.')) >= 3: try: r = requests.get('http://dyndns.yunohost.org/domains') except ConnectionError: From cde48996cf091986911bb5cfe5a5166bbfa88114 Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 23:08:03 +0200 Subject: [PATCH 19/58] [enh] Restore function --- actionsmap/yunohost.yml | 11 +++++++++++ backup.py | 35 +++++++++++++++++++++++++++++++++++ locales/en.json | 2 ++ locales/fr.json | 2 ++ 4 files changed, 50 insertions(+) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index fed4abd8f..f244f67ec 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -562,6 +562,17 @@ backup: action_help: Create an encrypted backup tarball api: POST /backup + ### backup_restore() + restore: + action_help: Restore from an encrypted backup tarball + api: POST /restore + configuration: + authenticate: false + lock: false + arguments: + path: + help: Path of the restauration package + ############################# # Monitor # diff --git a/backup.py b/backup.py index 52f09ac53..335998c4d 100644 --- a/backup.py +++ b/backup.py @@ -52,3 +52,38 @@ def backup_backup(): #TODO: Compress & encrypt msignals.display(m18n.n('backup_complete'), 'success') + + +def backup_restore(path): + """ + Restore from an encrypted backup tarball + + Keyword argument: + path -- Path to the restore directory + + """ + from yunohost.tools import tools_postinstall + from yunohost.hook import hook_callback + + path = os.path.abspath(path) + + try: + with open("%s/yunohost/current_host" % path, 'r') as f: + domain = f.readline().rstrip() + except IOError: + raise MoulinetteError(errno.EINVAL, m18n.n('invalid_restore_package')) + + #TODO Decrypt & extract tarball + + try: + with open('/etc/yunohost/installed') as f: + raise MoulinetteError(errno.EINVAL, m18n.n('yunohost_already_installed')) + except IOError: + tools_postinstall(domain, 'yunohost', True) + + # Run hook + hook_callback('restore', [path]) + + msignals.display(m18n.n('restore_complete'), 'success') + + diff --git a/locales/en.json b/locales/en.json index f615178fb..09d42bdd8 100644 --- a/locales/en.json +++ b/locales/en.json @@ -129,6 +129,8 @@ "system_upgraded" : "System successfully upgraded", "backup_complete" : "Backup complete", + "invalid_restore_package" : "Invalid restore package", + "restore_complete" : "Restore complete", "field_invalid" : "Invalid field '{:s}'", "mail_domain_unknown" : "Unknown mail address domain '{:s}'", diff --git a/locales/fr.json b/locales/fr.json index 60880cb57..a913951b3 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -129,6 +129,8 @@ "system_upgraded" : "Système mis à jour avec succès", "backup_complete" : "Sauvegarde terminée", + "invalid_restore_package" : "Dossier de restauration invalide", + "restore_complete" : "Restauration terminée", "field_invalid" : "Champ incorrect : {:s}", "mail_domain_unknown" : "Domaine '{:s}' de l'adresse mail inconnu", From dfe373d591cb35f6c0e91264f4578cdb083018cb Mon Sep 17 00:00:00 2001 From: kload Date: Sat, 25 Oct 2014 23:52:35 +0200 Subject: [PATCH 20/58] [fix] Do not lock backup action --- actionsmap/yunohost.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index f244f67ec..6fb6e9080 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -561,6 +561,8 @@ backup: backup: action_help: Create an encrypted backup tarball api: POST /backup + configuration: + lock: false ### backup_restore() restore: From f4e369d4bdfd82f61f6a59d6bb61fd2b6fda1857 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 00:03:09 +0200 Subject: [PATCH 21/58] [fix] Allow backup_restore action when YunoHost is not postinstalled --- bin/yunohost | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/yunohost b/bin/yunohost index 7913de091..947845586 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -38,7 +38,8 @@ if __name__ == '__main__': # Check that YunoHost is installed if not os.path.isfile('/etc/yunohost/installed') and \ - (len(args) < 2 or args[0] != 'tools' or args[1] != 'postinstall'): + (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \ + args[0] +' '+ args[1] != 'backup restore')): from moulinette.interfaces.cli import colorize, get_locale # Init i18n From 66a01dc066035549bf487240bf6e4949692e0290 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 00:04:04 +0200 Subject: [PATCH 22/58] [fix] Import error --- backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backup.py b/backup.py index 335998c4d..39e1feb81 100644 --- a/backup.py +++ b/backup.py @@ -26,6 +26,7 @@ import os import sys import json +import errno import time from moulinette.core import MoulinetteError From 1a06c4d7b2608bf0920c55d3c851b8b4fdb567f3 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 00:15:46 +0200 Subject: [PATCH 23/58] [fix] Add backup and restore hooks from scripts dir --- backup.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backup.py b/backup.py index 39e1feb81..c4c708255 100644 --- a/backup.py +++ b/backup.py @@ -28,6 +28,7 @@ import sys import json import errno import time +import shutil from moulinette.core import MoulinetteError @@ -47,6 +48,15 @@ def backup_backup(): os.system('chmod 755 /home/yunohost.backup /home/yunohost.backup/tmp') os.system('chown -hR admin: %s' % backup_dir) + # Add app's backup hooks + try: + for app_id in os.listdir('/etc/yunohost/apps'): + hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' + with open(hook, 'r') as f: + hook_add(app_id, hook) + except IOError: + pass + # Run hook hook_callback('backup', [backup_dir]) @@ -64,6 +74,7 @@ def backup_restore(path): """ from yunohost.tools import tools_postinstall + from yunohost.hook import hook_add from yunohost.hook import hook_callback path = os.path.abspath(path) @@ -82,6 +93,15 @@ def backup_restore(path): except IOError: tools_postinstall(domain, 'yunohost', True) + # Add app's restore hooks + try: + for app_id in os.listdir('/etc/yunohost/apps'): + hook = '/etc/yunohost/apps/'+ app_id +'/scripts/restore' + with open(hook, 'r') as f: + hook_add(app_id, hook) + except IOError: + pass + # Run hook hook_callback('restore', [path]) From 3976f0570361a44825af4cd545215e6e9057d849 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 00:30:40 +0200 Subject: [PATCH 24/58] [fix] import error --- backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backup.py b/backup.py index c4c708255..1bb1f5aba 100644 --- a/backup.py +++ b/backup.py @@ -37,6 +37,7 @@ def backup_backup(): Create an encrypted backup tarball """ + from yunohost.hook import hook_add from yunohost.hook import hook_callback backup_dirname = int(time.time()) From 7ab1e67a94cff88d964953b1f8705064163992ce Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 00:48:40 +0200 Subject: [PATCH 25/58] [fix] Allow restoration after postinstallation (risky) --- backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backup.py b/backup.py index 1bb1f5aba..fc1c21590 100644 --- a/backup.py +++ b/backup.py @@ -90,7 +90,8 @@ def backup_restore(path): try: with open('/etc/yunohost/installed') as f: - raise MoulinetteError(errno.EINVAL, m18n.n('yunohost_already_installed')) + #raise MoulinetteError(errno.EINVAL, m18n.n('yunohost_already_installed')) + pass except IOError: tools_postinstall(domain, 'yunohost', True) From fe9ee9777453547a090e08f2afa8761ba84e092d Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 26 Oct 2014 01:05:18 +0200 Subject: [PATCH 26/58] [enh] Warn user about unbackup/unrestored apps --- backup.py | 13 +++++++++++-- locales/en.json | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backup.py b/backup.py index fc1c21590..2a650574e 100644 --- a/backup.py +++ b/backup.py @@ -53,8 +53,12 @@ def backup_backup(): try: for app_id in os.listdir('/etc/yunohost/apps'): hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' - with open(hook, 'r') as f: + if os.path.isfile(hook): hook_add(app_id, hook) + else: + msignals.display(m18n.n('unbackup_app', app_id), + 'warning') + except IOError: pass @@ -91,6 +95,8 @@ def backup_restore(path): try: with open('/etc/yunohost/installed') as f: #raise MoulinetteError(errno.EINVAL, m18n.n('yunohost_already_installed')) + msignals.display(m18n.n('restoring_installed_system'), 'warning') + time.sleep(5) pass except IOError: tools_postinstall(domain, 'yunohost', True) @@ -99,8 +105,11 @@ def backup_restore(path): try: for app_id in os.listdir('/etc/yunohost/apps'): hook = '/etc/yunohost/apps/'+ app_id +'/scripts/restore' - with open(hook, 'r') as f: + if os.path.isfile(hook): hook_add(app_id, hook) + else: + msignals.display(m18n.n('unrestore_app', app_id), + 'warning') except IOError: pass diff --git a/locales/en.json b/locales/en.json index 09d42bdd8..2f5d4597d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -131,6 +131,9 @@ "backup_complete" : "Backup complete", "invalid_restore_package" : "Invalid restore package", "restore_complete" : "Restore complete", + "restoring_installed_system" : "You are trying to restore a backup on an already installed system, abort unless you know what you are doing!", + "unbackup_app" : "'{:s}' will NOT be saved", + "unrestore_app" : "'{:s}' will NOT be restored", "field_invalid" : "Invalid field '{:s}'", "mail_domain_unknown" : "Unknown mail address domain '{:s}'", From 8e220072aa0c0d370f37ec402439480d67827e5c Mon Sep 17 00:00:00 2001 From: opi Date: Thu, 30 Oct 2014 15:33:39 +0100 Subject: [PATCH 27/58] [enh] Add amavis to service.yml --- data/services.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/services.yml b/data/services.yml index f0ee728b4..e699c7b6a 100644 --- a/data/services.yml +++ b/data/services.yml @@ -39,3 +39,6 @@ yunohost-api: postgrey: status: service log: /var/log/mail.log +amavis: + status: service + log: /var/log/mail.log From 8e597109be0fc43d48e529797e16b8415f1666e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sat, 1 Nov 2014 17:50:28 +0100 Subject: [PATCH 28/58] [enh] Review executables structure and add logging configuration --- bin/yunohost | 154 ++++++++++++++++++++++++++++++++++++++--------- bin/yunohost-api | 150 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 248 insertions(+), 56 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 947845586..8f2ab92ae 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,53 +4,147 @@ import sys import os -from_source = False +# Either we are in a development environment or not +IN_DEVEL = False -# Run from source -basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) -if os.path.isdir('%s/moulinette' % basedir): - sys.path.insert(0, basedir) - from_source = True +# Either cache has to be used inside the moulinette or not +USE_CACHE = True -from moulinette import init, cli -from moulinette.interfaces.cli import colorize +# Either the output has to be encoded as a JSON encoded string or not +PRINT_JSON = False + +# Level for which loggers will log +LOGGERS_LEVEL = 'INFO' + +# Handlers that will be used by loggers +# - file: log to the file LOG_DIR/LOG_FILE +# - console: log to stdout +LOGGERS_HANDLERS = ['file'] + +# Directory and file to be used by logging +LOG_DIR = '/var/log/yunohost' +LOG_FILE = 'yunohost-cli.log' -## Main action +# Initialization & helpers functions ----------------------------------- + +def _die(message, title='Error:'): + """Print error message and exit""" + try: + from moulinette.interfaces.cli import colorize + except ImportError: + colorize = lambda msg, c: msg + print('%s %s' % (colorize(title, 'red'), message)) + sys.exit(1) + +def _check_in_devel(): + """Check and load if needed development environment""" + global IN_DEVEL, LOG_DIR + basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) + if os.path.isdir('%s/moulinette' % basedir) and not IN_DEVEL: + # Add base directory to python path + sys.path.insert(0, basedir) + + # Update global variables + IN_DEVEL = True + LOG_DIR = '%s/log' % basedir + +def _parse_argv(): + """Parse additional arguments and return remaining ones""" + argv = list(sys.argv) + argv.pop(0) + + if '--no-cache' in argv: + global USE_CACHE + USE_CACHE = False + argv.remove('--no-cache') + if '--json' in argv: + global PRINT_JSON + PRINT_JSON = True + argv.remove('--json') + if '--debug' in argv: + global LOGGERS_LEVEL + LOGGERS_LEVEL = 'DEBUG' + argv.remove('--debug') + if '--verbose' in argv: + global LOGGERS_HANDLERS + if 'console' not in LOGGERS_HANDLERS: + LOGGERS_HANDLERS.append('console') + argv.remove('--verbose') + return argv + +def _init_moulinette(): + """Configure logging and initialize the moulinette""" + from moulinette import init + + # Custom logging configuration + logging = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'simple': { + 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s - %(message)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'stream': 'ext://sys.stdout', + }, + 'file': { + 'class': 'logging.FileHandler', + 'formatter': 'precise', + 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + }, + }, + 'loggers': { + 'moulinette': { + 'level': LOGGERS_LEVEL, + 'handlers': LOGGERS_HANDLERS, + }, + 'yunohost': { + 'level': LOGGERS_LEVEL, + 'handlers': LOGGERS_HANDLERS, + }, + }, + } + + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) + + # Initialize moulinette + init(logging_config=logging, _from_source=IN_DEVEL) + + +# Main action ---------------------------------------------------------- if __name__ == '__main__': - # Run from source - init(_from_source=from_source) - - # Additional arguments - cache = True - json = False - if '--no-cache' in sys.argv: - cache = False - sys.argv.remove('--no-cache') - if '--json' in sys.argv: - json = True - sys.argv.remove('--json') - - # Retrieve remaining arguments - args = list(sys.argv) - args.pop(0) + _check_in_devel() + args = _parse_argv() + _init_moulinette() # Check that YunoHost is installed if not os.path.isfile('/etc/yunohost/installed') and \ (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \ args[0] +' '+ args[1] != 'backup restore')): - from moulinette.interfaces.cli import colorize, get_locale + from moulinette.interfaces.cli import get_locale # Init i18n m18n.load_namespace('yunohost') m18n.set_locale(get_locale()) # Print error and exit - print('%s %s' % (colorize(m18n.g('error'), 'red'), - m18n.n('yunohost_not_installed'))) - sys.exit(1) + _die(m18n.n('yunohost_not_installed'), m18n.g('error')) # Execute the action - ret = cli(['yunohost'], args, print_json=json, use_cache=cache) + from moulinette import cli + ret = cli(['yunohost'], args, print_json=PRINT_JSON, use_cache=USE_CACHE) sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index b8dcab558..515cc8361 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -4,18 +4,127 @@ import sys import os.path -from_source = False +# Either we are in a development environment or not +IN_DEVEL = False -# Run from source -basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) -if os.path.isdir('%s/moulinette' % basedir): - sys.path.insert(0, basedir) - from_source = True +# Either cache has to be used inside the moulinette or not +USE_CACHE = True -from moulinette import init, api, MoulinetteError +# Either WebSocket has to be installed by the moulinette or not +USE_WEBSOCKET = True + +# Level for which loggers will log +LOGGERS_LEVEL = 'INFO' + +# Handlers that will be used by loggers +# - file: log to the file LOG_DIR/LOG_FILE +# - console: log to stdout +LOGGERS_HANDLERS = ['file'] + +# Directory and file to be used by logging +LOG_DIR = '/var/log/yunohost' +LOG_FILE = 'yunohost-api.log' -## Callbacks for additional routes +# Initialization & helpers functions ----------------------------------- + +def _die(message, title='Error:'): + """Print error message and exit""" + try: + from moulinette.interfaces.cli import colorize + except ImportError: + colorize = lambda msg, c: msg + print('%s %s' % (colorize(title, 'red'), message)) + sys.exit(1) + +def _check_in_devel(): + """Check and load if needed development environment""" + global IN_DEVEL, LOG_DIR + basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) + if os.path.isdir('%s/moulinette' % basedir) and not IN_DEVEL: + # Add base directory to python path + sys.path.insert(0, basedir) + + # Update global variables + IN_DEVEL = True + LOG_DIR = '%s/log' % basedir + +def _parse_argv(): + """Parse additional arguments and return remaining ones""" + argv = list(sys.argv) + argv.pop(0) + + if '--no-cache' in argv: + global USE_CACHE + USE_CACHE = False + argv.remove('--no-cache') + if '--no-websocket' in argv: + global USE_WEBSOCKET + USE_WEBSOCKET = False + argv.remove('--no-websocket') + if '--debug' in argv: + global LOGGERS_LEVEL + LOGGERS_LEVEL = 'DEBUG' + argv.remove('--debug') + if '--verbose' in argv: + global LOGGERS_HANDLERS + if 'console' not in LOGGERS_HANDLERS: + LOGGERS_HANDLERS.append('console') + argv.remove('--verbose') + return argv + +def _init_moulinette(): + """Configure logging and initialize the moulinette""" + from moulinette import init + + # Custom logging configuration + logging = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'simple': { + 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s - %(message)s' + }, + 'precise': { + 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'stream': 'ext://sys.stdout', + }, + 'file': { + 'class': 'logging.handlers.WatchedFileHandler', + 'formatter': 'precise', + 'filename': '%s/%s' % (LOG_DIR, LOG_FILE), + }, + }, + 'loggers': { + 'moulinette': { + 'level': LOGGERS_LEVEL, + 'handlers': LOGGERS_HANDLERS, + }, + 'yunohost': { + 'level': LOGGERS_LEVEL, + 'handlers': LOGGERS_HANDLERS, + }, + }, + } + + # Create log directory + if not os.path.isdir(LOG_DIR): + try: + os.makedirs(LOG_DIR, 0750) + except os.error as e: + _die(str(e)) + + # Initialize moulinette + init(logging_config=logging, _from_source=IN_DEVEL) + + +# Callbacks for additional routes -------------------------------------- def is_installed(): """ @@ -28,30 +137,19 @@ def is_installed(): return { 'installed': installed } -## Main action +# Main action ---------------------------------------------------------- if __name__ == '__main__': - # Run from source - init(_from_source=from_source) - - # Additional arguments - cache = True - websocket = True - if '--no-cache' in sys.argv: - cache = False - sys.argv.remove('--no-cache') - if '--no-websocket' in sys.argv: - websocket = False - sys.argv.remove('--no-websocket') - # TODO: Add log argument + _check_in_devel() + _parse_argv() + _init_moulinette() + from moulinette import (api, MoulinetteError) try: # Run the server api(['yunohost'], port=6787, routes={('GET', '/installed'): is_installed}, - use_cache=cache, use_websocket=websocket) + use_cache=USE_CACHE, use_websocket=USE_WEBSOCKET) except MoulinetteError as e: - from moulinette.interfaces.cli import colorize - print('%s %s' % (colorize(m18n.g('error'), 'red'), e.strerror)) - sys.exit(e.errno) + _die(e.strerror, m18n.g('error')) sys.exit(0) From bd5d74ca28d88229595b2af2ff27b2972f6acc25 Mon Sep 17 00:00:00 2001 From: opi Date: Mon, 3 Nov 2014 23:43:38 +0100 Subject: [PATCH 29/58] [enh] Use https when calling dyndns api. --- domain.py | 2 +- tools.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/domain.py b/domain.py index 8218a9f65..76ed1c5a2 100644 --- a/domain.py +++ b/domain.py @@ -92,7 +92,7 @@ def domain_add(auth, domain, dyndns=False): from yunohost.dyndns import dyndns_subscribe try: - r = requests.get('http://dyndns.yunohost.org/domains') + r = requests.get('https://dyndns.yunohost.org/domains') except ConnectionError: pass else: diff --git a/tools.py b/tools.py index ed92bd9de..4ed21cb2d 100644 --- a/tools.py +++ b/tools.py @@ -193,7 +193,7 @@ def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): if dyndns and len(new_domain.split('.')) >= 3: try: - r = requests.get('http://dyndns.yunohost.org/domains') + r = requests.get('https://dyndns.yunohost.org/domains') except ConnectionError: pass else: @@ -231,14 +231,14 @@ def tools_postinstall(domain, password, ignore_dyndns=False): if len(domain.split('.')) >= 3 and not ignore_dyndns: try: - r = requests.get('http://dyndns.yunohost.org/domains') + r = requests.get('https://dyndns.yunohost.org/domains') except ConnectionError: pass else: dyndomains = json.loads(r.text) dyndomain = '.'.join(domain.split('.')[1:]) if dyndomain in dyndomains: - if requests.get('http://dyndns.yunohost.org/test/%s' % domain).status_code == 200: + if requests.get('https://dyndns.yunohost.org/test/%s' % domain).status_code == 200: dyndns=True else: raise MoulinetteError(errno.EEXIST, From c3cadc1d3cd0f7a6e06003f3475cb9d00a3e890a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Tue, 4 Nov 2014 00:39:49 +0100 Subject: [PATCH 30/58] [enh] Implement compression and add ignore_apps argument to backup_backup --- actionsmap/yunohost.yml | 4 ++ backup.py | 91 +++++++++++++++++++++++++++++++---------- locales/en.json | 3 ++ 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 6fb6e9080..fc8ed2e2e 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -563,6 +563,10 @@ backup: api: POST /backup configuration: lock: false + arguments: + --ignore-apps: + help: Do not backup apps + action: store_true ### backup_restore() restore: diff --git a/backup.py b/backup.py index 2a650574e..ba0bac4e3 100644 --- a/backup.py +++ b/backup.py @@ -29,43 +29,92 @@ import json import errno import time import shutil +import tarfile from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger -def backup_backup(): +backup_path = '/home/yunohost.backup' +archives_path = '%s/archives' % backup_path + +logger = getActionLogger('yunohost.backup') + + +def backup_backup(ignore_apps=False): """ Create an encrypted backup tarball + Keyword arguments: + ignore_apps -- Do not backup apps + """ from yunohost.hook import hook_add from yunohost.hook import hook_callback - backup_dirname = int(time.time()) - backup_dir = "/home/yunohost.backup/tmp/%s" % backup_dirname + timestamp = int(time.time()) + tmp_dir = "%s/tmp/%s" % (backup_path, timestamp) - # Create directory - try: os.listdir(backup_dir) - except OSError: os.makedirs(backup_dir) - os.system('chmod 755 /home/yunohost.backup /home/yunohost.backup/tmp') - os.system('chown -hR admin: %s' % backup_dir) + # Create temporary directory + if os.path.isdir(tmp_dir): + logger.warning("temporary directory for backup '%s' already exists", tmp_dir) + os.system('rm -rf %s' % tmp_dir) + try: + os.mkdir(tmp_dir, 0750) + except OSError: + # Create temporary directory recursively + os.makedirs(tmp_dir, 0750) + os.system('chown -hR admin: %s' % backup_path) + else: + os.system('chown -hR admin: %s' % tmp_dir) # Add app's backup hooks + if not ignore_apps: + try: + for app_id in os.listdir('/etc/yunohost/apps'): + hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' + if os.path.isfile(hook): + hook_add(app_id, hook) + else: + logger.warning("unable to find app's backup hook '%s'", hook) + msignals.display(m18n.n('unbackup_app', app_id), + 'warning') + except IOError as e: + logger.info("unable to add app's backup hooks: %s", str(e)) + + # Run hooks + m18n.display(m18n.n('backup_running_hooks')) + hook_callback('backup', [tmp_dir]) + + # TODO: Add a backup info file + + # Create the archive + m18n.display(m18n.n('backup_creating_archive')) + archive_file = "%s/%s.tar.gz" % (archives_path, timestamp) try: - for app_id in os.listdir('/etc/yunohost/apps'): - hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' - if os.path.isfile(hook): - hook_add(app_id, hook) - else: - msignals.display(m18n.n('unbackup_app', app_id), - 'warning') + tar = tarfile.open(archive_file, "w:gz") + except: + tar = None - except IOError: - pass + # Create the archives directory and retry + if not os.path.isdir(archives_path): + os.mkdir(archives_path, 0750) + try: + tar = tarfile.open(archive_file, "w:gz") + except: + logger.exception("unable to open the archive '%s' for writing " \ + "after creating directory '%s'", + archive_file, archive_dir) + tar = None + else: + logger.exception("unable to open the archive '%s' for writing", + archive_file) + if tar is None: + raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed')) + tar.add(tmp_dir, arcname='') + tar.close() - # Run hook - hook_callback('backup', [backup_dir]) - - #TODO: Compress & encrypt + # Remove temporary directory + os.system('rm -rf %s' % tmp_dir) msignals.display(m18n.n('backup_complete'), 'success') diff --git a/locales/en.json b/locales/en.json index 2f5d4597d..3c54b87aa 100644 --- a/locales/en.json +++ b/locales/en.json @@ -128,6 +128,9 @@ "packages_upgrade_failed" : "Unable to upgrade all packages", "system_upgraded" : "System successfully upgraded", + "backup_running_hooks" : "Running backup hooks...", + "backup_creating_archive" : "Creating the backup archive...", + "backup_archive_open_failed" : "Unable to open the backup archive", "backup_complete" : "Backup complete", "invalid_restore_package" : "Invalid restore package", "restore_complete" : "Restore complete", From 82d8b0ac29b43d84a9d676601d1add672378bfd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Tue, 4 Nov 2014 00:43:15 +0100 Subject: [PATCH 31/58] [enh] Rename backup_backup to backup_create --- actionsmap/yunohost.yml | 6 +++--- backup.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index fc8ed2e2e..a4e579425 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -557,9 +557,9 @@ backup: category_help: Manage backups actions: - ### backup_backup() - backup: - action_help: Create an encrypted backup tarball + ### backup_create() + create: + action_help: Backup and create a local archive api: POST /backup configuration: lock: false diff --git a/backup.py b/backup.py index ba0bac4e3..a8db4fb8b 100644 --- a/backup.py +++ b/backup.py @@ -40,9 +40,9 @@ archives_path = '%s/archives' % backup_path logger = getActionLogger('yunohost.backup') -def backup_backup(ignore_apps=False): +def backup_create(ignore_apps=False): """ - Create an encrypted backup tarball + Backup and create a local archive Keyword arguments: ignore_apps -- Do not backup apps From d1cb0b0d80569aac7be7d6fd5d57744c2252fb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Tue, 4 Nov 2014 14:11:18 +0100 Subject: [PATCH 32/58] [enh] Add basic 'list' and 'info' actions to the backup --- actionsmap/yunohost.yml | 17 +++++++++++++++++ backup.py | 42 +++++++++++++++++++++++++++++++++++++++++ locales/en.json | 5 ++++- locales/fr.json | 3 ++- 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index a4e579425..cba6f095a 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -579,6 +579,23 @@ backup: path: help: Path of the restauration package + ### backup_list() + list: + action_help: List available local backup archives + api: GET /backup/archives + configuration: + lock: false + + ### backup_info() + info: + action_help: Show info about a local backup archive + api: GET /backup/archives/ + configuration: + lock: false + arguments: + name: + help: Name of the local backup archive + ############################# # Monitor # diff --git a/backup.py b/backup.py index a8db4fb8b..0d8285bfd 100644 --- a/backup.py +++ b/backup.py @@ -168,3 +168,45 @@ def backup_restore(path): msignals.display(m18n.n('restore_complete'), 'success') +def backup_list(): + """ + List available local backup archives + + """ + result = [] + + try: + # Retrieve local archives + archives = os.listdir(archives_path) + except IOError as e: + logging.info("unable to iterate over local archives: %s", str(e)) + else: + # Iterate over local archives + for f in archives: + try: + name = f[:f.rindex('.tar.gz')] + except ValueError: + continue + result.append(name) + + return { 'archives': result } + +def backup_info(name): + """ + Get info about a local backup archive + + Keyword arguments: + name -- Name of the local backup archive + + """ + archive_file = '%s/%s.tar.gz' % (archives_path, name) + if not os.path.isfile(archive_file): + logger.error("no local backup archive found at '%s'", archive_file) + raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown')) + + return { + 'path': archive_file, + # TODO: Retrieve created_at from the info file + 'created_at': time.strftime(m18n.n('format_datetime_short'), + time.gmtime(int(name))), + } diff --git a/locales/en.json b/locales/en.json index 3c54b87aa..ff553847f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -131,6 +131,7 @@ "backup_running_hooks" : "Running backup hooks...", "backup_creating_archive" : "Creating the backup archive...", "backup_archive_open_failed" : "Unable to open the backup archive", + "backup_archive_name_unknown" : "Unknown local backup archive name", "backup_complete" : "Backup complete", "invalid_restore_package" : "Invalid restore package", "restore_complete" : "Restore complete", @@ -168,6 +169,8 @@ "pattern_password" : "Must be at least 3 characters long", "pattern_domain" : "Must be a valid domain name (e.g. my-domain.org)", "pattern_listname" : "Must be alphanumeric and underscore characters only", - "pattern_port" : "Must be a valid port number (i.e. 0-65535)" + "pattern_port" : "Must be a valid port number (i.e. 0-65535)", + + "format_datetime_short" : "%m/%d/%Y %I:%M %p" } diff --git a/locales/fr.json b/locales/fr.json index a913951b3..e922c79b3 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -162,6 +162,7 @@ "pattern_password" : "Doit être composé d'au moins 3 caractères", "pattern_domain" : "Doit être un nom de domaine valide (ex : mon-domaine.org)", "pattern_listname" : "Doit être composé uniquement de caractères alphanumérique et de tiret bas", - "pattern_port" : "Doit être un numéro de port valide (0-65535)" + "pattern_port" : "Doit être un numéro de port valide (0-65535)", + "format_datetime_short" : "%d/%m/%Y %H:%M" } From c594ce03a6beccdf100394523e7c510e0255421f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Wed, 5 Nov 2014 00:17:18 +0100 Subject: [PATCH 33/58] [enh] Manage local backup archive and extend backup_restore --- actionsmap/yunohost.yml | 15 ++++-- backup.py | 108 +++++++++++++++++++++++++++------------- locales/en.json | 11 ++-- locales/fr.json | 12 ++++- 4 files changed, 101 insertions(+), 45 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index cba6f095a..86859654d 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -559,7 +559,7 @@ backup: ### backup_create() create: - action_help: Backup and create a local archive + action_help: Create a backup local archive api: POST /backup configuration: lock: false @@ -570,14 +570,19 @@ backup: ### backup_restore() restore: - action_help: Restore from an encrypted backup tarball + action_help: Restore from a local backup archive api: POST /restore configuration: authenticate: false - lock: false arguments: - path: - help: Path of the restauration package + name: + help: Name of the local backup archive + --ignore-apps: + help: Do not restore apps + action: store_true + --force: + help: Force restauration on an already installed system + action: store_true ### backup_list() list: diff --git a/backup.py b/backup.py index 0d8285bfd..4aa1d2a0c 100644 --- a/backup.py +++ b/backup.py @@ -42,7 +42,7 @@ logger = getActionLogger('yunohost.backup') def backup_create(ignore_apps=False): """ - Backup and create a local archive + Create a backup local archive Keyword arguments: ignore_apps -- Do not backup apps @@ -67,7 +67,7 @@ def backup_create(ignore_apps=False): else: os.system('chown -hR admin: %s' % tmp_dir) - # Add app's backup hooks + # Add apps backup hook if not ignore_apps: try: for app_id in os.listdir('/etc/yunohost/apps'): @@ -79,16 +79,16 @@ def backup_create(ignore_apps=False): msignals.display(m18n.n('unbackup_app', app_id), 'warning') except IOError as e: - logger.info("unable to add app's backup hooks: %s", str(e)) + logger.info("unable to add apps backup hook: %s", str(e)) # Run hooks - m18n.display(m18n.n('backup_running_hooks')) + msignals.display(m18n.n('backup_running_hooks')) hook_callback('backup', [tmp_dir]) # TODO: Add a backup info file # Create the archive - m18n.display(m18n.n('backup_creating_archive')) + msignals.display(m18n.n('backup_creating_archive')) archive_file = "%s/%s.tar.gz" % (archives_path, timestamp) try: tar = tarfile.open(archive_file, "w:gz") @@ -119,51 +119,89 @@ def backup_create(ignore_apps=False): msignals.display(m18n.n('backup_complete'), 'success') -def backup_restore(path): +def backup_restore(name, ignore_apps=False, force=False): """ - Restore from an encrypted backup tarball + Restore from a local backup archive Keyword argument: - path -- Path to the restore directory + name -- Name of the local backup archive + ignore_apps -- Do not restore apps + force -- Force restauration on an already installed system """ - from yunohost.tools import tools_postinstall from yunohost.hook import hook_add from yunohost.hook import hook_callback - path = os.path.abspath(path) - + # Retrieve and open the archive + archive_file = backup_info(name)['path'] try: - with open("%s/yunohost/current_host" % path, 'r') as f: + tar = tarfile.open(archive_file, "r:gz") + except: + logger.exception("unable to open the archive '%s' for reading", + archive_file) + raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed')) + + # Check temporary directory + tmp_dir = "%s/tmp/%s" % (backup_path, name) + if os.path.isdir(tmp_dir): + logger.warning("temporary directory for restoration '%s' already exists", + tmp_dir) + os.system('rm -rf %s' % tmp_dir) + + # Extract the tarball + msignals.display(m18n.n('backup_extracting_archive')) + tar.extractall(tmp_dir) + tar.close() + + # Retrieve domain from the backup + try: + with open("%s/yunohost/current_host" % tmp_dir, 'r') as f: domain = f.readline().rstrip() except IOError: - raise MoulinetteError(errno.EINVAL, m18n.n('invalid_restore_package')) + logger.error("unable to retrieve domain from '%s/yunohost/current_host'", + tmp_dir) + raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) - #TODO Decrypt & extract tarball - - try: - with open('/etc/yunohost/installed') as f: - #raise MoulinetteError(errno.EINVAL, m18n.n('yunohost_already_installed')) - msignals.display(m18n.n('restoring_installed_system'), 'warning') - time.sleep(5) - pass - except IOError: + # Check if YunoHost is installed + if os.path.isfile('/etc/yunohost/installed'): + msignals.display(m18n.n('yunohost_already_installed'), 'warning') + if not force: + try: + # Ask confirmation for restoring + i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed', + answers='y/N')) + except NotImplemented: + pass + else: + if i == 'y' or i == 'Y': + force = True + if not force: + raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed')) + else: + from yunohost.tools import tools_postinstall + logger.info("executing the post-install...") tools_postinstall(domain, 'yunohost', True) - # Add app's restore hooks - try: - for app_id in os.listdir('/etc/yunohost/apps'): - hook = '/etc/yunohost/apps/'+ app_id +'/scripts/restore' - if os.path.isfile(hook): - hook_add(app_id, hook) - else: - msignals.display(m18n.n('unrestore_app', app_id), - 'warning') - except IOError: - pass + # Add apps restore hook + if not ignore_apps: + try: + # TODO: Check if the app_id is part of the backup archive + for app_id in os.listdir('/etc/yunohost/apps'): + hook = '/etc/yunohost/apps/'+ app_id +'/scripts/restore' + if os.path.isfile(hook): + hook_add(app_id, hook) + else: + msignals.display(m18n.n('unrestore_app', app_id), + 'warning') + except IOError as e: + logger.info("unable to add apps restore hook: %s", str(e)) - # Run hook - hook_callback('restore', [path]) + # Run hooks + msignals.display(m18n.n('restore_running_hooks')) + hook_callback('restore', [tmp_dir]) + + # Remove temporary directory + os.system('rm -rf %s' % tmp_dir) msignals.display(m18n.n('restore_complete'), 'success') diff --git a/locales/en.json b/locales/en.json index ff553847f..63dedbec0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -130,14 +130,17 @@ "backup_running_hooks" : "Running backup hooks...", "backup_creating_archive" : "Creating the backup archive...", + "backup_extracting_archive" : "Extracting the backup archive...", "backup_archive_open_failed" : "Unable to open the backup archive", "backup_archive_name_unknown" : "Unknown local backup archive name", "backup_complete" : "Backup complete", - "invalid_restore_package" : "Invalid restore package", + "backup_invalid_archive" : "Invalid backup archive", + "restore_confirm_yunohost_installed" : "Do you really want to restore an already installed system? [{answers:s}]", + "restore_running_hooks" : "Running restoration hooks...", + "restore_failed" : "Unable to restore the system", "restore_complete" : "Restore complete", - "restoring_installed_system" : "You are trying to restore a backup on an already installed system, abort unless you know what you are doing!", - "unbackup_app" : "'{:s}' will NOT be saved", - "unrestore_app" : "'{:s}' will NOT be restored", + "unbackup_app" : "App '{:s}' will not be saved", + "unrestore_app" : "App '{:s}' will not be restored", "field_invalid" : "Invalid field '{:s}'", "mail_domain_unknown" : "Unknown mail address domain '{:s}'", diff --git a/locales/fr.json b/locales/fr.json index e922c79b3..6eb5ef411 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -128,9 +128,19 @@ "packages_upgrade_failed" : "Impossible de mettre à jour tous les paquets", "system_upgraded" : "Système mis à jour avec succès", + "backup_running_hooks" : "Exécution des scripts de sauvegarde...", + "backup_creating_archive" : "Création de l'archive de sauvegarde...", + "backup_extracting_archive" : "Extraction de l'archive de sauvegarde...", + "backup_archive_open_failed" : "Impossible d'ouvrir l'archive de sauvegarde", + "backup_archive_name_unknown" : "Nom d'archive de sauvegarde locale inconnu", "backup_complete" : "Sauvegarde terminée", - "invalid_restore_package" : "Dossier de restauration invalide", + "backup_invalid_archive" : "Archive de sauvegarde incorrecte", + "restore_confirm_yunohost_installed" : "Voulez-vous vraiment restaurer un système déjà installé ? [{answers:s}]", + "restore_running_hooks" : "Exécution des scripts de restauration...", + "restore_failed" : "Impossible de restaurer le système", "restore_complete" : "Restauration terminée", + "unbackup_app" : "L'application '{:s}' ne sera pas sauvegardée", + "unrestore_app" : "L'application '{:s}' ne sera pas restaurée", "field_invalid" : "Champ incorrect : {:s}", "mail_domain_unknown" : "Domaine '{:s}' de l'adresse mail inconnu", From 3f180d254bc0dd4a6117ca6a82900203e53a2fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 6 Nov 2014 01:11:35 +0100 Subject: [PATCH 34/58] [enh] Add version to returned dict in app_info --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index f57e1fdfb..83ec12f55 100644 --- a/app.py +++ b/app.py @@ -215,7 +215,9 @@ def app_info(app, raw=False): # FIXME: Temporarly allow undefined license 'license': app_info['manifest'].get('license', m18n.n('license_undefined')), - #TODO: Add more infos + # FIXME: Temporarly allow undefined version + 'version' : app_info['manifest'].get('version', '-'), + #TODO: Add more info } From 2090958c8a1313ce113adc3d79a0aabbc0259182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 6 Nov 2014 01:14:34 +0100 Subject: [PATCH 35/58] [enh] Add and manage a backup info file --- backup.py | 68 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/backup.py b/backup.py index 4aa1d2a0c..ca743b521 100644 --- a/backup.py +++ b/backup.py @@ -54,6 +54,12 @@ def backup_create(ignore_apps=False): timestamp = int(time.time()) tmp_dir = "%s/tmp/%s" % (backup_path, timestamp) + # Initialize backup info + info = { + 'created_at': timestamp, + 'apps': {}, + } + # Create temporary directory if os.path.isdir(tmp_dir): logger.warning("temporary directory for backup '%s' already exists", tmp_dir) @@ -69,11 +75,18 @@ def backup_create(ignore_apps=False): # Add apps backup hook if not ignore_apps: + from yunohost.app import app_info try: for app_id in os.listdir('/etc/yunohost/apps'): hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' if os.path.isfile(hook): hook_add(app_id, hook) + + # Add app info + i = app_info(app_id) + info['apps'][app_id] = { + 'version': i['version'], + } else: logger.warning("unable to find app's backup hook '%s'", hook) msignals.display(m18n.n('unbackup_app', app_id), @@ -85,7 +98,9 @@ def backup_create(ignore_apps=False): msignals.display(m18n.n('backup_running_hooks')) hook_callback('backup', [tmp_dir]) - # TODO: Add a backup info file + # Create backup info file + with open("%s/info.json" % tmp_dir, 'w') as f: + f.write(json.dumps(info)) # Create the archive msignals.display(m18n.n('backup_creating_archive')) @@ -103,7 +118,7 @@ def backup_create(ignore_apps=False): except: logger.exception("unable to open the archive '%s' for writing " \ "after creating directory '%s'", - archive_file, archive_dir) + archive_file, archives_path) tar = None else: logger.exception("unable to open the archive '%s' for writing", @@ -113,7 +128,9 @@ def backup_create(ignore_apps=False): tar.add(tmp_dir, arcname='') tar.close() - # Remove temporary directory + # Copy info file and remove temporary directory + os.system('mv %s/info.json %s/%s.info.json' % + (tmp_dir, archives_path, timestamp)) os.system('rm -rf %s' % tmp_dir) msignals.display(m18n.n('backup_complete'), 'success') @@ -153,6 +170,18 @@ def backup_restore(name, ignore_apps=False, force=False): tar.extractall(tmp_dir) tar.close() + # Retrieve backup info + try: + with open("%s/info.json" % tmp_dir, 'r') as f: + info = json.load(f) + except IOError: + logger.error("unable to retrieve backup info from '%s/info.json'", + tmp_dir) + raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) + else: + logger.info("restoring from backup '%s' created on %s", name, + time.ctime(info['created_at'])) + # Retrieve domain from the backup try: with open("%s/yunohost/current_host" % tmp_dir, 'r') as f: @@ -184,17 +213,13 @@ def backup_restore(name, ignore_apps=False, force=False): # Add apps restore hook if not ignore_apps: - try: - # TODO: Check if the app_id is part of the backup archive - for app_id in os.listdir('/etc/yunohost/apps'): - hook = '/etc/yunohost/apps/'+ app_id +'/scripts/restore' - if os.path.isfile(hook): - hook_add(app_id, hook) - else: - msignals.display(m18n.n('unrestore_app', app_id), - 'warning') - except IOError as e: - logger.info("unable to add apps restore hook: %s", str(e)) + for app_id in info['apps'].keys(): + hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id + if os.path.isfile(hook): + hook_add(app_id, hook) + logger.info("app '%s' will be restored", app_id) + else: + msignals.display(m18n.n('unrestore_app', app_id), 'warning') # Run hooks msignals.display(m18n.n('restore_running_hooks')) @@ -242,9 +267,20 @@ def backup_info(name): logger.error("no local backup archive found at '%s'", archive_file) raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown')) + info_file = "%s/%s.info.json" % (archives_path, name) + try: + with open(info_file) as f: + # Retrieve backup info + info = json.load(f) + except: + # TODO: Attempt to extract backup info file from tarball + logger.exception("unable to retrive backup info file '%s'", + info_file) + raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) + return { 'path': archive_file, - # TODO: Retrieve created_at from the info file 'created_at': time.strftime(m18n.n('format_datetime_short'), - time.gmtime(int(name))), + time.gmtime(info['created_at'])), + 'apps': info['apps'], } From fdb51ac20092f55d0f6db3a7437d0674ba36c618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 6 Nov 2014 02:41:12 +0100 Subject: [PATCH 36/58] [enh] Allow to set custom backup archive name --- actionsmap/yunohost.yml | 7 +++++++ backup.py | 14 +++++++++----- locales/en.json | 2 ++ locales/fr.json | 3 +++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 86859654d..9ba88bf8a 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -564,6 +564,13 @@ backup: configuration: lock: false arguments: + -n: + full: --name + help: Name of the backup archive + extra: + pattern: + - '^[\w\-\.]{1,30}(? Date: Thu, 6 Nov 2014 02:49:22 +0100 Subject: [PATCH 37/58] [enh] Add optionnal 'description' info to the backup archive --- actionsmap/yunohost.yml | 3 +++ backup.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 9ba88bf8a..718114e12 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -571,6 +571,9 @@ backup: pattern: - '^[\w\-\.]{1,30}(? Date: Sat, 8 Nov 2014 14:25:06 +0100 Subject: [PATCH 38/58] [fix] Return the description for current locale only in app_list --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 83ec12f55..a51612aca 100644 --- a/app.py +++ b/app.py @@ -174,7 +174,8 @@ def app_list(offset=None, limit=None, filter=None, raw=False): list_dict.append({ 'id': app_id, 'name': app_info['manifest']['name'], - 'description': app_info['manifest']['description'], + 'description': _value_for_locale( + app_info['manifest']['description']), # FIXME: Temporarly allow undefined license 'license': app_info['manifest'].get('license', m18n.n('license_undefined')), From 71a145c4f1404975af3c63b363101c59e53de638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sun, 9 Nov 2014 14:11:03 +0100 Subject: [PATCH 39/58] [fix] Use stderr instead of stdout in 'console' logging handler --- bin/yunohost | 4 ++-- bin/yunohost-api | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 8f2ab92ae..84f7d5a29 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -18,7 +18,7 @@ LOGGERS_LEVEL = 'INFO' # Handlers that will be used by loggers # - file: log to the file LOG_DIR/LOG_FILE -# - console: log to stdout +# - console: log to stderr LOGGERS_HANDLERS = ['file'] # Directory and file to be used by logging @@ -93,7 +93,7 @@ def _init_moulinette(): 'console': { 'class': 'logging.StreamHandler', 'formatter': 'simple', - 'stream': 'ext://sys.stdout', + 'stream': 'ext://sys.stderr', }, 'file': { 'class': 'logging.FileHandler', diff --git a/bin/yunohost-api b/bin/yunohost-api index 515cc8361..dda606cfb 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -18,7 +18,7 @@ LOGGERS_LEVEL = 'INFO' # Handlers that will be used by loggers # - file: log to the file LOG_DIR/LOG_FILE -# - console: log to stdout +# - console: log to stderr LOGGERS_HANDLERS = ['file'] # Directory and file to be used by logging @@ -93,7 +93,7 @@ def _init_moulinette(): 'console': { 'class': 'logging.StreamHandler', 'formatter': 'simple', - 'stream': 'ext://sys.stdout', + 'stream': 'ext://sys.stderr', }, 'file': { 'class': 'logging.handlers.WatchedFileHandler', From 03750917695959b5c445eb0195e3d91b94fc3940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sun, 9 Nov 2014 16:16:33 +0100 Subject: [PATCH 40/58] [fix] Simplify domain regex pattern and prevent upper-case letter --- actionsmap/yunohost.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 718114e12..42325c8cc 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -254,7 +254,7 @@ domain: help: Domain name to add extra: pattern: - - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$' - pattern_domain -d: full: --dyndns @@ -272,7 +272,7 @@ domain: help: Domain to delete extra: pattern: - - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$' - pattern_domain ### domain_info() @@ -1035,13 +1035,13 @@ tools: full: --old-domain extra: pattern: - - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$' - pattern_domain -n: full: --new-domain extra: pattern: - - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$' - pattern_domain ### tools_postinstall() @@ -1058,7 +1058,7 @@ tools: extra: ask: ask_main_domain pattern: - - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+[a-z]{2,}$' - pattern_domain required: True -p: From 42b1189cd11f794afff6c7f53d8717d6c372f57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sun, 16 Nov 2014 13:00:08 +0100 Subject: [PATCH 41/58] [enh] Look for and load additional YunoHost modules --- bin/yunohost | 13 ++++++++++++- bin/yunohost-api | 12 +++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/bin/yunohost b/bin/yunohost index 84f7d5a29..b24fffb77 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -123,6 +123,16 @@ def _init_moulinette(): # Initialize moulinette init(logging_config=logging, _from_source=IN_DEVEL) +def _retrieve_namespaces(): + """Return the list of namespaces to load""" + from moulinette.actionsmap import ActionsMap + ret = ['yunohost'] + for n in ActionsMap.get_namespaces(): + # Append YunoHost modules + if n.startswith('ynh_'): + ret.append(n) + return ret + # Main action ---------------------------------------------------------- @@ -146,5 +156,6 @@ if __name__ == '__main__': # Execute the action from moulinette import cli - ret = cli(['yunohost'], args, print_json=PRINT_JSON, use_cache=USE_CACHE) + ret = cli(_retrieve_namespaces(), args, + print_json=PRINT_JSON, use_cache=USE_CACHE) sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index dda606cfb..a15ba4f34 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -123,6 +123,16 @@ def _init_moulinette(): # Initialize moulinette init(logging_config=logging, _from_source=IN_DEVEL) +def _retrieve_namespaces(): + """Return the list of namespaces to load""" + from moulinette.actionsmap import ActionsMap + ret = ['yunohost'] + for n in ActionsMap.get_namespaces(): + # Append YunoHost modules + if n.startswith('ynh_'): + ret.append(n) + return ret + # Callbacks for additional routes -------------------------------------- @@ -147,7 +157,7 @@ if __name__ == '__main__': from moulinette import (api, MoulinetteError) try: # Run the server - api(['yunohost'], port=6787, + api(_retrieve_namespaces(), port=6787, routes={('GET', '/installed'): is_installed}, use_cache=USE_CACHE, use_websocket=USE_WEBSOCKET) except MoulinetteError as e: From f7b591840cd4e0027613bfb28856e1efa3d68e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Mon, 17 Nov 2014 10:59:43 +0100 Subject: [PATCH 42/58] [enh] Add 'output_directory' and 'no_compress' arguments to backup_create --- actionsmap/yunohost.yml | 7 ++ backup.py | 138 +++++++++++++++++++++++++++------------- locales/en.json | 3 + locales/fr.json | 3 + 4 files changed, 107 insertions(+), 44 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 42325c8cc..160c0b088 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -574,6 +574,13 @@ backup: -d: full: --description help: Short description of the backup + -o: + full: --output-directory + help: Output directory for the backup + -r: + full: --no-compress + help: Do not create an archive file + action: store_true --ignore-apps: help: Do not backup apps action: store_true diff --git a/backup.py b/backup.py index 577ec083e..031ab9a28 100644 --- a/backup.py +++ b/backup.py @@ -24,6 +24,7 @@ Manage backups """ import os +import re import sys import json import errno @@ -40,25 +41,79 @@ archives_path = '%s/archives' % backup_path logger = getActionLogger('yunohost.backup') -def backup_create(name=None, description=None, ignore_apps=False): +def backup_create(name=None, description=None, output_directory=None, + no_compress=False, ignore_apps=False): """ Create a backup local archive Keyword arguments: name -- Name of the backup archive description -- Short description of the backup + output_directory -- Output directory for the backup + no_compress -- Do not create an archive file ignore_apps -- Do not backup apps """ + # TODO: Add a 'clean' argument to clean output directory from yunohost.hook import hook_add from yunohost.hook import hook_callback + tmp_dir = None + + # Validate and define backup name timestamp = int(time.time()) if not name: name = str(timestamp) if name in backup_list()['archives']: - raise MoulinetteError(errno.EINVAL, m18n.n('backup_archive_name_exists')) - tmp_dir = "%s/tmp/%s" % (backup_path, name) + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_archive_name_exists')) + + # Validate additional arguments + if no_compress and not output_directory: + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_output_directory_required')) + if output_directory: + output_directory = os.path.abspath(output_directory) + + # Check for forbidden folders + if output_directory.startswith(archives_path) or \ + re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$', + output_directory): + logger.error("forbidden output directory '%'", output_directory) + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_output_directory_forbidden')) + + # Create the output directory + if not os.path.isdir(output_directory): + logger.info("creating output directory '%s'", output_directory) + os.makedirs(output_directory, 0750) + # Check that output directory is empty + elif no_compress and os.listdir(output_directory): + logger.error("not empty output directory '%'", output_directory) + raise MoulinetteError(errno.EIO, + m18n.n('backup_output_directory_not_empty')) + + # Define temporary directory + if no_compress: + tmp_dir = output_directory + else: + output_directory = archives_path + + # Create temporary directory + if not tmp_dir: + tmp_dir = "%s/tmp/%s" % (backup_path, name) + if os.path.isdir(tmp_dir): + logger.warning("temporary directory for backup '%s' already exists", + tmp_dir) + os.system('rm -rf %s' % tmp_dir) + try: + os.mkdir(tmp_dir, 0750) + except OSError: + # Create temporary directory recursively + os.makedirs(tmp_dir, 0750) + os.system('chown -hR admin: %s' % backup_path) + else: + os.system('chown -hR admin: %s' % tmp_dir) # Initialize backup info info = { @@ -67,25 +122,12 @@ def backup_create(name=None, description=None, ignore_apps=False): 'apps': {}, } - # Create temporary directory - if os.path.isdir(tmp_dir): - logger.warning("temporary directory for backup '%s' already exists", tmp_dir) - os.system('rm -rf %s' % tmp_dir) - try: - os.mkdir(tmp_dir, 0750) - except OSError: - # Create temporary directory recursively - os.makedirs(tmp_dir, 0750) - os.system('chown -hR admin: %s' % backup_path) - else: - os.system('chown -hR admin: %s' % tmp_dir) - # Add apps backup hook if not ignore_apps: from yunohost.app import app_info try: for app_id in os.listdir('/etc/yunohost/apps'): - hook = '/etc/yunohost/apps/'+ app_id +'/scripts/backup' + hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id if os.path.isfile(hook): hook_add(app_id, hook) @@ -95,7 +137,8 @@ def backup_create(name=None, description=None, ignore_apps=False): 'version': i['version'], } else: - logger.warning("unable to find app's backup hook '%s'", hook) + logger.warning("unable to find app's backup hook '%s'", + hook) msignals.display(m18n.n('unbackup_app', app_id), 'warning') except IOError as e: @@ -110,34 +153,40 @@ def backup_create(name=None, description=None, ignore_apps=False): f.write(json.dumps(info)) # Create the archive - msignals.display(m18n.n('backup_creating_archive')) - archive_file = "%s/%s.tar.gz" % (archives_path, name) - try: - tar = tarfile.open(archive_file, "w:gz") - except: - tar = None + if not no_compress: + msignals.display(m18n.n('backup_creating_archive')) + archive_file = "%s/%s.tar.gz" % (output_directory, name) + try: + tar = tarfile.open(archive_file, "w:gz") + except: + tar = None - # Create the archives directory and retry - if not os.path.isdir(archives_path): - os.mkdir(archives_path, 0750) - try: - tar = tarfile.open(archive_file, "w:gz") - except: - logger.exception("unable to open the archive '%s' for writing " \ - "after creating directory '%s'", - archive_file, archives_path) - tar = None - else: - logger.exception("unable to open the archive '%s' for writing", - archive_file) - if tar is None: - raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed')) - tar.add(tmp_dir, arcname='') - tar.close() + # Create the archives directory and retry + if not os.path.isdir(archives_path): + os.mkdir(archives_path, 0750) + try: + tar = tarfile.open(archive_file, "w:gz") + except: + logger.exception("unable to open the archive '%s' for writing " + "after creating directory '%s'", + archive_file, archives_path) + tar = None + else: + logger.exception("unable to open the archive '%s' for writing", + archive_file) + if tar is None: + raise MoulinetteError(errno.EIO, + m18n.n('backup_archive_open_failed')) + tar.add(tmp_dir, arcname='') + tar.close() - # Copy info file and remove temporary directory - os.system('mv %s/info.json %s/%s.info.json' % (tmp_dir, archives_path, name)) - os.system('rm -rf %s' % tmp_dir) + # Copy info file + os.system('mv %s/info.json %s/%s.info.json' % + (tmp_dir, archives_path, name)) + + # Clean temporary directory + if tmp_dir != output_directory: + os.system('rm -rf %s' % tmp_dir) msignals.display(m18n.n('backup_complete'), 'success') @@ -260,6 +309,7 @@ def backup_list(): return { 'archives': result } + def backup_info(name): """ Get info about a local backup archive diff --git a/locales/en.json b/locales/en.json index e93ded801..d4af179ad 100644 --- a/locales/en.json +++ b/locales/en.json @@ -128,6 +128,9 @@ "packages_upgrade_failed" : "Unable to upgrade all packages", "system_upgraded" : "System successfully upgraded", + "backup_output_directory_required" : "You must provide an output directory for the backup", + "backup_output_directory_forbidden" : "Forbidden output directory", + "backup_output_directory_not_empty" : "Output directory is not empty", "backup_running_hooks" : "Running backup hooks...", "backup_creating_archive" : "Creating the backup archive...", "backup_extracting_archive" : "Extracting the backup archive...", diff --git a/locales/fr.json b/locales/fr.json index 7e0f257fd..5942a5a6c 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -128,6 +128,9 @@ "packages_upgrade_failed" : "Impossible de mettre à jour tous les paquets", "system_upgraded" : "Système mis à jour avec succès", + "backup_output_directory_required" : "Vous devez spécifier un dossier de sortie pour la sauvegarde", + "backup_output_directory_forbidden" : "Dossier de sortie interdit", + "backup_output_directory_not_empty" : "Le dossier de sortie n'est pas vide", "backup_running_hooks" : "Exécution des scripts de sauvegarde...", "backup_creating_archive" : "Création de l'archive de sauvegarde...", "backup_extracting_archive" : "Extraction de l'archive de sauvegarde...", From 798343d83d8a2ea54d101a60d98d330c9fdbc6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sat, 22 Nov 2014 19:50:08 +0100 Subject: [PATCH 43/58] [fix] Use authenticator to change admin password without asking old one --- actionsmap/yunohost.yml | 7 ++----- tools.py | 18 +++++++----------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 160c0b088..d8521fa5a 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -1016,12 +1016,9 @@ tools: adminpw: action_help: Change admin password api: PUT /adminpw + configuration: + authenticate: all arguments: - -o: - full: --old-password - extra: - password: ask_current_admin_password - required: True -n: full: --new-password extra: diff --git a/tools.py b/tools.py index 4ed21cb2d..2aa226d53 100644 --- a/tools.py +++ b/tools.py @@ -73,26 +73,22 @@ def tools_ldapinit(auth): msignals.display(m18n.n('ldap_initialized'), 'success') -def tools_adminpw(old_password, new_password): +def tools_adminpw(auth, new_password): """ Change admin password Keyword argument: new_password - old_password """ - old_password.replace('"', '\\"') - old_password.replace('&', '\\&') - new_password.replace('"', '\\"') - new_password.replace('&', '\\&') - result = os.system('ldappasswd -h localhost -D cn=admin,dc=yunohost,dc=org -w "%s" -a "%s" -s "%s"' % (old_password, old_password, new_password)) - - if result == 0: - msignals.display(m18n.n('admin_password_changed'), 'success') - else: + try: + auth.con.passwd_s('cn=admin,dc=yunohost,dc=org', None, new_password) + except: + logger.exception('unable to change admin password') raise MoulinetteError(errno.EPERM, m18n.n('admin_password_change_failed')) + else: + msignals.display(m18n.n('admin_password_changed'), 'success') def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False): From 608c5dc531b4174c6599c361597076e52c8a386c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Sun, 23 Nov 2014 13:35:37 +0100 Subject: [PATCH 44/58] [i18n] Specify lower-case requirement for pattern_username --- locales/en.json | 2 +- locales/fr.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index d4af179ad..17c9b7bfd 100644 --- a/locales/en.json +++ b/locales/en.json @@ -169,7 +169,7 @@ "ask_new_admin_password" : "New administration password", "ask_main_domain" : "Main domain", "ask_list_to_remove" : "List to remove", - "pattern_username" : "Must be alphanumeric and underscore characters only", + "pattern_username" : "Must be lower-case alphanumeric and underscore characters only", "pattern_firstname" : "Must be a valid first name", "pattern_lastname" : "Must be a valid last name", "pattern_email" : "Must be a valid email address (e.g. someone@domain.org)", diff --git a/locales/fr.json b/locales/fr.json index 5942a5a6c..ecb0a4978 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -169,7 +169,7 @@ "ask_new_admin_password" : "Nouveau mot de passe d'administration", "ask_main_domain" : "Domaine principal", "ask_list_to_remove" : "Liste à supprimer", - "pattern_username" : "Doit être composé uniquement de caractères alphanumérique et de tiret bas", + "pattern_username" : "Doit être composé uniquement de caractères alphanumérique minuscule et de tiret bas", "pattern_firstname" : "Doit être un prénom valide", "pattern_lastname" : "Doit être un nom valide", "pattern_email" : "Doit être une adresse mail valide (ex. : someone@domain.org)", From f4472ed6e14f2c7909d8b212d94da8b921b9786c Mon Sep 17 00:00:00 2001 From: opi Date: Tue, 2 Dec 2014 09:27:48 +0100 Subject: [PATCH 45/58] [enh] Use SSL/TLS when calling dyndns API. --- dyndns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dyndns.py b/dyndns.py index 5d5257815..27650ad99 100644 --- a/dyndns.py +++ b/dyndns.py @@ -50,7 +50,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Verify if domain is available try: - if requests.get('http://%s/test/%s' % (subscribe_host, domain)).status_code != 200: + if requests.get('https://%s/test/%s' % (subscribe_host, domain)).status_code != 200: raise MoulinetteError(errno.EEXIST, m18n.n('dyndns_unavailable')) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) @@ -71,7 +71,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Send subscription try: - r = requests.post('http://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={ 'subdomain': domain }) + r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={ 'subdomain': domain }) except ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if r.status_code != 201: From cc034dbf63a906ae061f633198faadfe06d1d316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 27 Nov 2014 17:45:14 +0100 Subject: [PATCH 46/58] [enh] Add hook_list action with custom hook folder's integration --- actionsmap/yunohost.yml | 8 ++++++ hook.py | 57 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index d8521fa5a..da7f31450 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -1134,6 +1134,14 @@ hook: app: help: Scripts related to app will be removed + ### hook_list() + list: + action_help: List available hooks for an action + api: GET /hooks/ + arguments: + action: + help: Action name + ### hook_callback() callback: action_help: Execute all scripts binded to an action diff --git a/hook.py b/hook.py index 5144ea0f3..371cdbd40 100644 --- a/hook.py +++ b/hook.py @@ -32,8 +32,13 @@ import subprocess from shlex import split as arg_split from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger hook_folder = '/usr/share/yunohost/hooks/' +custom_hook_folder = '/etc/yunohost/hooks.d/' + +logger = getActionLogger('yunohost.hook') + def hook_add(app, file): """ @@ -45,11 +50,7 @@ def hook_add(app, file): """ path, filename = os.path.split(file) - if '-' in filename: - priority, action = filename.split('-') - else: - priority = '50' - action = filename + priority, action = _extract_filename_parts(filename) try: os.listdir(hook_folder + action) except OSError: os.makedirs(hook_folder + action) @@ -77,6 +78,42 @@ def hook_remove(app): except OSError: pass +def hook_list(action): + """ + List available hooks for an action + + Keyword argument: + action -- Action name + + """ + hooks = {} + + def _append_folder(folder): + for f in os.listdir(folder + action): + path = '%s%s/%s' % (folder, action, f) + priority, name = _extract_filename_parts(f) + try: + hooks[priority][name] = path + except KeyError: + hooks[priority] = {name: path} + + try: + # Append system hooks first + _append_folder(hook_folder) + except OSError: + logger.debug("system hook folder not found for action '%s' in %s", + action, hook_folder) + + try: + # Append custom hooks + _append_folder(custom_hook_folder) + except OSError: + logger.debug("custom hook folder not found for action '%s' in %s", + action, custom_hook_folder) + + return hooks + + def hook_callback(action, args=None): """ Execute all scripts binded to an action @@ -206,3 +243,13 @@ def hook_exec(file, args=None): stream.close() return returncode + + +def _extract_filename_parts(filename): + """Extract hook parts from filename""" + if '-' in filename: + priority, action = filename.split('-', 1) + else: + priority = '50' + action = filename + return priority, action From 3e1fc78786505e4f80902ad84ba92246dc5b5942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 4 Dec 2014 15:45:14 +0100 Subject: [PATCH 47/58] [enh] Choose the property to list hook by and show info in hook_list --- actionsmap/yunohost.yml | 12 +++++++++ hook.py | 60 ++++++++++++++++++++++++++++++++++------- locales/en.json | 1 + locales/fr.json | 1 + 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index da7f31450..436104f1d 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -1141,6 +1141,18 @@ hook: arguments: action: help: Action name + -l: + full: --list-by + help: Property to list hook by + choices: + - name + - priority + - folder + default: name + -i: + full: --show-info + help: Show hook information + action: store_true ### hook_callback() callback: diff --git a/hook.py b/hook.py index 371cdbd40..d71accf0f 100644 --- a/hook.py +++ b/hook.py @@ -78,40 +78,80 @@ def hook_remove(app): except OSError: pass -def hook_list(action): +def hook_list(action, list_by='name', show_info=False): """ List available hooks for an action Keyword argument: action -- Action name + list_by -- Property to list hook by + show_info -- Show hook information """ - hooks = {} + result = {} - def _append_folder(folder): + # Process the property to list hook by + if list_by == 'priority': + if show_info: + def _append_hook(d, priority, name, path): + # Use the priority as key and a dict of hooks names + # with their info as value + value = { 'path': path } + try: + d[priority][name] = value + except KeyError: + d[priority] = { name: value } + else: + def _append_hook(d, priority, name, path): + # Use the priority as key and the name as value + try: + d[priority].add(name) + except KeyError: + d[priority] = set([name]) + elif list_by == 'name' or list_by == 'folder': + if show_info: + def _append_hook(d, priority, name, path): + # Use the name as key and hook info as value + d[name] = { 'priority': priority, 'path': path } + else: + if list_by == 'name': + result = set() + def _append_hook(d, priority, name, path): + # Add only the name + d.add(name) + else: + raise MoulinetteError(errno.EINVAL, m18n.n('hook_list_by_invalid')) + + def _append_folder(d, folder): + # Iterate over and add hook from a folder for f in os.listdir(folder + action): path = '%s%s/%s' % (folder, action, f) priority, name = _extract_filename_parts(f) - try: - hooks[priority][name] = path - except KeyError: - hooks[priority] = {name: path} + _append_hook(d, priority, name, path) try: # Append system hooks first - _append_folder(hook_folder) + if list_by == 'folder': + result['system'] = dict() if show_info else set() + _append_folder(result['system'], hook_folder) + else: + _append_folder(result, hook_folder) except OSError: logger.debug("system hook folder not found for action '%s' in %s", action, hook_folder) try: # Append custom hooks - _append_folder(custom_hook_folder) + if list_by == 'folder': + result['custom'] = dict() if show_info else set() + _append_folder(result['custom'], custom_hook_folder) + else: + _append_folder(result, custom_hook_folder) except OSError: logger.debug("custom hook folder not found for action '%s' in %s", action, custom_hook_folder) - return hooks + return { 'hooks': result } def hook_callback(action, args=None): diff --git a/locales/en.json b/locales/en.json index 17c9b7bfd..e5327fda0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -77,6 +77,7 @@ "upnp_disabled" : "uPnP successfully disabled", "firewall_reloaded" : "Firewall successfully reloaded", + "hook_list_by_invalid" : "Invalid property to list hook by", "hook_choice_invalid" : "Invalid choice '{:s}'", "hook_argument_missing" : "Missing argument '{:s}'", diff --git a/locales/fr.json b/locales/fr.json index ecb0a4978..7c94f9584 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -77,6 +77,7 @@ "upnp_disabled" : "uPnP désactivé avec succès", "firewall_reloaded" : "Pare-feu rechargé avec succès", + "hook_list_by_invalid" : "Propriété pour lister les scripts incorrecte", "hook_choice_invalid" : "Choix incorrect : '{:s}'", "hook_argument_missing" : "Argument manquant : '{:s}'", From b977f11edad58a0e7e73c5249771ce59aea260ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 4 Dec 2014 15:46:55 +0100 Subject: [PATCH 48/58] [fix] Import and define logger in tools module --- tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools.py b/tools.py index 2aa226d53..1e4fce2f4 100644 --- a/tools.py +++ b/tools.py @@ -36,9 +36,13 @@ import apt import apt.progress from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger apps_setting_path= '/etc/yunohost/apps/' +logger = getActionLogger('yunohost.tools') + + def tools_ldapinit(auth): """ YunoHost LDAP initialization From 44df7f6351a5fb2ccd1efd0358a654265499d1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 4 Dec 2014 18:45:20 +0100 Subject: [PATCH 49/58] [enh] Use hook_list and be more verbose in hook_callback --- hook.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/hook.py b/hook.py index d71accf0f..127a7a2b0 100644 --- a/hook.py +++ b/hook.py @@ -163,18 +163,31 @@ def hook_callback(action, args=None): args -- Ordered list of arguments to pass to the script """ - try: os.listdir(hook_folder + action) - except OSError: pass - else: - if args is None: - args = [] - elif not isinstance(args, list): - args = [args] + result = { 'succeed': list(), 'failed': list() } - for hook in sorted(os.listdir(hook_folder + action)): + # Retrieve hooks by priority + hooks = hook_list(action, list_by='priority', show_info=True)['hooks'] + if not hooks: + return result + + # Format arguments + if args is None: + args = [] + elif not isinstance(args, list): + args = [args] + + # Iterate over hooks and execute them + for priority in sorted(hooks): + for name, info in iter(hooks[priority].items()): try: - hook_exec(file=hook_folder + action +'/'+ hook, args=args) - except: pass + hook_exec(info['path'], args=args) + except: + logger.exception("error while executing hook '%s'", + info['path']) + result['failed'].append(name) + else: + result['succeed'].append(name) + return result def hook_check(file): From dcd4b925dec2e971c9660cf6fe55da1118bcfe08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 4 Dec 2014 18:47:02 +0100 Subject: [PATCH 50/58] [enh] Use custom hook folder in hook_add and hook_remove --- hook.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hook.py b/hook.py index 127a7a2b0..5bb67a89f 100644 --- a/hook.py +++ b/hook.py @@ -52,10 +52,10 @@ def hook_add(app, file): path, filename = os.path.split(file) priority, action = _extract_filename_parts(filename) - try: os.listdir(hook_folder + action) - except OSError: os.makedirs(hook_folder + action) + try: os.listdir(custom_hook_folder + action) + except OSError: os.makedirs(custom_hook_folder + action) - finalpath = hook_folder + action +'/'+ priority +'-'+ app + finalpath = custom_hook_folder + action +'/'+ priority +'-'+ app os.system('cp %s %s' % (file, finalpath)) os.system('chown -hR admin: %s' % hook_folder) @@ -71,10 +71,10 @@ def hook_remove(app): """ try: - for action in os.listdir(hook_folder): - for script in os.listdir(hook_folder + action): + for action in os.listdir(custom_hook_folder): + for script in os.listdir(custom_hook_folder + action): if script.endswith(app): - os.remove(hook_folder + action +'/'+ script) + os.remove(custom_hook_folder + action +'/'+ script) except OSError: pass From d890307ab1f02cc10fa0426b96d57c1f75ecad57 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 7 Dec 2014 19:26:44 +0100 Subject: [PATCH 51/58] [enh] Use dnsmasq instead of bind9 --- domain.py | 65 +++++++++++++++---------------------------------- locales/en.json | 1 + locales/fr.json | 1 + 3 files changed, 22 insertions(+), 45 deletions(-) diff --git a/domain.py b/domain.py index 76ed1c5a2..b1b1dd169 100644 --- a/domain.py +++ b/domain.py @@ -147,46 +147,33 @@ def domain_add(auth, domain, dyndns=False): attr_dict['virtualdomain'] = domain + dnsmasq_config_path='/etc/dnsmasq.d' try: - with open('/var/lib/bind/%s.zone' % domain) as f: pass + os.listdir(dnsmasq_config_path) + except OSError: + msignals.display(m18n.n('dnsmasq_isnt_installed'), + 'warning') + os.makedirs(dnsmasq_config_path) + + try: + with open('%s/%s' % (dnsmasq_config_path, domain)) as f: pass except IOError as e: zone_lines = [ - '$TTL 38400', - '%s. IN SOA ns.%s. root.%s. %s 10800 3600 604800 38400' % (domain, domain, domain, timestamp), - '%s. IN NS ns.%s.' % (domain, domain), - '%s. IN A %s' % (domain, ip), - '%s. IN MX 5 %s.' % (domain, domain), - '%s. IN TXT "v=spf1 mx a -all"' % domain, - 'ns.%s. IN A %s' % (domain, ip), - '_xmpp-client._tcp.%s. IN SRV 0 5 5222 %s.' % (domain, domain), - '_xmpp-server._tcp.%s. IN SRV 0 5 5269 %s.' % (domain, domain), - '_jabber._tcp.%s. IN SRV 0 5 5269 %s.' % (domain, domain), + 'address=/%s/%s' % (domain, ip), + 'txt-record=%s,"v=spf1 mx a -all"' % domain, + 'mx-host=%s,%s,5' % (domain, domain), + 'srv-host=_xmpp-client._tcp.%s,%s,5222,0,5' % (domain, domain), + 'srv-host=_xmpp-server._tcp.%s,%s,5269,0,5' % (domain, domain), + 'srv-host=_jabber._tcp.%s,%s,5269,0,5' % (domain, domain), ] - with open('/var/lib/bind/%s.zone' % domain, 'w') as zone: + with open('%s/%s' % (dnsmasq_config_path, domain), 'w') as zone: for line in zone_lines: zone.write(line + '\n') - - os.system('chown bind /var/lib/bind/%s.zone' % domain) + os.system('service dnsmasq restart') else: - raise MoulinetteError(errno.EEXIST, - m18n.n('domain_zone_exists')) - - conf_lines = [ - 'zone "%s" {' % domain, - ' type master;', - ' file "/var/lib/bind/%s.zone";' % domain, - ' allow-transfer {', - ' 127.0.0.1;', - ' localnets;', - ' };', - '};' - ] - with open('/etc/bind/named.conf.local', 'a') as conf: - for line in conf_lines: - conf.write(line + '\n') - - os.system('service bind9 reload') + msignals.display(m18n.n('domain_zone_exists'), + 'warning') # XMPP try: @@ -265,7 +252,7 @@ def domain_remove(auth, domain, force=False): if auth.remove('virtualdomain=' + domain + ',ou=domains') or force: command_list = [ 'rm -rf /etc/yunohost/certs/%s' % domain, - 'rm -f /var/lib/bind/%s.zone' % domain, + 'rm -f /etc/dnsmasq.d/%s' % domain, 'rm -rf /var/lib/metronome/%s' % domain.replace('.', '%2e'), 'rm -f /etc/metronome/conf.d/%s.cfg.lua' % domain, 'rm -rf /etc/nginx/conf.d/%s.d' % domain, @@ -275,18 +262,6 @@ def domain_remove(auth, domain, force=False): if os.system(command) != 0: msignals.display(m18n.n('path_removal_failed', command[7:]), 'warning') - with open('/etc/bind/named.conf.local', 'r') as conf: - conf_lines = conf.readlines() - with open('/etc/bind/named.conf.local', 'w') as conf: - in_block = False - for line in conf_lines: - if re.search(r'^zone "%s' % domain, line): - in_block = True - if in_block: - if re.search(r'^};$', line): - in_block = False - else: - conf.write(line) else: raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed')) diff --git a/locales/en.json b/locales/en.json index e5327fda0..d42bf4e6d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -47,6 +47,7 @@ "domain_dyndns_root_unknown" : "Unknown DynDNS root domain", "domain_cert_gen_failed" : "Unable to generate certificate", "domain_exists" : "Domain already exists", + "dnsmasq_isnt_installed" : "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'" "domain_zone_exists" : "DNS zone file already exists", "domain_zone_not_found" : "DNS zone file not found for domain {:s}", "domain_creation_failed" : "Unable to create domain", diff --git a/locales/fr.json b/locales/fr.json index 7c94f9584..f206e7eec 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -47,6 +47,7 @@ "domain_dyndns_root_unknown" : "Domaine DynDNS principal inconnu", "domain_cert_gen_failed" : "Impossible de générer le certificat", "domain_exists" : "Le domaine existe déjà", + "dnsmasq_isnt_installed" : "dnsmasq ne semble pas être installé, veuillez lancer 'apt-get remove bind9 && apt-get install dnsmasq'" "domain_zone_exists" : "Le fichier de zone DNS existe déjà", "domain_zone_not_found" : "Fichier de zone DNS introuvable pour le domaine {:s}", "domain_creation_failed" : "Impossible de créer le domaine", From 908ad8929a4d5e98e0c472f8a4710a425cac0ac0 Mon Sep 17 00:00:00 2001 From: kload Date: Sun, 7 Dec 2014 21:44:39 +0100 Subject: [PATCH 52/58] [enh] Use dnsmasq instead of bind9 --- domain.py | 2 +- locales/en.json | 2 +- locales/fr.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/domain.py b/domain.py index b1b1dd169..87cd4f6ae 100644 --- a/domain.py +++ b/domain.py @@ -267,7 +267,7 @@ def domain_remove(auth, domain, force=False): os.system('yunohost app ssowatconf > /dev/null 2>&1') os.system('service nginx reload') - os.system('service bind9 reload') + os.system('service dnsmasq restart') os.system('service metronome restart') msignals.display(m18n.n('domain_deleted'), 'success') diff --git a/locales/en.json b/locales/en.json index d42bf4e6d..14283c2c6 100644 --- a/locales/en.json +++ b/locales/en.json @@ -47,7 +47,7 @@ "domain_dyndns_root_unknown" : "Unknown DynDNS root domain", "domain_cert_gen_failed" : "Unable to generate certificate", "domain_exists" : "Domain already exists", - "dnsmasq_isnt_installed" : "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'" + "dnsmasq_isnt_installed" : "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'", "domain_zone_exists" : "DNS zone file already exists", "domain_zone_not_found" : "DNS zone file not found for domain {:s}", "domain_creation_failed" : "Unable to create domain", diff --git a/locales/fr.json b/locales/fr.json index f206e7eec..2280a3de3 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -47,7 +47,7 @@ "domain_dyndns_root_unknown" : "Domaine DynDNS principal inconnu", "domain_cert_gen_failed" : "Impossible de générer le certificat", "domain_exists" : "Le domaine existe déjà", - "dnsmasq_isnt_installed" : "dnsmasq ne semble pas être installé, veuillez lancer 'apt-get remove bind9 && apt-get install dnsmasq'" + "dnsmasq_isnt_installed" : "dnsmasq ne semble pas être installé, veuillez lancer 'apt-get remove bind9 && apt-get install dnsmasq'", "domain_zone_exists" : "Le fichier de zone DNS existe déjà", "domain_zone_not_found" : "Fichier de zone DNS introuvable pour le domaine {:s}", "domain_creation_failed" : "Impossible de créer le domaine", From ba023199154a47b6a40558a0dcdc0baf3c6eaee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Wed, 10 Dec 2014 23:07:01 +0100 Subject: [PATCH 53/58] [fix] List hooks with the same name but another priority in hook_list --- hook.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hook.py b/hook.py index 5bb67a89f..94b5e5cb3 100644 --- a/hook.py +++ b/hook.py @@ -111,8 +111,19 @@ def hook_list(action, list_by='name', show_info=False): elif list_by == 'name' or list_by == 'folder': if show_info: def _append_hook(d, priority, name, path): - # Use the name as key and hook info as value - d[name] = { 'priority': priority, 'path': path } + # Use the name as key and a list of hooks info - the + # executed ones with this name - as value + l = d.get(name, list()) + for h in l: + # Only one priority for the hook is accepted + if h['priority'] == priority: + # Custom hooks overwrite system ones and they + # are appended at the end - so overwite it + if h['path'] != path: + h['path'] = path + return + l.append({ 'priority': priority, 'path': path }) + d[name] = l else: if list_by == 'name': result = set() From d1b31d5f33ca6b59f1e703e21ddbd012bc9652f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Fri, 12 Dec 2014 12:10:45 +0100 Subject: [PATCH 54/58] [enh] Allow to select hooks names to execute in hook_callback --- actionsmap/yunohost.yml | 4 ++++ app.py | 6 +++--- backup.py | 4 ++-- firewall.py | 3 ++- hook.py | 35 ++++++++++++++++++++++++++++------- locales/en.json | 1 + locales/fr.json | 1 + user.py | 3 ++- 8 files changed, 43 insertions(+), 14 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 436104f1d..55d973a69 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -1161,6 +1161,10 @@ hook: arguments: action: help: Action name + -n: + full: --hooks + help: List of hooks names to execute + nargs: '*' -a: full: --args help: Ordered list of arguments to pass to the script diff --git a/app.py b/app.py index a51612aca..f0049b1b7 100644 --- a/app.py +++ b/app.py @@ -591,7 +591,7 @@ def app_addaccess(auth, apps, users): new_users = new_users +','+ allowed_user app_setting(app, 'allowed_users', new_users.strip()) - hook_callback('post_app_addaccess', [app, new_users]) + hook_callback('post_app_addaccess', args=[app, new_users]) app_ssowatconf(auth) @@ -644,7 +644,7 @@ def app_removeaccess(auth, apps, users): new_users=new_users+','+user['username'] app_setting(app, 'allowed_users', new_users.strip()) - hook_callback('post_app_removeaccess', [app, new_users]) + hook_callback('post_app_removeaccess', args=[app, new_users]) app_ssowatconf(auth) @@ -677,7 +677,7 @@ def app_clearaccess(auth, apps): if 'allowed_users' in app_settings: app_setting(app, 'allowed_users', delete=True) - hook_callback('post_app_clearaccess', [app]) + hook_callback('post_app_clearaccess', args=[app]) app_ssowatconf(auth) diff --git a/backup.py b/backup.py index 031ab9a28..874a34e52 100644 --- a/backup.py +++ b/backup.py @@ -146,7 +146,7 @@ def backup_create(name=None, description=None, output_directory=None, # Run hooks msignals.display(m18n.n('backup_running_hooks')) - hook_callback('backup', [tmp_dir]) + hook_callback('backup', args=[tmp_dir]) # Create backup info file with open("%s/info.json" % tmp_dir, 'w') as f: @@ -278,7 +278,7 @@ def backup_restore(name, ignore_apps=False, force=False): # Run hooks msignals.display(m18n.n('restore_running_hooks')) - hook_callback('restore', [tmp_dir]) + hook_callback('restore', args=[tmp_dir]) # Remove temporary directory os.system('rm -rf %s' % tmp_dir) diff --git a/firewall.py b/firewall.py index 9abea8ec2..08b63b3bc 100644 --- a/firewall.py +++ b/firewall.py @@ -186,7 +186,8 @@ def firewall_reload(): for port in firewall['ipv4'][protocol]: os.system("iptables -A INPUT -p %s --dport %d -j ACCEPT" % (protocol, port)) - hook_callback('post_iptable_rules', [upnp, os.path.exists("/proc/net/if_inet6")]) + hook_callback('post_iptable_rules', + args=[upnp, os.path.exists("/proc/net/if_inet6")]) os.system("iptables -A INPUT -i lo -j ACCEPT") os.system("iptables -A INPUT -p icmp -j ACCEPT") diff --git a/hook.py b/hook.py index 94b5e5cb3..4c7595de2 100644 --- a/hook.py +++ b/hook.py @@ -165,20 +165,40 @@ def hook_list(action, list_by='name', show_info=False): return { 'hooks': result } -def hook_callback(action, args=None): +def hook_callback(action, hooks=[], args=None): """ Execute all scripts binded to an action Keyword argument: action -- Action name + hooks -- List of hooks names to execute args -- Ordered list of arguments to pass to the script """ result = { 'succeed': list(), 'failed': list() } + hooks_dict = {} - # Retrieve hooks by priority - hooks = hook_list(action, list_by='priority', show_info=True)['hooks'] + # Retrieve hooks if not hooks: + hooks_dict = hook_list(action, list_by='priority', + show_info=True)['hooks'] + else: + hooks_names = hook_list(action, list_by='name', + show_info=True)['hooks'] + # Iterate over given hooks names list + for n in hooks: + try: + hl = hooks_names[n] + except KeyError: + raise MoulinetteError(errno.EINVAL, + m18n.n('hook_name_unknown', n)) + # Iterate over hooks with this name + for h in hl: + # Update hooks dict + d = hooks_dict.get(h['priority'], dict()) + d.update({ n: { 'path': h['path'] }}) + hooks_dict[h['priority']] = d + if not hooks_dict: return result # Format arguments @@ -188,16 +208,17 @@ def hook_callback(action, args=None): args = [args] # Iterate over hooks and execute them - for priority in sorted(hooks): - for name, info in iter(hooks[priority].items()): + for priority in sorted(hooks_dict): + for name, info in iter(hooks_dict[priority].items()): + filename = '%s-%s' % (priority, name) try: hook_exec(info['path'], args=args) except: logger.exception("error while executing hook '%s'", info['path']) - result['failed'].append(name) + result['failed'].append(filename) else: - result['succeed'].append(name) + result['succeed'].append(filename) return result diff --git a/locales/en.json b/locales/en.json index 14283c2c6..ae3229f3f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -79,6 +79,7 @@ "firewall_reloaded" : "Firewall successfully reloaded", "hook_list_by_invalid" : "Invalid property to list hook by", + "hook_name_unknown" : "Unknown hook name '{:s}'", "hook_choice_invalid" : "Invalid choice '{:s}'", "hook_argument_missing" : "Missing argument '{:s}'", diff --git a/locales/fr.json b/locales/fr.json index 2280a3de3..827ff34d1 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -79,6 +79,7 @@ "firewall_reloaded" : "Pare-feu rechargé avec succès", "hook_list_by_invalid" : "Propriété pour lister les scripts incorrecte", + "hook_name_unknown" : "Nom de script '{:s}' inconnu", "hook_choice_invalid" : "Choix incorrect : '{:s}'", "hook_argument_missing" : "Argument manquant : '{:s}'", diff --git a/user.py b/user.py index 15ac62d2d..ed922a322 100644 --- a/user.py +++ b/user.py @@ -186,7 +186,8 @@ def user_create(auth, username, firstname, lastname, mail, password): app_ssowatconf(auth) #TODO: Send a welcome mail to user msignals.display(m18n.n('user_created'), 'success') - hook_callback('post_user_create', [username, mail, password, firstname, lastname]) + hook_callback('post_user_create', + args=[username, mail, password, firstname, lastname]) return { 'fullname' : fullname, 'username' : username, 'mail' : mail } From 02a701bcb6e512dfc4930908c5f56e7b69592ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Fri, 12 Dec 2014 12:25:37 +0100 Subject: [PATCH 55/58] [enh] Allow to select hooks names in backup_create/restore too --- actionsmap/yunohost.yml | 6 ++++++ backup.py | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/actionsmap/yunohost.yml b/actionsmap/yunohost.yml index 55d973a69..d73157433 100644 --- a/actionsmap/yunohost.yml +++ b/actionsmap/yunohost.yml @@ -581,6 +581,9 @@ backup: full: --no-compress help: Do not create an archive file action: store_true + --hooks: + help: List of backup hooks names to execute + nargs: '*' --ignore-apps: help: Do not backup apps action: store_true @@ -594,6 +597,9 @@ backup: arguments: name: help: Name of the local backup archive + --hooks: + help: List of restauration hooks names to execute + nargs: '*' --ignore-apps: help: Do not restore apps action: store_true diff --git a/backup.py b/backup.py index 874a34e52..2270f16e7 100644 --- a/backup.py +++ b/backup.py @@ -42,7 +42,7 @@ logger = getActionLogger('yunohost.backup') def backup_create(name=None, description=None, output_directory=None, - no_compress=False, ignore_apps=False): + no_compress=False, hooks=[], ignore_apps=False): """ Create a backup local archive @@ -51,6 +51,7 @@ def backup_create(name=None, description=None, output_directory=None, description -- Short description of the backup output_directory -- Output directory for the backup no_compress -- Do not create an archive file + hooks -- List of backup hooks names to execute ignore_apps -- Do not backup apps """ @@ -146,7 +147,7 @@ def backup_create(name=None, description=None, output_directory=None, # Run hooks msignals.display(m18n.n('backup_running_hooks')) - hook_callback('backup', args=[tmp_dir]) + hook_callback('backup', hooks, args=[tmp_dir]) # Create backup info file with open("%s/info.json" % tmp_dir, 'w') as f: @@ -191,12 +192,13 @@ def backup_create(name=None, description=None, output_directory=None, msignals.display(m18n.n('backup_complete'), 'success') -def backup_restore(name, ignore_apps=False, force=False): +def backup_restore(name, hooks=[], ignore_apps=False, force=False): """ Restore from a local backup archive Keyword argument: name -- Name of the local backup archive + hooks -- List of restoration hooks names to execute ignore_apps -- Do not restore apps force -- Force restauration on an already installed system @@ -278,7 +280,7 @@ def backup_restore(name, ignore_apps=False, force=False): # Run hooks msignals.display(m18n.n('restore_running_hooks')) - hook_callback('restore', args=[tmp_dir]) + hook_callback('restore', hooks, args=[tmp_dir]) # Remove temporary directory os.system('rm -rf %s' % tmp_dir) From 5339a281603cc8eaaad513859a0adf3b2437a6a5 Mon Sep 17 00:00:00 2001 From: abeudin Date: Sat, 13 Dec 2014 14:26:54 +0100 Subject: [PATCH 56/58] [fix] upgrade without app and argument -u --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index f0049b1b7..6b9f18a3f 100644 --- a/app.py +++ b/app.py @@ -284,7 +284,7 @@ def app_upgrade(auth, app=[], url=None, file=None): upgraded_apps = [] # If no app is specified, upgrade all apps - if not app: + if not app and url == None: app = os.listdir(apps_setting_path) elif not isinstance(app, list): app = [ app ] From 805ceb79beea66481e77c733289e76a99f8df777 Mon Sep 17 00:00:00 2001 From: abeudin Date: Sat, 13 Dec 2014 16:32:26 +0100 Subject: [PATCH 57/58] Update app.py --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 6b9f18a3f..8e2c0f17b 100644 --- a/app.py +++ b/app.py @@ -284,7 +284,7 @@ def app_upgrade(auth, app=[], url=None, file=None): upgraded_apps = [] # If no app is specified, upgrade all apps - if not app and url == None: + if not app and (not url or file): app = os.listdir(apps_setting_path) elif not isinstance(app, list): app = [ app ] From 3a7294ed029b7d2b8375cc8fcf6a5d3ee097f08d Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Sat, 13 Dec 2014 22:14:45 +0100 Subject: [PATCH 58/58] Update app.py --- app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 8e2c0f17b..262fd5711 100644 --- a/app.py +++ b/app.py @@ -284,8 +284,9 @@ def app_upgrade(auth, app=[], url=None, file=None): upgraded_apps = [] # If no app is specified, upgrade all apps - if not app and (not url or file): - app = os.listdir(apps_setting_path) + if not app: + if (not url and not file): + app = os.listdir(apps_setting_path) elif not isinstance(app, list): app = [ app ]