[enh] Adapt yunohost_tools + backport update/upgrade function from prerefactoring branch

data/actionsmap/yunohost.yml

@@ -909,6 +909,8 @@ tools:
         ldapinit:
             action_help: YunoHost LDAP initialization
             api: POST /ldap
+            configuration:
+                authenticate: all
 
         ### tools_adminpw()
         adminpw:
@@ -928,6 +930,8 @@ tools:
         maindomain:
             action_help: Main domain change tool
             api: PUT /domain/main
+            configuration:
+                authenticate: all
             arguments:
                 -o:
                     full: --old-domain
@@ -971,22 +975,26 @@ tools:
         ### tools_update()
         update:
             action_help: YunoHost update
-            api: POST /update
+            api: PUT /update
-
+            arguments:
-        ### tools_changelog()
+                --ignore-apps:
-        changelog:
+                    help: Ignore apps cache update and changelog
-            action_help: YunoHost changelog
+                    action: store_true
-            api: POST /changelog
+                --ignore-packages:
+                    help: Ignore APT cache update and changelog
+                    action: store_true
 
         ### tools_upgrade()
         upgrade:
             action_help: YunoHost upgrade
-            api: POST /upgrade
+            api: PUT /upgrade
-
+            arguments:
-        ### tools_upgradelog()
+                --ignore-apps:
-        upgradelog:
+                    help: Ignore apps upgrade
-            action_help: Show dpkg log
+                    action: store_true
-            api: POST /upgradelog
+                --ignore-packages:
+                    help: Ignore APT packages upgrade
+                    action: store_true
 
 
 #############################

lib/yunohost/tools.py

@@ -23,60 +23,53 @@
 
     Specific tools
 """
-import logging
-logging.warning('the module yunohost.tools has not been revisited and updated yet')
-
 import os
 import sys
 import yaml
 import re
 import getpass
-import subprocess
 import requests
 import json
-from subprocess import Popen, PIPE
+import apt
-from domain import domain_add, domain_list
+import apt.progress
-from dyndns import dyndns_subscribe
-from backup import backup_init
-from app import app_ssowatconf
 
-from moulinette.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg
+from moulinette.helpers import validate
+from moulinette.core import MoulinetteError
 
+apps_setting_path= '/etc/yunohost/apps/'
 
-def tools_ldapinit(password=None):
+def tools_ldapinit(auth):
     """
     YunoHost LDAP initialization
 
 
     """
-    with YunoHostLDAP() as yldap:
+    with open('ldap_scheme.yml') as f:
+        ldap_map = yaml.load(f)
 
-        with open('ldap_scheme.yml') as f:
+    for rdn, attr_dict in ldap_map['parents'].items():
-            ldap_map = yaml.load(f)
+        try: auth.add(rdn, attr_dict)
+        except: pass
 
-        for rdn, attr_dict in ldap_map['parents'].items():
+    for rdn, attr_dict in ldap_map['children'].items():
-            try: yldap.add(rdn, attr_dict)
+        try: auth.add(rdn, attr_dict)
-            except: pass
+        except: pass
 
-        for rdn, attr_dict in ldap_map['children'].items():
+    admin_dict = {
-            try: yldap.add(rdn, attr_dict)
+        'cn': 'admin',
-            except: pass
+        'uid': 'admin',
+        'description': 'LDAP Administrator',
+        'gidNumber': '1007',
+        'uidNumber': '1007',
+        'homeDirectory': '/home/admin',
+        'loginShell': '/bin/bash',
+        'objectClass': ['organizationalRole', 'posixAccount', 'simpleSecurityObject'],
+        'userPassword': 'yunohost'
+    }
 
-        admin_dict = {
+    auth.update('cn=admin', admin_dict)
-            'cn': 'admin',
-            'uid': 'admin',
-            'description': 'LDAP Administrator',
-            'gidNumber': '1007',
-            'uidNumber': '1007',
-            'homeDirectory': '/home/admin',
-            'loginShell': '/bin/bash',
-            'objectClass': ['organizationalRole', 'posixAccount', 'simpleSecurityObject'],
-            'userPassword': 'yunohost'
-        }
 
-        yldap.update('cn=admin', admin_dict)
+    msignals.display(_("LDAP has been successfully initialized"), 'success')
-
-    win_msg(_("LDAP has been successfully initialized"))
 
 
 def tools_adminpw(old_password, new_password):
@@ -90,21 +83,21 @@ def tools_adminpw(old_password, new_password):
     """
     # Validate password length
     if len(new_password) < 4:
-        raise YunoHostError(22, _("Password is too short"))
+        raise MoulinetteError(22, _("Password is too short"))
 
     old_password.replace('"', '\\"')
     old_password.replace('&', '\\&')
     new_password.replace('"', '\\"')
     new_password.replace('&', '\\&')
-    result = os.system('ldappasswd -h localhost -D cn=admin,dc=yunohost,dc=org -w "'+ old_password +'" -a "'+ old_password +'" -s "' + new_password + '"')
+    result = os.system('ldappasswd -h localhost -D cn=admin,dc=yunohost,dc=org -w "%s" -a "%s" -s "%s"' % (old_password, old_password, new_password))
 
     if result == 0:
-        win_msg(_("Admin password has been changed"))
+        msignals.display(_("Admin password has been changed"), 'success')
     else:
-        raise YunoHostError(22, _("Invalid password"))
+        raise MoulinetteError(22, _("Invalid password"))
 
 
-def tools_maindomain(old_domain=None, new_domain=None, dyndns=False):
+def tools_maindomain(auth, old_domain=None, new_domain=None, dyndns=False):
     """
     Main domain change tool
 
@@ -113,6 +106,8 @@ def tools_maindomain(old_domain=None, new_domain=None, dyndns=False):
         old_domain
 
     """
+    from yunohost.domain import domain_add
+    from yunohost.dyndns import dyndns_subscribe
 
     if not old_domain:
         with open('/etc/yunohost/current_host', 'r') as f:
@@ -146,15 +141,15 @@ def tools_maindomain(old_domain=None, new_domain=None, dyndns=False):
             for line in lines:
                 sources.write(re.sub(r''+ old_domain +'', new_domain, line))
 
-    domain_add([new_domain], main=True)
+    domain_add(auth, [new_domain], main=True)
 
     os.system('rm /etc/ssl/private/yunohost_key.pem')
     os.system('rm /etc/ssl/certs/yunohost_crt.pem')
 
     command_list = [
-        'ln -s /etc/yunohost/certs/'+ new_domain +'/key.pem /etc/ssl/private/yunohost_key.pem',
+        'ln -s /etc/yunohost/certs/%s/key.pem /etc/ssl/private/yunohost_key.pem' % new_domain,
-        'ln -s /etc/yunohost/certs/'+ new_domain +'/crt.pem /etc/ssl/certs/yunohost_crt.pem',
+        'ln -s /etc/yunohost/certs/%s/crt.pem /etc/ssl/certs/yunohost_crt.pem'   % new_domain,
-        'echo '+ new_domain +' > /etc/yunohost/current_host',
+        'echo %s > /etc/yunohost/current_host' % new_domain,
         'service nginx restart',
         'service metronome restart',
         'service postfix restart',
@@ -170,7 +165,7 @@ def tools_maindomain(old_domain=None, new_domain=None, dyndns=False):
 
     for command in command_list:
         if os.system(command) != 0:
-            raise YunoHostError(17, _("There were a problem during domain changing"))
+            raise MoulinetteError(17, _("There were a problem during domain changing"))
 
     if dyndns: dyndns_subscribe(domain=new_domain)
     elif len(new_domain.split('.')) >= 3:
@@ -180,7 +175,7 @@ def tools_maindomain(old_domain=None, new_domain=None, dyndns=False):
         if dyndomain in dyndomains:
             dyndns_subscribe(domain=new_domain)
 
-    win_msg(_("Main domain has been successfully changed"))
+    msignals.display(_("Main domain has been successfully changed"), 'success')
 
 
 def tools_postinstall(domain, password, dyndns=False):
@@ -193,22 +188,25 @@ def tools_postinstall(domain, password, dyndns=False):
         password -- YunoHost admin password
 
     """
+    from yunohost.backup import backup_init
+    from yunohost.app import app_ssowatconf
+
     try:
         with open('/etc/yunohost/installed') as f: pass
     except IOError:
         print('Installing YunoHost')
     else:
-        raise YunoHostError(17, _("YunoHost is already installed"))
+        raise MoulinetteError(17, _("YunoHost is already installed"))
 
     if len(domain.split('.')) >= 3:
         r = requests.get('http://dyndns.yunohost.org/domains')
         dyndomains = json.loads(r.text)
         dyndomain  = '.'.join(domain.split('.')[1:])
         if dyndomain in dyndomains:
-            if requests.get('http://dyndns.yunohost.org/test/'+ domain).status_code == 200:
+            if requests.get('http://dyndns.yunohost.org/test/%s' % domain).status_code == 200:
                 dyndns=True
             else:
-                raise YunoHostError(17, _("Domain is already taken"))
+                raise MoulinetteError(17, _("Domain is already taken"))
 
     # Create required folders
     folders_to_create = [
@@ -230,102 +228,143 @@ def tools_postinstall(domain, password, dyndns=False):
     # Create SSL CA
     ssl_dir = '/usr/share/yunohost/yunohost-config/ssl/yunoCA'
     command_list = [
-        'echo "01" > '+ ssl_dir +'/serial',
+        'echo "01" > %s/serial' % ssl_dir,
-        'rm '+ ssl_dir +'/index.txt',
+        'rm %s/index.txt'       % ssl_dir,
-        'touch '+ ssl_dir +'/index.txt',
+        'touch %s/index.txt'    % ssl_dir,
-        'cp '+ ssl_dir +'/openssl.cnf '+ ssl_dir +'/openssl.ca.cnf ',
+        'cp %s/openssl.cnf %s/openssl.ca.cnf' % (ssl_dir, ssl_dir),
-        'sed -i "s/yunohost.org/'+ domain +'/g" '+ ssl_dir +'/openssl.ca.cnf ',
+        'sed -i "s/yunohost.org/%s/g" %s/openssl.ca.cnf ' % (domain, ssl_dir),
-        'openssl req -x509 -new -config '+ ssl_dir +'/openssl.ca.cnf -days 3650 -out '+ ssl_dir +'/ca/cacert.pem -keyout '+ ssl_dir +'/ca/cakey.pem -nodes -batch',
+        'openssl req -x509 -new -config %s/openssl.ca.cnf -days 3650 -out %s/ca/cacert.pem -keyout %s/ca/cakey.pem -nodes -batch' % (ssl_dir, ssl_dir, ssl_dir),
-        'cp '+ ssl_dir +'/ca/cacert.pem /etc/ssl/certs/ca-yunohost_crt.pem',
+        'cp %s/ca/cacert.pem /etc/ssl/certs/ca-yunohost_crt.pem' % ssl_dir,
         'update-ca-certificates'
     ]
 
     for command in command_list:
         if os.system(command) != 0:
-            raise YunoHostError(17, _("There were a problem during CA creation"))
+            raise MoulinetteError(17, _("There were a problem during CA creation"))
 
-    with YunoHostLDAP(password='yunohost') as yldap:
+    # Initialize YunoHost LDAP base
+    tools_ldapinit(auth)
 
-        # Initialize YunoHost LDAP base
+    # Initialize backup system
-        tools_ldapinit(password)
+    backup_init()
 
-        # Initialize backup system
+    # New domain config
-        backup_init()
+    tools_maindomain(auth, old_domain='yunohost.org', new_domain=domain, dyndns=dyndns)
 
-        # New domain config
+    # Generate SSOwat configuration file
-        tools_maindomain(old_domain='yunohost.org', new_domain=domain, dyndns=dyndns)
+    app_ssowatconf(auth)
 
-        # Generate SSOwat configuration file
+    # Change LDAP admin password
-        app_ssowatconf()
+    tools_adminpw(old_password='yunohost', new_password=password)
 
-        # Change LDAP admin password
+    os.system('touch /etc/yunohost/installed')
-        tools_adminpw(old_password='yunohost', new_password=password)
+    os.system('service yunohost-api restart &')
 
-        os.system('touch /etc/yunohost/installed')
+    msignals.display(_("YunoHost has been successfully configured"), 'success')
-        os.system('service yunohost-api restart &')
-
-    win_msg(_("YunoHost has been successfully configured"))
 
 
-def tools_update():
+def tools_update(ignore_apps=False, ignore_packages=False):
     """
-    Update distribution
+    Update apps & package cache, then display changelog
+
+    Keyword arguments:
+        ignore_apps -- Ignore app list update and changelog
+        ignore_packages -- Ignore apt cache update and changelog
 
     """
-    process = Popen("/usr/bin/checkupdate", stdout=PIPE)
+    from yunohost.app import app_fetchlist, app_info
-    stdout, stderr = process.communicate()
-    if process.returncode == 1:
-        win_msg( _("Not upgrade found"))
-    elif process.returncode == 2:
-        raise YunoHostError(17, _("Error during update"))
-    else:
-        return { "Update" : stdout.splitlines() }
 
+    packages = []
+    if not ignore_packages:
+        cache = apt.Cache()
+        # Update APT cache
+        if not cache.update():
+            raise MoulinetteError(1, _("An error occured during APT cache update"))
 
-def tools_changelog():
+        cache.open(None)
-    """
+        cache.upgrade(True)
-    Show Changelog
 
-    """
+        # Add changelogs to the result
-    if os.path.isfile('/tmp/yunohost/update_status'):
+        for pkg in cache.get_changes():
-        with open('/tmp/yunohost/changelog', 'r') as f:
+            packages.append({
-            read_data = f.read()
+                'name': pkg.name,
-            return { "Changelog" : read_data.splitlines() }
+                'fullname': pkg.fullname,
-    else:
+                'changelog': pkg.get_changelog()
-        raise YunoHostError(17, _("Launch update before upgrade"))
+            })
         
-
+    apps = []
-def tools_upgrade():
+    if not ignore_apps:
-    """
+        app_fetchlist()
-    Upgrade distribution
+        app_list = os.listdir(apps_setting_path)
-
+        if len(app_list) > 0:
-    """
+            for app_id in app_list:
-    if os.path.isfile('/tmp/yunohost/upgrade.run'):
+                if '__' in app_id:
-        win_msg( _("Upgrade in progress"))
+                    original_app_id = app_id[:app_id.index('__')]
-    else:
-        if os.path.isfile('/tmp/yunohost/upgrade_status'):
-            with open('/tmp/yunohost/upgrade_status', 'r') as f:
-                read_data = f.read()
-                os.system('rm /tmp/yunohost/upgrade_status')
-                if read_data.strip() == "OK":
-                    win_msg( _("YunoHost has been successfully upgraded"))
                 else:
-                    raise YunoHostError(17, _("Error during upgrade"))
+                    original_app_id = app_id
-        elif os.path.isfile('/tmp/yunohost/update_status'):
+
-            os.system('at now -f /usr/share/yunohost/upgrade')
+                current_app_dict = app_info(app_id,  raw=True)
-            win_msg( _("Upgrade in progress"))
+                new_app_dict     = app_info(original_app_id, raw=True)
-        else:
+
-            raise YunoHostError(17, _("Launch update before upgrade"))
+                # Custom app
+                if 'lastUpdate' not in new_app_dict or 'git' not in new_app_dict:
+                    continue
+
+                if (new_app_dict['lastUpdate'] > current_app_dict['lastUpdate']) \
+                      or ('update_time' not in current_app_dict['settings'] \
+                           and (new_app_dict['lastUpdate'] > current_app_dict['settings']['install_time'])) \
+                      or ('update_time' in current_app_dict['settings'] \
+                           and (new_app_dict['lastUpdate'] > current_app_dict['settings']['update_time'])):
+                    apps.append({
+                        'id': app_id,
+                        'label': current_app_dict['settings']['label']
+                    })
+    
+    if len(apps) == 0 and len(packages) == 0:
+        msignals.display(_("There is nothing to upgrade right now"), 'success')
+
+    return { 'packages': packages, 'apps': apps }
 
 
-def tools_upgradelog():
+
+def tools_upgrade(ignore_apps=False, ignore_packages=False):
     """
-    Show upgrade log
+    Update apps & package cache, then display changelog
+
+    Keyword arguments:
+        ignore_apps -- Ignore apps upgrade
+        ignore_packages -- Ignore APT packages upgrade
 
     """
-    if os.path.isfile('/tmp/yunohost/upgrade.run'):
+    from yunohost.app import app_upgrade
-        win_msg( _("Upgrade in progress"))
+
-    else:
+    if not ignore_packages:
-        with open('/tmp/yunohost/update_log', 'r') as f:
+        cache = apt.Cache()
-            read_data = f.read()
+        cache.open(None)
-            return { "DPKG LOG" : read_data.splitlines() }
+        cache.upgrade(True)
+        
+        # If API call
+        if not os.isatty(1):
+            critical_packages = ["yunohost-cli", "yunohost-admin", "yunohost-config-nginx", "ssowat", "python"]
+            for pkg in cache.get_changes():
+                if pkg.name in critical_packages:
+                    # Temporarily keep package ...
+                    pkg.mark_keep()
+                    # ... and set a hourly cron up to upgrade critical packages
+                    with open('/etc/cron.d/yunohost-upgrade', 'w+') as f:
+                        f.write('00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install '+ ' '.join(critical_packages) + ' -y && rm -f /etc/cron.d/yunohost-upgrade')
+        try:
+            # Apply APT changes
+            cache.commit(apt.progress.text.AcquireProgress(), apt.progress.base.InstallProgress())
+        except: pass
+
+    if not ignore_apps:
+        try:
+            app_upgrade()
+        except: pass
+
+    msignals.display(_("System successfully upgraded"), 'success')
+
+    # Return API logs if it is an API call
+    if not os.isatty(1):
+        return { "log": service_log('yunohost-api', number="100").values()[0] }