Merge pull request #808 from YunoHost/more-accurate-tests

More accurate tests

src/yunohost/app.py

@@ -42,7 +42,6 @@ from datetime import datetime
 from moulinette import msignals, m18n, msettings
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_json
-from moulinette.utils.filesystem import read_json, read_toml
 
 from yunohost.service import service_log, service_status, _run_service_command
 from yunohost.utils import packages

src/yunohost/tests/conftest.py

@@ -1,9 +1,32 @@
+import pytest
 import sys
 import moulinette
 
+from moulinette import m18n
+from yunohost.utils.error import YunohostError
+from contextlib import contextmanager
+
 sys.path.append("..")
 
 
+
+@contextmanager
+def message(mocker, key, **kwargs):
+    mocker.spy(m18n, "n")
+    yield
+    m18n.n.assert_any_call(key, **kwargs)
+
+
+@contextmanager
+def raiseYunohostError(mocker, key, **kwargs):
+    with pytest.raises(YunohostError) as e_info:
+        yield
+    assert e_info._excinfo[1].key == key
+    if kwargs:
+        assert e_info._excinfo[1].kwargs == kwargs
+
+
+
 def pytest_addoption(parser):
     parser.addoption("--yunodebug", action="store_true", default=False)
 

src/yunohost/tests/test_apps.py

@@ -4,6 +4,8 @@ import pytest
 import shutil
 import requests
 
+from conftest import message, raiseYunohostError
+
 from moulinette import m18n
 from moulinette.utils.filesystem import mkdir
 
@@ -113,9 +115,9 @@ def app_is_not_installed(domain, app):
 def app_is_exposed_on_http(domain, path, message_in_page):
 
     try:
-        r = requests.get("http://127.0.0.1" + path + "/", headers={"Host": domain}, timeout=10)
+        r = requests.get("http://127.0.0.1" + path + "/", headers={"Host": domain}, timeout=10, verify=False)
         return r.status_code == 200 and message_in_page in r.text
-    except Exception:
+    except Exception as e:
         return False
 
 
@@ -190,11 +192,11 @@ def test_legacy_app_install_private(secondary_domain):
     assert app_is_not_installed(secondary_domain, "legacy_app")
 
 
-def test_legacy_app_install_unknown_domain():
+def test_legacy_app_install_unknown_domain(mocker):
 
     with pytest.raises(YunohostError):
+        with message(mocker, "app_argument_invalid"):
             install_legacy_app("whatever.nope", "/legacy")
-        # TODO check error message
 
     assert app_is_not_installed("whatever.nope", "legacy_app")
 
@@ -221,55 +223,51 @@ def test_legacy_app_install_multiple_instances(secondary_domain):
     assert app_is_not_installed(secondary_domain, "legacy_app__2")
 
 
-def test_legacy_app_install_path_unavailable(secondary_domain):
+def test_legacy_app_install_path_unavailable(mocker, secondary_domain):
 
     # These will be removed in teardown
     install_legacy_app(secondary_domain, "/legacy")
 
     with pytest.raises(YunohostError):
+        with message(mocker, "app_location_unavailable"):
             install_legacy_app(secondary_domain, "/")
-        # TODO check error message
 
     assert app_is_installed(secondary_domain, "legacy_app")
     assert app_is_not_installed(secondary_domain, "legacy_app__2")
 
 
-def test_legacy_app_install_bad_args():
+def test_legacy_app_install_with_nginx_down(mocker, secondary_domain):
-
-    with pytest.raises(YunohostError):
-        install_legacy_app("this.domain.does.not.exists", "/legacy")
-
-
-def test_legacy_app_install_with_nginx_down(secondary_domain):
 
     os.system("systemctl stop nginx")
 
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, "app_action_cannot_be_ran_because_required_services_down"):
         install_legacy_app(secondary_domain, "/legacy")
 
 
-def test_legacy_app_failed_install(secondary_domain):
+def test_legacy_app_failed_install(mocker, secondary_domain):
 
     # This will conflict with the folder that the app
     # attempts to create, making the install fail
     mkdir("/var/www/legacy_app/", 0o750)
 
     with pytest.raises(YunohostError):
+        with message(mocker, 'app_install_script_failed'):
             install_legacy_app(secondary_domain, "/legacy")
-        # TODO check error message
 
     assert app_is_not_installed(secondary_domain, "legacy_app")
 
 
-def test_legacy_app_failed_remove(secondary_domain):
+def test_legacy_app_failed_remove(mocker, secondary_domain):
 
     install_legacy_app(secondary_domain, "/legacy")
 
     # The remove script runs with set -eu and attempt to remove this
     # file without -f, so will fail if it's not there ;)
     os.remove("/etc/nginx/conf.d/%s.d/%s.conf" % (secondary_domain, "legacy_app"))
-    with pytest.raises(YunohostError):
+
-        app_remove("legacy")
+    # TODO / FIXME : can't easily validate that 'app_not_properly_removed'
+    # is triggered for weird reasons ...
+    app_remove("legacy_app")
 
     #
     # Well here, we hit the classical issue where if an app removal script
@@ -286,59 +284,61 @@ def test_full_domain_app(secondary_domain):
     assert app_is_exposed_on_http(secondary_domain, "/", "This is a dummy app")
 
 
-def test_full_domain_app_with_conflicts(secondary_domain):
+def test_full_domain_app_with_conflicts(mocker, secondary_domain):
 
     install_legacy_app(secondary_domain, "/legacy")
 
-    # TODO : once #808 is merged, add test that the message raised is 'app_full_domain_unavailable'
+    with raiseYunohostError(mocker, "app_full_domain_unavailable"):
-    with pytest.raises(YunohostError):
         install_full_domain_app(secondary_domain)
 
 
-def test_systemfuckedup_during_app_install(secondary_domain):
+def test_systemfuckedup_during_app_install(mocker, secondary_domain):
 
     with pytest.raises(YunohostError):
+        with message(mocker, "app_install_failed"):
+            with message(mocker, 'app_action_broke_system'):
                 install_break_yo_system(secondary_domain, breakwhat="install")
-        os.system("nginx -t")
-        os.system("systemctl status nginx")
 
     assert app_is_not_installed(secondary_domain, "break_yo_system")
 
 
-def test_systemfuckedup_during_app_remove(secondary_domain):
+def test_systemfuckedup_during_app_remove(mocker, secondary_domain):
 
     install_break_yo_system(secondary_domain, breakwhat="remove")
 
     with pytest.raises(YunohostError):
+        with message(mocker, 'app_action_broke_system'):
+            with message(mocker, 'app_removed'):
                 app_remove("break_yo_system")
-        os.system("nginx -t")
-        os.system("systemctl status nginx")
 
     assert app_is_not_installed(secondary_domain, "break_yo_system")
 
 
-def test_systemfuckedup_during_app_install_and_remove(secondary_domain):
+def test_systemfuckedup_during_app_install_and_remove(mocker, secondary_domain):
 
     with pytest.raises(YunohostError):
+        with message(mocker, "app_install_failed"):
+            with message(mocker, 'app_action_broke_system'):
                 install_break_yo_system(secondary_domain, breakwhat="everything")
 
     assert app_is_not_installed(secondary_domain, "break_yo_system")
 
 
-def test_systemfuckedup_during_app_upgrade(secondary_domain):
+def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain):
 
     install_break_yo_system(secondary_domain, breakwhat="upgrade")
 
     with pytest.raises(YunohostError):
+        with message(mocker, 'app_action_broke_system'):
             app_upgrade("break_yo_system", file="./tests/apps/break_yo_system_ynh")
 
 
-def test_failed_multiple_app_upgrade(secondary_domain):
+def test_failed_multiple_app_upgrade(mocker, secondary_domain):
 
     install_legacy_app(secondary_domain, "/legacy")
     install_break_yo_system(secondary_domain, breakwhat="upgrade")
 
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, 'app_not_upgraded'):
         app_upgrade(["break_yo_system", "legacy_app"],
                     file={"break_yo_system": "./tests/apps/break_yo_system_ynh",
                           "legacy": "./tests/apps/legacy_app_ynh"})

src/yunohost/tests/test_backuprestore.py

@@ -2,14 +2,13 @@ import pytest
 import os
 import shutil
 import subprocess
-from mock import ANY
 
-from moulinette import m18n
+from conftest import message, raiseYunohostError
+
 from yunohost.app import app_install, app_remove, app_ssowatconf
 from yunohost.app import _is_installed
 from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete, _recursive_umount
 from yunohost.domain import _get_maindomain
-from yunohost.utils.error import YunohostError
 from yunohost.user import user_permission_list, user_create, user_list, user_delete
 from yunohost.tests.test_permission import check_LDAP_db_integrity, check_permission_for_apps
 
@@ -206,9 +205,10 @@ def add_archive_system_from_2p4():
 #
 
 
-def test_backup_only_ldap():
+def test_backup_only_ldap(mocker):
 
     # Create the backup
+    with message(mocker, "backup_created"):
         backup_create(system=["conf_ldap"], apps=None)
 
     archives = backup_list()["archives"]
@@ -222,23 +222,21 @@ def test_backup_only_ldap():
 
 def test_backup_system_part_that_does_not_exists(mocker):
 
-    mocker.spy(m18n, "n")
-
     # Create the backup
-    with pytest.raises(YunohostError):
+    with message(mocker, 'backup_hook_unknown', hook="doesnt_exist"):
-        backup_create(system=["yolol"], apps=None)
+        with raiseYunohostError(mocker, "backup_nothings_done"):
+            backup_create(system=["doesnt_exist"], apps=None)
 
-    m18n.n.assert_any_call('backup_hook_unknown', hook="yolol")
-    m18n.n.assert_any_call('backup_nothings_done')
 
 #
 # System backup and restore                                                  #
 #
 
 
-def test_backup_and_restore_all_sys():
+def test_backup_and_restore_all_sys(mocker):
 
     # Create the backup
+    with message(mocker, "backup_created"):
         backup_create(system=[], apps=None)
 
     archives = backup_list()["archives"]
@@ -255,6 +253,7 @@ def test_backup_and_restore_all_sys():
     assert not os.path.exists("/etc/ssowat/conf.json")
 
     # Restore the backup
+    with message(mocker, "restore_complete"):
         backup_restore(name=archives[0], force=True,
                        system=[], apps=None)
 
@@ -270,12 +269,14 @@ def test_backup_and_restore_all_sys():
 def test_restore_system_from_Ynh2p4(monkeypatch, mocker):
 
     # Backup current system
+    with message(mocker, "backup_created"):
         backup_create(system=[], apps=None)
     archives = backup_list()["archives"]
     assert len(archives) == 2
 
     # Restore system archive from 2.4
     try:
+        with message(mocker, "restore_complete"):
             backup_restore(name=backup_list()["archives"][1],
                            system=[],
                            apps=None,
@@ -306,13 +307,11 @@ def test_backup_script_failure_handling(monkeypatch, mocker):
     # call with monkeypatch). We also patch m18n to check later it's been called
     # with the expected error message key
     monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
-    mocker.spy(m18n, "n")
 
-    with pytest.raises(YunohostError):
+    with message(mocker,  'backup_app_failed', app='backup_recommended_app'):
+        with raiseYunohostError(mocker, 'backup_nothings_done'):
             backup_create(system=None, apps=["backup_recommended_app"])
 
-    m18n.n.assert_any_call('backup_app_failed', app='backup_recommended_app')
-
 
 @pytest.mark.with_backup_recommended_app_installed
 def test_backup_not_enough_free_space(monkeypatch, mocker):
@@ -327,26 +326,18 @@ def test_backup_not_enough_free_space(monkeypatch, mocker):
     monkeypatch.setattr("yunohost.backup.free_space_in_directory",
                         custom_free_space_in_directory)
 
-    mocker.spy(m18n, "n")
+    with raiseYunohostError(mocker, 'not_enough_disk_space'):
-
-    with pytest.raises(YunohostError):
         backup_create(system=None, apps=["backup_recommended_app"])
 
-    m18n.n.assert_any_call('not_enough_disk_space', path=ANY)
-
 
 def test_backup_app_not_installed(mocker):
 
     assert not _is_installed("wordpress")
 
-    mocker.spy(m18n, "n")
+    with message(mocker, "unbackup_app", app="wordpress"):
-
+        with raiseYunohostError(mocker, 'backup_nothings_done'):
-    with pytest.raises(YunohostError):
             backup_create(system=None, apps=["wordpress"])
 
-    m18n.n.assert_any_call("unbackup_app", app="wordpress")
-    m18n.n.assert_any_call('backup_nothings_done')
-
 
 @pytest.mark.with_backup_recommended_app_installed
 def test_backup_app_with_no_backup_script(mocker):
@@ -355,14 +346,10 @@ def test_backup_app_with_no_backup_script(mocker):
     os.system("rm %s" % backup_script)
     assert not os.path.exists(backup_script)
 
-    mocker.spy(m18n, "n")
+    with message(mocker, "backup_with_no_backup_script_for_app", app="backup_recommended_app"):
-
+        with raiseYunohostError(mocker, 'backup_nothings_done'):
-    with pytest.raises(YunohostError):
             backup_create(system=None, apps=["backup_recommended_app"])
 
-    m18n.n.assert_any_call("backup_with_no_backup_script_for_app", app="backup_recommended_app")
-    m18n.n.assert_any_call('backup_nothings_done')
-
 
 @pytest.mark.with_backup_recommended_app_installed
 def test_backup_app_with_no_restore_script(mocker):
@@ -371,20 +358,18 @@ def test_backup_app_with_no_restore_script(mocker):
     os.system("rm %s" % restore_script)
     assert not os.path.exists(restore_script)
 
-    mocker.spy(m18n, "n")
-
     # Backuping an app with no restore script will only display a warning to the
     # user...
 
+    with message(mocker, "backup_with_no_restore_script_for_app", app="backup_recommended_app"):
         backup_create(system=None, apps=["backup_recommended_app"])
 
-    m18n.n.assert_any_call("backup_with_no_restore_script_for_app", app="backup_recommended_app")
-
 
 @pytest.mark.clean_opt_dir
-def test_backup_with_different_output_directory():
+def test_backup_with_different_output_directory(mocker):
 
     # Create the backup
+    with message(mocker, "backup_created"):
         backup_create(system=["conf_ssh"], apps=None,
                       output_directory="/opt/test_backup_output_directory",
                       name="backup")
@@ -401,8 +386,10 @@ def test_backup_with_different_output_directory():
 
 
 @pytest.mark.clean_opt_dir
-def test_backup_with_no_compress():
+def test_backup_with_no_compress(mocker):
+
     # Create the backup
+    with message(mocker, "backup_created"):
         backup_create(system=["conf_nginx"], apps=None,
                       output_directory="/opt/test_backup_output_directory",
                       no_compress=True,
@@ -416,8 +403,9 @@ def test_backup_with_no_compress():
 #
 
 @pytest.mark.with_wordpress_archive_from_2p4
-def test_restore_app_wordpress_from_Ynh2p4():
+def test_restore_app_wordpress_from_Ynh2p4(mocker):
 
+    with message(mocker, "restore_complete"):
         backup_restore(system=None, name=backup_list()["archives"][0],
                        apps=["wordpress"])
 
@@ -431,16 +419,14 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker):
             raise Exception
 
     monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
-    mocker.spy(m18n, "n")
 
     assert not _is_installed("wordpress")
 
-    with pytest.raises(YunohostError):
+    with message(mocker, 'restore_app_failed', app='wordpress'):
+        with raiseYunohostError(mocker, 'restore_nothings_done'):
             backup_restore(system=None, name=backup_list()["archives"][0],
                            apps=["wordpress"])
 
-    m18n.n.assert_any_call('restore_app_failed', app='wordpress')
-    m18n.n.assert_any_call('restore_nothings_done')
     assert not _is_installed("wordpress")
 
 
@@ -452,18 +438,13 @@ def test_restore_app_not_enough_free_space(monkeypatch, mocker):
 
     monkeypatch.setattr("yunohost.backup.free_space_in_directory",
                         custom_free_space_in_directory)
-    mocker.spy(m18n, "n")
 
     assert not _is_installed("wordpress")
 
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, 'restore_not_enough_disk_space'):
         backup_restore(system=None, name=backup_list()["archives"][0],
                        apps=["wordpress"])
 
-    m18n.n.assert_any_call('restore_not_enough_disk_space',
-                           free_space=0,
-                           margin=ANY,
-                           needed_space=ANY)
     assert not _is_installed("wordpress")
 
 
@@ -473,13 +454,11 @@ def test_restore_app_not_in_backup(mocker):
     assert not _is_installed("wordpress")
     assert not _is_installed("yoloswag")
 
-    mocker.spy(m18n, "n")
+    with message(mocker, 'backup_archive_app_not_found', app="yoloswag"):
-
+        with raiseYunohostError(mocker, 'restore_nothings_done'):
-    with pytest.raises(YunohostError):
             backup_restore(system=None, name=backup_list()["archives"][0],
                            apps=["yoloswag"])
 
-    m18n.n.assert_any_call('backup_archive_app_not_found', app="yoloswag")
     assert not _is_installed("wordpress")
     assert not _is_installed("yoloswag")
 
@@ -489,38 +468,36 @@ def test_restore_app_already_installed(mocker):
 
     assert not _is_installed("wordpress")
 
+    with message(mocker, "restore_complete"):
         backup_restore(system=None, name=backup_list()["archives"][0],
                        apps=["wordpress"])
 
     assert _is_installed("wordpress")
 
-    mocker.spy(m18n, "n")
+    with message(mocker, 'restore_already_installed_app', app="wordpress"):
-    with pytest.raises(YunohostError):
+        with raiseYunohostError(mocker, 'restore_nothings_done'):
             backup_restore(system=None, name=backup_list()["archives"][0],
                            apps=["wordpress"])
 
-    m18n.n.assert_any_call('restore_already_installed_app', app="wordpress")
-    m18n.n.assert_any_call('restore_nothings_done')
-
     assert _is_installed("wordpress")
 
 
 @pytest.mark.with_legacy_app_installed
-def test_backup_and_restore_legacy_app():
+def test_backup_and_restore_legacy_app(mocker):
 
-    _test_backup_and_restore_app("legacy_app")
+    _test_backup_and_restore_app(mocker, "legacy_app")
 
 
 @pytest.mark.with_backup_recommended_app_installed
-def test_backup_and_restore_recommended_app():
+def test_backup_and_restore_recommended_app(mocker):
 
-    _test_backup_and_restore_app("backup_recommended_app")
+    _test_backup_and_restore_app(mocker, "backup_recommended_app")
 
 
 @pytest.mark.with_backup_recommended_app_installed_with_ynh_restore
-def test_backup_and_restore_with_ynh_restore():
+def test_backup_and_restore_with_ynh_restore(mocker):
 
-    _test_backup_and_restore_app("backup_recommended_app")
+    _test_backup_and_restore_app(mocker, "backup_recommended_app")
 
 @pytest.mark.with_permission_app_installed
 def test_backup_and_restore_permission_app():
@@ -552,9 +529,10 @@ def test_backup_and_restore_permission_app():
     assert res['permissions_app.dev']['allowed'] == []
 
 
-def _test_backup_and_restore_app(app):
+def _test_backup_and_restore_app(mocker, app):
 
     # Create a backup of this app
+    with message(mocker, "backup_created"):
         backup_create(system=None, apps=[app])
 
     archives = backup_list()["archives"]
@@ -571,6 +549,7 @@ def _test_backup_and_restore_app(app):
     assert app+".main" not in user_permission_list()['permissions']
 
     # Restore the app
+    with message(mocker, "restore_complete"):
         backup_restore(system=None, name=archives[0],
                        apps=[app])
 
@@ -593,13 +572,11 @@ def test_restore_archive_with_no_json(mocker):
 
     assert "badbackup" in backup_list()["archives"]
 
-    mocker.spy(m18n, "n")
+    with raiseYunohostError(mocker, 'backup_invalid_archive'):
-    with pytest.raises(YunohostError):
         backup_restore(name="badbackup", force=True)
-    m18n.n.assert_any_call('backup_invalid_archive')
 
 
-def test_backup_binds_are_readonly(monkeypatch):
+def test_backup_binds_are_readonly(mocker, monkeypatch):
 
     def custom_mount_and_backup(self, backup_manager):
         self.manager = backup_manager
@@ -620,4 +597,5 @@ def test_backup_binds_are_readonly(monkeypatch):
                         custom_mount_and_backup)
 
     # Create the backup
+    with message(mocker, "backup_created"):
         backup_create(system=[])

src/yunohost/tests/test_permission.py

@@ -1,19 +1,20 @@
 import requests
 import pytest
 
-from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map
+from conftest import message, raiseYunohostError
 
-from yunohost.user import user_list, user_info, user_create, user_delete, user_update, \
+from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map
-                          user_group_list, user_group_create, user_group_delete, user_group_update, user_group_info
+from yunohost.user import user_list, user_create, user_delete, \
+                          user_group_list, user_group_delete
 from yunohost.permission import user_permission_update, user_permission_list, user_permission_reset, \
                                 permission_create, permission_delete, permission_url
 from yunohost.domain import _get_maindomain
-from yunohost.utils.error import YunohostError
 
 # Get main domain
 maindomain = _get_maindomain()
 dummy_password = "test123Ynh"
 
+
 def clean_user_groups_permission():
     for u in user_list()['users']:
         user_delete(u)
@@ -26,6 +27,7 @@ def clean_user_groups_permission():
         if any(p.startswith(name) for name in ["wiki", "blog", "site", "permissions_app"]):
             permission_delete(p, force=True, sync_perm=False)
 
+
 def setup_function(function):
     clean_user_groups_permission()
 
@@ -35,6 +37,7 @@ def setup_function(function):
     permission_create("blog.main", sync_perm=False)
     user_permission_update("blog.main", remove="all_users", add="alice")
 
+
 def teardown_function(function):
     clean_user_groups_permission()
     try:
@@ -46,12 +49,14 @@ def teardown_function(function):
     except:
         pass
 
+
 @pytest.fixture(autouse=True)
 def check_LDAP_db_integrity_call():
     check_LDAP_db_integrity()
     yield
     check_LDAP_db_integrity()
 
+
 def check_LDAP_db_integrity():
     # Here we check that all attributes in all object are sychronized.
     # Here is the list of attributes per object:
@@ -187,6 +192,7 @@ def can_access_webpage(webpath, logged_as=None):
     # If we can't access it, we got redirected to the SSO
     return not r.url.startswith(sso_url)
 
+
 #
 # List functions
 #
@@ -208,7 +214,9 @@ def test_permission_list():
 # Create - Remove functions
 #
 
-def test_permission_create_main():
+
+def test_permission_create_main(mocker):
+    with message(mocker, "permission_created", permission="site.main"):
         permission_create("site.main")
 
     res = user_permission_list(full=True)['permissions']
@@ -217,7 +225,8 @@ def test_permission_create_main():
     assert set(res['site.main']['corresponding_users']) == set(["alice", "bob"])
 
 
-def test_permission_create_extra():
+def test_permission_create_extra(mocker):
+    with message(mocker, "permission_created", permission="site.test"):
         permission_create("site.test")
 
     res = user_permission_list(full=True)['permissions']
@@ -235,7 +244,8 @@ def test_permission_create_with_allowed():
     assert res['site.test']['allowed'] == ["alice"]
 
 
-def test_permission_delete():
+def test_permission_delete(mocker):
+    with message(mocker, "permission_deleted", permission="wiki.main"):
         permission_delete("wiki.main", force=True)
 
     res = user_permission_list()['permissions']
@@ -245,12 +255,14 @@ def test_permission_delete():
 # Error on create - remove function
 #
 
-def test_permission_create_already_existing():
+
-    with pytest.raises(YunohostError):
+def test_permission_create_already_existing(mocker):
+    with raiseYunohostError(mocker, "permission_already_exist"):
         permission_create("wiki.main")
 
-def test_permission_delete_doesnt_existing():
+
-    with pytest.raises(YunohostError):
+def test_permission_delete_doesnt_existing(mocker):
+    with raiseYunohostError(mocker, "permission_not_found"):
         permission_delete("doesnt.exist", force=True)
 
     res = user_permission_list()['permissions']
@@ -259,8 +271,9 @@ def test_permission_delete_doesnt_existing():
     assert "mail.main" in res
     assert "xmpp.main" in res
 
-def test_permission_delete_main_without_force():
+
-    with pytest.raises(YunohostError):
+def test_permission_delete_main_without_force(mocker):
+    with raiseYunohostError(mocker, "permission_cannot_remove_main"):
         permission_delete("blog.main")
 
     res = user_permission_list()['permissions']
@@ -272,44 +285,65 @@ def test_permission_delete_main_without_force():
 
 # user side functions
 
-def test_permission_add_group():
+
+def test_permission_add_group(mocker):
+    with message(mocker, "permission_updated", permission="wiki.main"):
         user_permission_update("wiki.main", add="alice")
 
     res = user_permission_list(full=True)['permissions']
     assert set(res['wiki.main']['allowed']) == set(["all_users", "alice"])
     assert set(res['wiki.main']['corresponding_users']) == set(["alice", "bob"])
 
-def test_permission_remove_group():
+
+def test_permission_remove_group(mocker):
+    with message(mocker, "permission_updated", permission="blog.main"):
         user_permission_update("blog.main", remove="alice")
 
     res = user_permission_list(full=True)['permissions']
     assert res['blog.main']['allowed'] == []
     assert res['blog.main']['corresponding_users'] == []
 
-def test_permission_add_and_remove_group():
+
+def test_permission_add_and_remove_group(mocker):
+    with message(mocker, "permission_updated", permission="wiki.main"):
         user_permission_update("wiki.main", add="alice", remove="all_users")
 
     res = user_permission_list(full=True)['permissions']
     assert res['wiki.main']['allowed'] == ["alice"]
     assert res['wiki.main']['corresponding_users'] == ["alice"]
 
-def test_permission_add_group_already_allowed():
+
+def test_permission_add_group_already_allowed(mocker):
+    with message(mocker, "permission_already_allowed", permission="blog.main", group="alice"):
         user_permission_update("blog.main", add="alice")
 
     res = user_permission_list(full=True)['permissions']
     assert res['blog.main']['allowed'] == ["alice"]
     assert res['blog.main']['corresponding_users'] == ["alice"]
 
-def test_permission_remove_group_already_not_allowed():
+
+def test_permission_remove_group_already_not_allowed(mocker):
+    with message(mocker, "permission_already_disallowed", permission="blog.main", group="bob"):
         user_permission_update("blog.main", remove="bob")
 
     res = user_permission_list(full=True)['permissions']
     assert res['blog.main']['allowed'] == ["alice"]
     assert res['blog.main']['corresponding_users'] == ["alice"]
 
-def test_permission_reset():
+
+def test_permission_reset(mocker):
+    with message(mocker, "permission_updated", permission="blog.main"):
+        user_permission_reset("blog.main")
+
+    res = user_permission_list(full=True)['permissions']
+    assert res['blog.main']['allowed'] == ["all_users"]
+    assert set(res['blog.main']['corresponding_users']) == set(["alice", "bob"])
+
+
+def test_permission_reset_idempotency():
     # Reset permission
     user_permission_reset("blog.main")
+    user_permission_reset("blog.main")
 
     res = user_permission_list(full=True)['permissions']
     assert res['blog.main']['allowed'] == ["all_users"]
@@ -330,18 +364,21 @@ def test_permission_reset_idempotency():
 # Error on update function
 #
 
-def test_permission_add_group_that_doesnt_exist():
+
-    with pytest.raises(YunohostError):
+def test_permission_add_group_that_doesnt_exist(mocker):
+    with raiseYunohostError(mocker, "group_unknown"):
         user_permission_update("blog.main", add="doesnt_exist")
 
     res = user_permission_list(full=True)['permissions']
     assert res['blog.main']['allowed'] == ["alice"]
     assert res['blog.main']['corresponding_users'] == ["alice"]
 
-def test_permission_update_permission_that_doesnt_exist():
+
-    with pytest.raises(YunohostError):
+def test_permission_update_permission_that_doesnt_exist(mocker):
+    with raiseYunohostError(mocker, "permission_not_found"):
         user_permission_update("doesnt.exist", add="alice")
 
+
 # Permission url management
 
 def test_permission_redefine_url():
@@ -360,6 +397,7 @@ def test_permission_remove_url():
 # Application interaction
 #
 
+
 def test_permission_app_install():
     app_install("./tests/apps/permissions_app_ynh",
                 args="domain=%s&path=%s&is_public=0&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
@@ -397,6 +435,7 @@ def test_permission_app_remove():
     res = user_permission_list(full=True)['permissions']
     assert not any(p.startswith("permissions_app.") for p in res.keys())
 
+
 def test_permission_app_change_url():
     app_install("./tests/apps/permissions_app_ynh",
                 args="domain=%s&path=%s&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
@@ -443,6 +482,7 @@ def test_permission_app_propagation_on_ssowat():
     assert can_access_webpage(app_webroot+"/admin", logged_as="alice")
     assert not can_access_webpage(app_webroot+"/admin", logged_as="bob")
 
+
 def test_permission_legacy_app_propagation_on_ssowat():
 
     app_install("./tests/apps/legacy_app_ynh",

src/yunohost/tests/test_user-group.py

@@ -1,14 +1,16 @@
 import pytest
 
+from conftest import message, raiseYunohostError
+
 from yunohost.user import user_list, user_info, user_create, user_delete, user_update, \
-                          user_group_list, user_group_create, user_group_delete, user_group_update, user_group_info
+                          user_group_list, user_group_create, user_group_delete, user_group_update
 from yunohost.domain import _get_maindomain
-from yunohost.utils.error import YunohostError
 from yunohost.tests.test_permission import check_LDAP_db_integrity
 
 # Get main domain
 maindomain = _get_maindomain()
 
+
 def clean_user_groups():
     for u in user_list()['users']:
         user_delete(u)
@@ -17,6 +19,7 @@ def clean_user_groups():
         if g not in ["all_users", "visitors"]:
             user_group_delete(g)
 
+
 def setup_function(function):
     clean_user_groups()
 
@@ -29,9 +32,11 @@ def setup_function(function):
     user_group_update("dev", add=["alice"])
     user_group_update("apps", add=["bob"])
 
+
 def teardown_function(function):
     clean_user_groups()
 
+
 @pytest.fixture(autouse=True)
 def check_LDAP_db_integrity_call():
     check_LDAP_db_integrity()
@@ -42,6 +47,7 @@ def check_LDAP_db_integrity_call():
 # List functions
 #
 
+
 def test_list_users():
     res = user_list()['users']
 
@@ -49,6 +55,7 @@ def test_list_users():
     assert "bob" in res
     assert "jack" in res
 
+
 def test_list_groups():
     res = user_group_list()['groups']
 
@@ -65,7 +72,10 @@ def test_list_groups():
 # Create - Remove functions
 #
 
-def test_create_user():
+
+def test_create_user(mocker):
+
+    with message(mocker, "user_created"):
         user_create("albert", "Albert", "Good", "alber@" + maindomain, "test123Ynh")
 
     group_res = user_group_list()['groups']
@@ -74,7 +84,10 @@ def test_create_user():
     assert "albert" in group_res['albert']['members']
     assert "albert" in group_res['all_users']['members']
 
-def test_del_user():
+
+def test_del_user(mocker):
+
+    with message(mocker, "user_deleted"):
         user_delete("alice")
 
     group_res = user_group_list()['groups']
@@ -82,7 +95,10 @@ def test_del_user():
     assert "alice" not in group_res
     assert "alice" not in group_res['all_users']['members']
 
-def test_create_group():
+
+def test_create_group(mocker):
+
+    with message(mocker, "group_created", group="adminsys"):
         user_group_create("adminsys")
 
     group_res = user_group_list()['groups']
@@ -90,7 +106,10 @@ def test_create_group():
     assert "members" in group_res['adminsys'].keys()
     assert group_res["adminsys"]["members"] == []
 
-def test_del_group():
+
+def test_del_group(mocker):
+
+    with message(mocker, "group_deleted", group="dev"):
         user_group_delete("dev")
 
     group_res = user_group_list()['groups']
@@ -100,74 +119,93 @@ def test_del_group():
 # Error on create / remove function
 #
 
-def test_create_user_with_mail_address_already_taken():
+
-    with pytest.raises(YunohostError):
+def test_create_user_with_mail_address_already_taken(mocker):
+    with raiseYunohostError(mocker, "user_creation_failed"):
         user_create("alice2", "Alice", "White", "alice@" + maindomain, "test123Ynh")
 
-def test_create_user_with_password_too_simple():
+
-    with pytest.raises(YunohostError):
+def test_create_user_with_password_too_simple(mocker):
+    with raiseYunohostError(mocker, "password_listed"):
         user_create("other", "Alice", "White", "other@" + maindomain, "12")
 
-def test_create_user_already_exists():
+
-    with pytest.raises(YunohostError):
+def test_create_user_already_exists(mocker):
+    with raiseYunohostError(mocker, "user_already_exists"):
         user_create("alice", "Alice", "White", "other@" + maindomain, "test123Ynh")
 
-def test_update_user_with_mail_address_already_taken():
+
-    with pytest.raises(YunohostError):
+def test_update_user_with_mail_address_already_taken(mocker):
+    with raiseYunohostError(mocker, "user_update_failed"):
         user_update("bob", add_mailalias="alice@" + maindomain)
 
-def test_del_user_that_does_not_exist():
+
-    with pytest.raises(YunohostError):
+def test_del_user_that_does_not_exist(mocker):
+    with raiseYunohostError(mocker, "user_unknown"):
         user_delete("doesnt_exist")
 
-def test_create_group_all_users():
+
+def test_create_group_all_users(mocker):
     # Check groups already exist with special group "all_users"
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, "group_already_exist"):
         user_group_create("all_users")
 
-def test_create_group_already_exists():
+
+def test_create_group_already_exists(mocker):
     # Check groups already exist (regular groups)
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, "group_already_exist"):
         user_group_create("dev")
 
-def test_del_group_all_users():
+
-    with pytest.raises(YunohostError):
+def test_del_group_all_users(mocker):
+    with raiseYunohostError(mocker, "group_cannot_be_deleted"):
         user_group_delete("all_users")
 
-def test_del_group_that_does_not_exist():
+
-    with pytest.raises(YunohostError):
+def test_del_group_that_does_not_exist(mocker):
+    with raiseYunohostError(mocker, "group_unknown"):
         user_group_delete("doesnt_exist")
 
 #
 # Update function
 #
 
-def test_update_user():
+
+def test_update_user(mocker):
+    with message(mocker, "user_updated"):
         user_update("alice", firstname="NewName", lastname="NewLast")
 
     info = user_info("alice")
     assert info['firstname'] == "NewName"
     assert info['lastname'] == "NewLast"
 
-def test_update_group_add_user():
+
+def test_update_group_add_user(mocker):
+    with message(mocker, "group_updated", group="dev"):
         user_group_update("dev", add=["bob"])
 
     group_res = user_group_list()['groups']
     assert set(group_res['dev']['members']) == set(["alice", "bob"])
 
-def test_update_group_add_user_already_in():
+
+def test_update_group_add_user_already_in(mocker):
+    with message(mocker, "group_user_already_in_group", user="bob", group="apps"):
         user_group_update("apps", add=["bob"])
 
     group_res = user_group_list()['groups']
     assert group_res['apps']['members'] == ["bob"]
 
-def test_update_group_remove_user():
+
+def test_update_group_remove_user(mocker):
+    with message(mocker, "group_updated", group="apps"):
         user_group_update("apps", remove=["bob"])
 
     group_res = user_group_list()['groups']
     assert group_res['apps']['members'] == []
 
-def test_update_group_remove_user_not_already_in():
+
+def test_update_group_remove_user_not_already_in(mocker):
+    with message(mocker, "group_user_not_in_group", user="jack", group="apps"):
         user_group_update("apps", remove=["jack"])
 
     group_res = user_group_list()['groups']
@@ -177,29 +215,33 @@ def test_update_group_remove_user_not_already_in():
 # Error on update functions
 #
 
-def test_update_user_that_doesnt_exist():
+
-    with pytest.raises(YunohostError):
+def test_update_user_that_doesnt_exist(mocker):
+    with raiseYunohostError(mocker, "user_unknown"):
         user_update("doesnt_exist", firstname="NewName", lastname="NewLast")
 
-def test_update_group_that_doesnt_exist():
+
-    # Check groups not found
+def test_update_group_that_doesnt_exist(mocker):
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, "group_unknown"):
         user_group_update("doesnt_exist", add=["alice"])
 
-def test_update_group_all_users_manually():
+
-    with pytest.raises(YunohostError):
+def test_update_group_all_users_manually(mocker):
+    with raiseYunohostError(mocker, "group_cannot_edit_all_users"):
         user_group_update("all_users", remove=["alice"])
 
     assert "alice" in user_group_list()["groups"]["all_users"]["members"]
 
-def test_update_group_primary_manually():
+
-    with pytest.raises(YunohostError):
+def test_update_group_primary_manually(mocker):
+    with raiseYunohostError(mocker, "group_cannot_edit_primary_group"):
         user_group_update("alice", remove=["alice"])
+
     assert "alice" in user_group_list()["groups"]["alice"]["members"]
 
-def test_update_group_add_user_that_doesnt_exist():
+
-    # Check add bad user in group
+def test_update_group_add_user_that_doesnt_exist(mocker):
-    with pytest.raises(YunohostError):
+    with raiseYunohostError(mocker, "user_unknown"):
         user_group_update("dev", add=["doesnt_exist"])
 
     assert "doesnt_exist" not in user_group_list()["groups"]["dev"]["members"]

src/yunohost/utils/error.py

@@ -33,6 +33,8 @@ class YunohostError(MoulinetteError):
     """
 
     def __init__(self, key, raw_msg=False, *args, **kwargs):
+        self.key = key # Saving the key is useful for unit testing
+        self.kwargs = kwargs # Saving the key is useful for unit testing
         if raw_msg:
             msg = key
         else: