diff --git a/locales/en.json b/locales/en.json
index 20a0c7703..0d80565a7 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -498,6 +498,7 @@
     "permission_created": "Permission '{permission:s}' created",
     "permission_creation_failed": "Could not create permission '{permission}': {error}",
     "permission_currently_allowed_for_all_users": "This permission is currently granted to all users in addition to other groups. You probably want to either remove the 'all_users' permission or remove the other groups it is currently granted to.",
+    "permission_cant_add_to_all_users": "The permission {permission} can not be added to all users.",
     "permission_deleted": "Permission '{permission:s}' deleted",
     "permission_deletion_failed": "Could not delete permission '{permission}': {error}",
     "permission_not_found": "Permission '{permission:s}' not found",
diff --git a/src/yunohost/permission.py b/src/yunohost/permission.py
index e0a3c6be8..df30b40c4 100644
--- a/src/yunohost/permission.py
+++ b/src/yunohost/permission.py
@@ -184,6 +184,10 @@ def user_permission_update(
     ) and not force:
         raise YunohostValidationError("permission_protected", permission=permission)
 
+    # Refuse to add "all_users" to ssh/sftp permissions
+    if permission.split(".")[0] in ["ssh", "sftp"] and (add and "all_users" in add) and not force:
+        raise YunohostValidationError("permission_cant_add_to_all_users", permission=permission)
+
     # Fetch currently allowed groups for this permission
 
     current_allowed_groups = existing_permission["allowed"]