diff --git a/helpers/apt b/helpers/apt
index a38289dd2..ee352aebe 100644
--- a/helpers/apt
+++ b/helpers/apt
@@ -497,8 +497,14 @@ ynh_remove_extra_repo() {
     ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list"
     # Sury pinning is managed by the regenconf in the core...
     [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name"
-    ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" >/dev/null
-    ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc" >/dev/null
+    if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then
+        ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg"
+    fi
+
+    # (Do we even create a .asc file anywhere ...?)
+    if [ -e /etc/apt/trusted.gpg.d/$name.asc ]; then
+        ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc"
+    fi
 
     # Update the list of package to exclude the old repo
     ynh_package_update